diff --git a/.github/workflows/rust-wire.yml b/.github/workflows/rust-wire.yml new file mode 100644 index 00000000..d282c278 --- /dev/null +++ b/.github/workflows/rust-wire.yml @@ -0,0 +1,101 @@ +name: Rust native conformance +on: + # Daily changes are accepted locally; retain explicit conformance reproduction. + workflow_dispatch: +permissions: + contents: read +concurrency: + group: rust-wire-${{ github.event.pull_request.head.repo.full_name || github.repository }}-${{ github.head_ref || github.ref_name }} + cancel-in-progress: false +jobs: + conformance: + runs-on: ubuntu-latest + timeout-minutes: 30 + services: + postgres: + image: postgres:16 + env: + POSTGRES_USER: msg_test + POSTGRES_PASSWORD: msg_test_local_only + POSTGRES_DB: postgres + ports: ['5432:5432'] + options: >- + --health-cmd "pg_isready -U msg_test -d postgres" + --health-interval 5s --health-timeout 5s --health-retries 12 + env: + MSG_PG_TEST_ADMIN_DSN: 'host=127.0.0.1 port=5432 user=msg_test password=msg_test_local_only dbname=postgres sslmode=disable' + steps: + - uses: actions/checkout@v7 + with: + persist-credentials: false + - uses: actions/setup-python@v7 + with: + python-version: '3.15' + allow-prereleases: true + - name: Install Rust toolchain and Python reference dependencies + run: | + rustup toolchain install 1.85.0 --profile minimal --component rustfmt,clippy + python -m pip install cryptography==46.0.3 jsonschema==4.25.1 ruff==0.16.9 "psycopg[binary]==3.3.6" + - name: Record exact tested revision and environment + run: | + mkdir -p rust/artifacts + git rev-parse HEAD > rust/artifacts/source-commit.txt + git rev-parse HEAD^{tree} > rust/artifacts/source-tree.txt + rustc +1.85.0 --version > rust/artifacts/toolchains.txt + python --version >> rust/artifacts/toolchains.txt + - name: Check Rust formatting without modifying sources + working-directory: rust + run: cargo +1.85.0 fmt --all -- --check + - name: Check Python reference harness style + if: ${{ !cancelled() }} + run: | + python -m ruff check rust/tests + python -m ruff format --check rust/tests + - name: Native tests + if: ${{ !cancelled() }} + working-directory: rust + run: | + set -euo pipefail + cargo +1.85.0 test --locked --workspace --all-targets 2>&1 | tee artifacts/native-tests.txt + cargo +1.85.0 test --locked --workspace --doc 2>&1 | tee artifacts/doc-tests.txt + - name: Clippy + if: ${{ !cancelled() }} + working-directory: rust + run: cargo +1.85.0 clippy --locked --workspace --all-targets -- -D warnings + - name: Python and Rust differential checks + if: ${{ !cancelled() }} + env: + PYTHONPATH: src + run: | + cargo +1.85.0 build --locked --manifest-path rust/Cargo.toml --workspace --examples + python rust/tests/test_python_parity.py --binary rust/target/debug/examples/wire_check --report rust/artifacts/parity.json + python rust/tests/test_identity_parity.py --binary rust/target/debug/examples/authority_check --report rust/artifacts/identity-parity.json + python rust/tests/test_storage_parity.py --binary rust/target/debug/examples/writer_check --report rust/artifacts/storage-parity.json + python rust/tests/test_authorization_parity.py --binary rust/target/debug/examples/authority_check --report rust/artifacts/authorization.json + python rust/tests/test_authorization_parity.py --postgres --binary rust/target/debug/examples/authority_check --report rust/artifacts/authorization-pg.json + python rust/tests/test_postgres_parity.py --binary rust/target/debug/examples/pg_writer_check --report rust/artifacts/postgres.json + python rust/tests/test_postgres_process.py --binary rust/target/debug/examples/pg_writer_check --report rust/artifacts/postgres-process.json + python rust/tests/test_content_parity.py rust/target/debug/examples/content_check --report rust/artifacts/content-debug.json + - name: Optimized native differential checks + if: ${{ !cancelled() }} + env: + PYTHONPATH: src + run: | + cargo +1.85.0 build --release --locked --manifest-path rust/Cargo.toml --workspace --examples + python rust/tests/test_python_parity.py --binary rust/target/release/examples/wire_check --report rust/artifacts/parity-release.json + python rust/tests/test_identity_parity.py --binary rust/target/release/examples/authority_check --report rust/artifacts/identity-parity-release.json + python rust/tests/test_storage_parity.py --binary rust/target/release/examples/writer_check --report rust/artifacts/storage-parity-release.json + python rust/tests/test_authorization_parity.py --binary rust/target/release/examples/authority_check --report rust/artifacts/authorization-release.json + python rust/tests/test_authorization_parity.py --postgres --binary rust/target/release/examples/authority_check --report rust/artifacts/authorization-pg-release.json + python rust/tests/test_postgres_parity.py --binary rust/target/release/examples/pg_writer_check --report rust/artifacts/postgres-release.json + python rust/tests/test_postgres_process.py --binary rust/target/release/examples/pg_writer_check --report rust/artifacts/postgres-process-release.json + python rust/tests/test_content_parity.py rust/target/release/examples/content_check --report rust/artifacts/content-release.json + - name: Preserve evidence and dependency lock + if: ${{ always() }} + uses: actions/upload-artifact@v7 + with: + name: rust-wire-${{ github.sha }} + path: | + rust/artifacts/ + rust/Cargo.lock + if-no-files-found: warn diff --git a/rust/.gitignore b/rust/.gitignore new file mode 100644 index 00000000..b608df42 --- /dev/null +++ b/rust/.gitignore @@ -0,0 +1,2 @@ +/target/ +/artifacts/ diff --git a/rust/Cargo.lock b/rust/Cargo.lock new file mode 100644 index 00000000..afc9bed7 --- /dev/null +++ b/rust/Cargo.lock @@ -0,0 +1,1538 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "ahash" +version = "0.8.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" +dependencies = [ + "cfg-if", + "once_cell", + "version_check", + "zerocopy", +] + +[[package]] +name = "android_system_properties" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae221649c9976a6f6c56ae1facf410f3ddb33cc661c4b7b61020a912d4237fbc" +dependencies = [ + "libc", +] + +[[package]] +name = "async-trait" +version = "0.1.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82f6aeea286b8eb4dd3431a1be1b59d290ace00f5bfd8e2a159bc2a05e2c1667" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.6", +] + +[[package]] +name = "autocfg" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" + +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + +[[package]] +name = "base64ct" +version = "1.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06" + +[[package]] +name = "bitflags" +version = "2.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06" + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "block-buffer" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2f6c7dbe95a6ed67ad9f18e57daf93a2f034c524b99fd2b76d18fdfeb6660aa" +dependencies = [ + "hybrid-array", +] + +[[package]] +name = "bumpalo" +version = "3.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" + +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + +[[package]] +name = "bytes" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" + +[[package]] +name = "cc" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f74872d07caf508b30a21f6836e7d7016a2eaf7d9ff4f48deaa58cd8a0407630" +dependencies = [ + "find-msvc-tools", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600" + +[[package]] +name = "chacha20" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65c35e4b699c7e15ccbe7ee35c005e4fc0a278d22238a2857e6ce2dadeda1b06" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.1", + "rand_core 0.10.1", +] + +[[package]] +name = "chrono" +version = "0.4.42" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "145052bdd345b87320e369255277e3fb5152762ad123a901ef5c262dd38fe8d2" +dependencies = [ + "iana-time-zone", + "num-traits", + "windows-link", +] + +[[package]] +name = "cmov" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c9ea0ac24bc397ab3c98583a3c9ba74fa56b09a4449bbe172b9b1ddb016027a" + +[[package]] +name = "const-oid" +version = "0.9.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" + +[[package]] +name = "const-oid" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6ef517f0926dd24a1582492c791b6a4818a4d94e789a334894aa15b0d12f55c" + +[[package]] +name = "core-foundation" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + +[[package]] +name = "cpufeatures" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ca28b0ae3115b884660db4118d803791fd6756b6e88f39c0f3f7859060d7566" +dependencies = [ + "libc", +] + +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "crypto-common" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453" +dependencies = [ + "hybrid-array", +] + +[[package]] +name = "ctutils" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d5515a3834141de9eafb9717ad39eea8247b5674e6066c404e8c4b365d2a29e" +dependencies = [ + "cmov", +] + +[[package]] +name = "curve25519-dalek" +version = "4.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97fb8b7c4503de7d6ae7b42ab72a5a59857b4c937ec27a3d4539dba95b5ab2be" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "curve25519-dalek-derive", + "digest 0.10.7", + "fiat-crypto", + "rustc_version", + "subtle", + "zeroize", +] + +[[package]] +name = "curve25519-dalek-derive" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "der" +version = "0.7.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7c1832837b905bbfb5101e07cc24c8deddf52f93225eee6ead5f4d63d53ddcb" +dependencies = [ + "const-oid 0.9.6", + "zeroize", +] + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer 0.10.4", + "crypto-common 0.1.7", +] + +[[package]] +name = "digest" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" +dependencies = [ + "block-buffer 0.12.1", + "const-oid 0.10.2", + "crypto-common 0.2.2", + "ctutils", +] + +[[package]] +name = "ed25519" +version = "2.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "115531babc129696a58c64a4fef0a8bf9e9698629fb97e9e40767d235cfbcd53" +dependencies = [ + "pkcs8", + "signature", +] + +[[package]] +name = "ed25519-dalek" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70e796c081cee67dc755e1a36a0a172b897fab85fc3f6bc48307991f64e4eca9" +dependencies = [ + "curve25519-dalek", + "ed25519", + "serde", + "sha2 0.10.9", + "subtle", + "zeroize", +] + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys", +] + +[[package]] +name = "fallible-iterator" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7" + +[[package]] +name = "fallible-iterator" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2acce4a10f12dc2fb14a218589d4f1f62ef011b2d0cc4b3cb1bba8e94da14649" + +[[package]] +name = "fallible-streaming-iterator" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a" + +[[package]] +name = "fastrand" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" + +[[package]] +name = "fiat-crypto" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28dea519a9695b9977216879a3ebfddf92f1c08c05d984f8996aecd6ecdc811d" + +[[package]] +name = "find-msvc-tools" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aedcfb3409746eddb02b9e19ebda1c3394f759a152e48ee875a0844d1b955484" + +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + +[[package]] +name = "fs2" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9564fc758e15025b46aa6643b1b77d047d1a56a1aea6e01002ac0c7026876213" +dependencies = [ + "libc", + "winapi", +] + +[[package]] +name = "futures-channel" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1f9e3d69d39e4862ffed03ed071a76f9a13ba1d9109d355b0f0aa6b15e393c4" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e" + +[[package]] +name = "futures-sink" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1944426bf7d03f1d14f708785e4b33efd750b36d48a157b836b3efc15ede8e1d" + +[[package]] +name = "futures-task" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd" + +[[package]] +name = "futures-util" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc" +dependencies = [ + "futures-core", + "futures-sink", + "futures-task", + "pin-project-lite", + "slab", +] + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "libc", + "wasi 0.11.1+wasi-snapshot-preview1", +] + +[[package]] +name = "getrandom" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +dependencies = [ + "cfg-if", + "libc", + "r-efi", + "rand_core 0.10.1", +] + +[[package]] +name = "hashbrown" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" +dependencies = [ + "ahash", +] + +[[package]] +name = "hashlink" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ba4ff7128dee98c7dc9794b6a411377e1404dba1c97deb8d1a55297bd25d8af" +dependencies = [ + "hashbrown", +] + +[[package]] +name = "hmac" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6303bc9732ae41b04cb554b844a762b4115a61bfaa81e3e83050991eeb56863f" +dependencies = [ + "digest 0.11.3", +] + +[[package]] +name = "hybrid-array" +version = "0.4.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27f864f10dfb56725ce5ce5472bc52252c8f93a4ab86327122cebf62c5f59a17" +dependencies = [ + "typenum", +] + +[[package]] +name = "iana-time-zone" +version = "0.1.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "log", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "js-sys" +version = "0.3.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7883d941dae510fb2d978fc3fe018c71c9e2892fd38854de3e8b92c2e5ad9cc5" +dependencies = [ + "cfg-if", + "futures-util", + "wasm-bindgen", +] + +[[package]] +name = "libc" +version = "0.2.190" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce5d3ddc6d3fa000eb1536d85e147bfe31aacaba692ed6a876f95cb7c855be78" + +[[package]] +name = "libredox" +version = "0.1.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61ff90caf6077a803a240f62fdbe88645a890bbca49ef8174c3cb0404362171d" +dependencies = [ + "libc", +] + +[[package]] +name = "libsqlite3-sys" +version = "0.30.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e99fb7a497b1e3339bc746195567ed8d3e24945ecd636e3619d20b9de9e9149" +dependencies = [ + "cc", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + +[[package]] +name = "lock_api" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" +dependencies = [ + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" + +[[package]] +name = "md-5" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69b6441f590336821bb897fb28fc622898ccceb1d6cea3fde5ea86b090c4de98" +dependencies = [ + "cfg-if", + "digest 0.11.3", +] + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "mio" +version = "1.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1788edb87fdc09c7e26304471e2f5be8cdefb1b6930d6e3985fc02ff53bf86ee" +dependencies = [ + "libc", + "wasi 0.11.1+wasi-snapshot-preview1", + "windows-sys", +] + +[[package]] +name = "msg-core" +version = "0.1.0" +dependencies = [ + "base64", + "ryu", + "serde", + "serde_json", + "sha2 0.10.9", +] + +[[package]] +name = "msg-crypto" +version = "0.1.0" +dependencies = [ + "ed25519-dalek", + "msg-core", + "serde", + "sha2 0.10.9", +] + +[[package]] +name = "msg-identity" +version = "0.1.0" +dependencies = [ + "chrono", + "msg-core", + "msg-crypto", + "msg-protocol", + "serde", + "serde_json", + "sha2 0.10.9", + "subtle", +] + +[[package]] +name = "msg-protocol" +version = "0.1.0" +dependencies = [ + "chrono", + "msg-core", + "msg-crypto", + "serde_json", +] + +[[package]] +name = "msg-storage" +version = "0.1.0" +dependencies = [ + "bytes", + "fs2", + "libc", + "msg-core", + "msg-crypto", + "msg-identity", + "msg-protocol", + "native-tls", + "postgres", + "postgres-native-tls", + "rusqlite", + "serde", + "serde_json", + "sha2 0.10.9", +] + +[[package]] +name = "native-tls" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87de3442987e9dbec73158d5c715e7ad9072fda936bb03d19d7fa10e00520f0e" +dependencies = [ + "libc", + "log", + "openssl", + "openssl-probe", + "openssl-sys", + "schannel", + "security-framework", + "security-framework-sys", + "tempfile", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + +[[package]] +name = "objc2-core-foundation" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a180dd8642fa45cdb7dd721cd4c11b1cadd4929ce112ebd8b9f5803cc79d536" +dependencies = [ + "bitflags", +] + +[[package]] +name = "objc2-system-configuration" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7216bd11cbda54ccabcab84d523dc93b858ec75ecfb3a7d89513fa22464da396" +dependencies = [ + "objc2-core-foundation", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "openssl" +version = "0.10.75" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08838db121398ad17ab8531ce9de97b244589089e290a384c900cb9ff7434328" +dependencies = [ + "bitflags", + "cfg-if", + "foreign-types", + "libc", + "once_cell", + "openssl-macros", + "openssl-sys", +] + +[[package]] +name = "openssl-macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "openssl-probe" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d05e27ee213611ffe7d6348b942e8f942b37114c00cc03cec254295a4a17852e" + +[[package]] +name = "openssl-sys" +version = "0.9.117" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b47e7e6bb2c38cd930d25a23b40fa52e068c10e85f3e03a7f5ba5aaca5713695" +dependencies = [ + "cc", + "libc", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "parking_lot" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-link", +] + +[[package]] +name = "percent-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + +[[package]] +name = "phf" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1562dc717473dbaa4c1f85a36410e03c047b2e7df7f45ee938fbef64ae7fadf" +dependencies = [ + "phf_shared", + "serde", +] + +[[package]] +name = "phf_shared" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e57fef6bc5981e38c2ce2d63bfa546861309f875b8a75f092d1d54ae2d64f266" +dependencies = [ + "siphasher", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "pkcs8" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" +dependencies = [ + "der", + "spki", +] + +[[package]] +name = "pkg-config" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6b464fbc74e149a392436b17d523f769e057cb6877f6a5c4618bc6f11800548" + +[[package]] +name = "postgres" +version = "0.19.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7c48ece1c6cda0db61b058c1721378da76855140e9214339fa1317decacb176" +dependencies = [ + "bytes", + "fallible-iterator 0.2.0", + "futures-util", + "log", + "tokio", + "tokio-postgres", +] + +[[package]] +name = "postgres-native-tls" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac73153d92e4bde922bd6f1dfba7f1ab8132266c031153b55e20a1521cd36d49" +dependencies = [ + "native-tls", + "tokio", + "tokio-native-tls", + "tokio-postgres", +] + +[[package]] +name = "postgres-protocol" +version = "0.6.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08808e3c483c46e999108051c78334f473d5adb59d78bb80a1268c7e6aa6c514" +dependencies = [ + "base64", + "byteorder", + "bytes", + "fallible-iterator 0.2.0", + "hmac", + "md-5", + "memchr", + "rand", + "sha2 0.11.0", + "stringprep", +] + +[[package]] +name = "postgres-types" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "851ca9db4932932d69f3ea811b1abe63087a0f740a47692619dd40d4899b68be" +dependencies = [ + "bytes", + "chrono", + "fallible-iterator 0.2.0", + "postgres-protocol", + "serde_core", + "serde_json", +] + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "rand" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65c9fb96cbc91e3478eaae79a69fcd3f1ae4ad052e471fe6732fff548984b4af" +dependencies = [ + "chacha20", + "getrandom 0.4.3", + "rand_core 0.10.1", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom 0.2.17", +] + +[[package]] +name = "rand_core" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" + +[[package]] +name = "redox_syscall" +version = "0.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" +dependencies = [ + "bitflags", +] + +[[package]] +name = "rusqlite" +version = "0.32.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7753b721174eb8ff87a9a0e799e2d7bc3749323e773db92e0984debb00019d6e" +dependencies = [ + "bitflags", + "fallible-iterator 0.3.0", + "fallible-streaming-iterator", + "hashlink", + "libsqlite3-sys", + "smallvec", +] + +[[package]] +name = "rustc_version" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" +dependencies = [ + "semver", +] + +[[package]] +name = "rustix" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "891efababe418670775f199f0d233d84843c227a0949a883ce15b37c78d6629d" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys", +] + +[[package]] +name = "rustversion" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" + +[[package]] +name = "ryu" +version = "1.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28d3b2b1366ec20994f1fd18c3c594f05c5dd4bc44d8bb0c1c632c8d6829481f" + +[[package]] +name = "schannel" +version = "0.1.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91c1b7e4904c873ef0710c1f407dde2e6287de2bebc1bbbf7d430bb7cbffd939" +dependencies = [ + "windows-sys", +] + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "security-framework" +version = "2.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02" +dependencies = [ + "bitflags", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2691df843ecc5d231c0b14ece2acc3efb62c0a398c7e1d875f3983ce020e3" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "semver" +version = "1.0.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" + +[[package]] +name = "serde" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "serde_json" +version = "1.0.145" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "402a6f66d8c709116cf22f558eab210f5a50187f702eb4d7e5ef38d9a7f1c79c" +dependencies = [ + "itoa", + "memchr", + "ryu", + "serde", + "serde_core", +] + +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "digest 0.10.7", +] + +[[package]] +name = "sha2" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "446ba717509524cb3f22f17ecc096f10f4822d76ab5c0b9822c5f9c284e825f4" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.1", + "digest 0.11.3", +] + +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + +[[package]] +name = "signature" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de" +dependencies = [ + "rand_core 0.6.4", +] + +[[package]] +name = "siphasher" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33f4fe9184a62d842c9ef383018f3306d8ba224fd9d836f56d7288308847c256" + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "smallvec" +version = "1.16.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9395f0f0eee849a9b707b2f06bb92a6a422090e2123bb2ef8e87a0e61892a8e" + +[[package]] +name = "socket2" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d1e2c7f27f8d4cb10542a02c49005dbd6e93095799d6f3be745fae9f8fedd4" +dependencies = [ + "libc", + "windows-sys", +] + +[[package]] +name = "spki" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d" +dependencies = [ + "base64ct", + "der", +] + +[[package]] +name = "stringprep" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b4df3d392d81bd458a8a621b8bffbd2302a12ffe288a9d931670948749463b1" +dependencies = [ + "unicode-bidi", + "unicode-normalization", + "unicode-properties", +] + +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "3.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8593e8e72159ed2257d083c7a454a85cbf854f37a0966d8d483aff8c8a3ebcee" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom 0.4.3", + "once_cell", + "rustix", + "windows-sys", +] + +[[package]] +name = "tinyvec" +version = "1.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd3ca314f692efd6c868f8408f53fe444634a845f96c028b97d35f6a1f79f0ee" + +[[package]] +name = "tokio" +version = "1.53.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e95f91fcc7a621e8b030f6aa23c71fe9838ae2fb4d8118b75602a328f5144044" +dependencies = [ + "bytes", + "libc", + "mio", + "pin-project-lite", + "socket2", + "windows-sys", +] + +[[package]] +name = "tokio-native-tls" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" +dependencies = [ + "native-tls", + "tokio", +] + +[[package]] +name = "tokio-postgres" +version = "0.7.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a528f7d280f6d5b9cd149635c8705b0dd049754bc67d81d31fa25169a93809d3" +dependencies = [ + "async-trait", + "byteorder", + "bytes", + "fallible-iterator 0.2.0", + "futures-channel", + "futures-util", + "log", + "parking_lot", + "percent-encoding", + "phf", + "pin-project-lite", + "postgres-protocol", + "postgres-types", + "rand", + "socket2", + "tokio", + "tokio-util", + "whoami", +] + +[[package]] +name = "tokio-util" +version = "0.7.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "494815d09bf52b5548659851081238f0ca39ff638363907596da739561c62c52" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "libc", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "typenum" +version = "1.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + +[[package]] +name = "unicode-bidi" +version = "0.3.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c1cb5db39152898a79168971543b1cb5020dff7fe43c8dc468b0885f5e29df5" + +[[package]] +name = "unicode-ident" +version = "1.0.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d245f478577f809a851594d02313b640fb437e0bb33866753cff937863096954" + +[[package]] +name = "unicode-normalization" +version = "0.1.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fd4f6878c9cb28d874b009da9e8d183b5abc80117c40bbd187a1fde336be6e8" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "unicode-properties" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7df058c713841ad818f1dc5d3fd88063241cc61f49f5fbea4b951e8cf5a8d71d" + +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "wasi" +version = "0.14.7+wasi-0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "883478de20367e224c0090af9cf5f9fa85bed63a95c1abf3afc5c083ebc06e8c" +dependencies = [ + "wasip2", +] + +[[package]] +name = "wasip2" +version = "1.0.1+wasi-0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0562428422c63773dad2c345a1882263bbf4d65cf3f42e90921f787ef5ad58e7" +dependencies = [ + "wit-bindgen", +] + +[[package]] +name = "wasite" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66fe902b4a6b8028a753d5424909b764ccf79b7a209eac9bf97e59cda9f71a42" +dependencies = [ + "wasi 0.14.7+wasi-0.2.4", +] + +[[package]] +name = "wasm-bindgen" +version = "0.2.129" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9bb54f33acc68fd454578d9820b0bde1a1a3d17aa17bb7b6595806d02886d409" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.129" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e29d0c35b16e224a7eeb5cd2d25e3e1968fbd65604117b44d3b789d00ee8535" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.129" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f501a8bc3719dba86ef8ae4728879c08001bea749eb1333ac5b91e040e2a6b7" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn 3.0.6", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.129" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23f0c9c52aa7cd7d77769a4cfe2a9adb1b331f489a41d912ce14513d5ab995c6" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "web-sys" +version = "0.3.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88261b9deccee56594c11a3460c462c41f58d148598fe70ad77070126a68aba4" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "whoami" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "626c4bac6755d76ffc12cb01b2eac751db1996b9e0041de9aa02c8c211ddc82c" +dependencies = [ + "libc", + "libredox", + "objc2-system-configuration", + "wasite", + "web-sys", +] + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows-core" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-implement" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "windows-interface" +version = "0.59.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-result" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-strings" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "wit-bindgen" +version = "0.46.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f17a85883d4e6d00e8a97c586de764dabcc06133f7f1d55dce5cdc070ad7fe59" + +[[package]] +name = "zerocopy" +version = "0.8.59" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6df92bf3d9227be3d53173901ddbffac2babc27ae50f397776ffd6dc33f800cb" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.59" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac4f328cf2f05d084e496c3e9c3f33ed0a183656a16e1fcec4d464d8373aec82" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "zeroize" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e" diff --git a/rust/Cargo.toml b/rust/Cargo.toml new file mode 100644 index 00000000..e9b953c1 --- /dev/null +++ b/rust/Cargo.toml @@ -0,0 +1,36 @@ +[workspace] +members = ["crates/msg-core", "crates/msg-crypto", "crates/msg-protocol", "crates/msg-identity", "crates/msg-storage"] +resolver = "2" + +[workspace.package] +version = "0.1.0" +edition = "2021" +rust-version = "1.85" +license = "MIT" +publish = false + +[workspace.dependencies] +base64 = "=0.22.1" +chrono = { version = "=0.4.42", default-features = false, features = ["std"] } +ed25519-dalek = "=2.2.0" +ryu = "=1.0.20" +serde = { version = "=1.0.228", features = ["derive"] } +serde_json = { version = "=1.0.145", features = ["raw_value", "arbitrary_precision"] } +sha2 = "=0.10.9" +subtle = "=2.6.1" +fs2 = "=0.4.3" +libc = "=0.2.190" +rusqlite = { version = "=0.32.1", features = ["bundled"] } + +postgres = { version = "=0.19.12", features = ["with-serde_json-1", "with-chrono-0_4"] } +postgres-native-tls = "=0.5.2" +native-tls = "=0.2.14" +bytes = "=1.12.1" + +[workspace.lints.rust] +unsafe_code = "forbid" + +[profile.release] +lto = "thin" +codegen-units = 1 +strip = "debuginfo" diff --git a/rust/MIGRATION.md b/rust/MIGRATION.md new file mode 100644 index 00000000..64f557f6 --- /dev/null +++ b/rust/MIGRATION.md @@ -0,0 +1,81 @@ +# Native migration acceptance checklist + +**Completed / remaining: 5 / 6.** This counts the fixed eleven packages below, +not a percentage of effort, code lines, operations or production readiness. +A foundation package is not the corresponding end-to-end server feature. +Keep the IDs stable in later progress reports; split/redefine one only explicitly. + +| ID | Package and completion boundary | Status | +|---|---|---| +| M01 | Strict canonical wire/packet/crypto primitives and Python parity | Implemented | +| M02 | Transaction-bound identity admission, certificate and resource-policy primitives; not issuance handlers | Implemented | +| M03 | Native existing-schema SQLite metadata sessions, writers and process fencing | Implemented | +| M04 | Native existing-schema PostgreSQL metadata sessions, writers and process fencing | Implemented | +| M05 | Native Git/blob content, ranges, pins and revision graphs; valid-data interoperability | Implemented in slice 5 | +| M06 | LFS shared-object lifecycle and durable metadata/content recovery; no ambiguous-commit data loss | Remaining | +| M07 | Real operation/capability registry, authorized executor, business handlers, receipts and secret delivery | Remaining | +| M08 | Native identity/OAuth/browser-session issuance, rotation, refresh, revocation and encrypted-key wrapping | Remaining | +| M09 | Native `msgd serve`, HTTP/MCP/GET and other supported transports with current contracts | Remaining | +| M10 | Native `msgd worker`, leases/outbox, retries, recovery fences and actual side effects | Remaining | +| M11 | Installable native daemon, isolated cutover/rollback rehearsal, full end-to-end acceptance and equivalent benchmarks | Remaining | + +Python retains `msg` CLI, administrator/migration scripts and developer tooling. +There is still no native production daemon in this PR. Root signing remains local +only. Do not merge/deploy solely because these primitive suites pass. + +## Slice 5 acceptance and boundaries + +- 14 new native content tests; current workspace: 46 unit tests and 5 doc tests. +- 170 Python/Rust content checks per debug/release profile. These include actual + shared storage, 64 KiB boundary ranges, hard-link retention, canonical revision + manifests and ancestry, pin interoperability, corrupted objects, ignored Git + environment/hooks, and process kills before/after publication. +- Existing wire, identity, SQLite writer and SQLite authorization differential + suites remain enabled: respectively 18,447 / 61,734 / 71 / 878 checks. The large + identity count includes the documented simple mode matrix, not independent + end-to-end identity scenarios. PostgreSQL suites remain mandatory in cloud CI. +- `cargo fmt --check`, Ruff check/format, Clippy `-D warnings` and locked builds + remain mandatory. No warning suppression or production Python changes were + used to obtain a native pass. +- CI writes source commit/tree, compiler version, lockfile and per-profile JSON + reports to its artifact. Inspect the actual run for acceptance; a workflow + trigger or a previous commit's green status is not evidence for a new tree. + +Stricter invalid-data behavior is intentional: corrupt content is not delivered, +paths/indices are bounded and checked, reused revision IDs are immutable, and +native commit timestamps are deterministic. This is valid-data compatibility, +not a claim to reproduce every Python error or accept every malformed old record. +Directories are trusted service-owned configuration, not caller-selected paths +or an openat capability sandbox. Full reads use verified disk snapshots; request +admission must account for their disk/I/O/concurrency costs. + +No partial operation may destroy externally published data merely because an SQL +COMMIT acknowledgement was lost. M06/M07 must integrate explicit reconciliation +with the existing writer fence and ledger before serving real requests. Staging +left by SIGKILL is retained intentionally; this slice does not add unsafe GC. + +## Pre-existing full-regression blocker + +Before this slice, Rewrite contracts run **37299218496**, source head +`de810d15da747b7be34b64f4e2324e5f5e7da6f7`, failed all eight started test shards. +Their JUnit artifacts were inspected (4,759 test entries): **32 failures/errors**, +of which **29** end in `recovery_schema_unsupported`. The other three are: + +| Test | Existing failure location | +|---|---| +| `test_default_build_checks_packaged_publication_snapshot` | `tests/test_dictionary.py:178` | +| `test_universe_account_hint_is_minimal_current_identity_without_private_reads` | `tests/test_universe.py:51` | +| `test_accepts_postgres_and_optional_valkey_urls_without_revealing_secrets` | `tests/test_pg_config.py:26` | + +These are observed baseline failures, not fixed by changing native content code. +Do not label a new failure as pre-existing without matching its actual evidence. +The full regression/cutover gate remains open independently of native conformance. + +## Coordination and evidence + +Read repository `AGENTS.md`, the entry skill and deployed `/AGENTS.md` before this +slice. The editor has no selected local MSG signing identity; no authorized +private coordination thread could be created. Use the existing draft PR #235 and +this checklist as the fallback. No production account, public coordination post, +production data write or deployment was created. The temporary public-toolchain +export workflow is removed from the final source tree. diff --git a/rust/README.md b/rust/README.md new file mode 100644 index 00000000..444645d8 --- /dev/null +++ b/rust/README.md @@ -0,0 +1,310 @@ +# Native Rust server migration + +Python keeps the CLI, administrator/migration scripts, and development helpers. +Native Rust code is not yet a replacement for the running Python service. Do not +change production entry points until the remaining executor, transport, content-recovery, +worker, and deployment gates below pass. + +## Fifth implementation slice: native Git/blob content storage + +`msg-storage::content::GitContentStore` ports the existing `private.git`, `index`, +`binary`, `pins` and `revisions` layout without invoking Python. It streams +admitted input through 64 KiB buffers, checks optional SHA-256 expectations before +publication, uses direct Git plumbing for text/JSON/templates, and preserves +existing binary inodes (including LFS hard links). Index replacement and published +objects/ref updates are durable before success; failed input cannot publish an +index. Binary publication is on the destination filesystem, including when +staging is configured elsewhere. + +Reads verify a private snapshot before delivering any requested range. This uses +constant buffered memory but O(blob size) temporary disk and full-object I/O, even +for a small range. The future adapter must enforce admission, disk and concurrency +quotas; this is not a free random-access optimization. Pins use the existing +hashed lease/digest names. Revision trees contain the same canonical manifests, +content and parent commits. Existing Python commits are reused unchanged; newly +created native commits use the revision time for deterministic retries. A reused +revision ID with different content/parents is rejected rather than overwritten. + +Git runs with a cleared environment, no inherited/global Git configuration or +hooks, bounded captured output and a kill-and-wait timeout. Git stdin/stdout use +private files rather than deadlock-prone pipes. Paths/indices reject traversal, +symlinks, non-regular files and malformed object IDs. Configured directories must +be service-owned and protected from hostile writers: this is **not an openat +sandbox against same-UID races**. Existing shared-reader layouts must already +have the required modes; no implicit permission migration is performed. + +The API is synchronous, Unix-only and a **trusted storage primitive**. Callers +must supply byte quotas and retain the metadata writer fence until the call has +actually finished; abandoning a blocking task does not cancel Git. Input-reader +cancellation and request scheduling belong to the still-pending adapters. +No handler, listener, root administration or production entry point is changed. + +Fourteen native tests exercise success, corrupt objects/indices, quotas, failed +input, ranges, hard links, path confinement checks, shared modes, pins, immutable +revision retries and actual Git timeout/reaping. The Python differential suite +has **170 checks per profile**, importing the actual Python store and reading and +writing the same disposable directories. It covers both directions, Git objects, +canonical manifests/parent graphs, cross-language pins, a 4 MiB streamed blob, +hermetic Git and real process kills before publication and after a lost response. +`content_check` is a marker-gated local fixture, never an operation endpoint. + +These tests do **not** implement LFS shared-object publication/collection, a +cross-store metadata/content recovery journal, SIGKILL cleanup or an authorized +executor. Interrupted staging is deliberately retained, not blindly collected. +See [the fixed migration checklist](MIGRATION.md) for completed/remaining scope. + +## Fourth implementation slice: resource authority and PostgreSQL + +`msg-identity::authorization` implements transaction-bound resource policy: +ordinary mode/group checks, ancestors, runtime/quarantine fences, current +certificate ceilings, managed-resource rules, topic roles and bans, direct +conversation participants/blocking, tool gates, and exact resource sharing with +live source-chain and owner/credential checks. Authentication alone is never a +permission to execute. Replayed results must still be checked against current +resource access by the future executor. + +`msg-storage::postgres` opens the existing Python-migrated database without +creating or upgrading any schema. The startup checks cover required columns, +exact primary/unique keys, required foreign keys, and the unique root index. +Each request owns a dedicated connection and repeatable-read snapshot. Writers +hold Python's same session advisory lock until rollback compensation finishes. +Transaction-pooled proxies are not supported. Remote connections require TLS +with certificate and hostname validation, with no plaintext fallback; unsupported +libpq connection options are rejected rather than silently ignored. + +The native PostgreSQL session implements authority reads, canonical resources +and hexadecimal aliases, migration-only old paths, revisions/relations, tags, +optimistic resource updates, identities/memberships/email settings, credentials, +CSRs/certificates, transfer/chunk metadata, events/audit, results, and durable job +records. These typed storage calls are not business operations. They cannot +replace the still-pending authenticated executor, issuance/refresh workflows, +blob/Git recovery or effect runner. + +All typed writes poison the transaction on failure, including validation errors +that a caller catches. Nested savepoints can recover an aborted inner unit only +after successful rollback. A lost COMMIT acknowledgement returns an uncertain +outcome, retains potentially referenced external data, and releases no secrets. +No network-facing handler uses these local test drivers. + +New differential coverage consists of 878 resource-policy checks against each +backend (468 are a mode/identity/check matrix), 102 PostgreSQL record comparisons, +2 additional native safety checks, 13 real process/protocol-failure checks, and +4 malformed-schema checks per build profile. The PostgreSQL tests use actual +Python-created PostgreSQL databases, not a SQLite simulation. A local test-only +wire proxy drops an acknowledged COMMIT to verify real uncertain-response and +same-ID retry behavior. All fixture databases have generated names; the harness +refuses nonlocal servers and never truncates an existing application database. + +The CI workflow repeats the old and new suites in debug and optimized builds, +records the tested source SHA/tree and dependency lock, and uploads the reports. +Independent cloud results must be read before claiming CI acceptance. The +one-time offline editor export workflows are removed. + +## Ownership boundary + +Retain Python `msg` CLI, administration/migration scripts and developer tools. +Migrate server-side `msgd serve`, `msgd worker`, domain/identity, storage, +HTTP/MCP and crypto/protocol to Rust. Do not rename endpoints or invent a second +signature protocol. The tests import the actual Python implementation in the +same checkout, rather than a translated copy of the expected behavior. + +The original inspected main baseline is +`7e16ca75ff6af4313365814c23912b5997b222c5`. This slice extends the phase 3 implementation at +`fffea47606cf` on the existing draft PR #235. + +## Implemented natively + +| Crate | Implemented responsibility | Python reference | +|---|---|---| +| `msg-core` | Bounded strict JSON, duplicate rejection, canonical numbers/Unicode, lossless typed JSON, SHA-256/base64url | `core/codec.py` | +| `msg-crypto` | Existing key/subject IDs, purpose-framed Ed25519 signing/verification | `security/crypto.py` | +| `msg-protocol` | Request shape/defaults, UTC time, business digest/signature bytes, immutable field access | `core/packet.py`, `core/requests.py`, `core/models.py` | +| `msg-identity` | Typed identity/credential/certificate/CSR records, credential and current-state admission, pinned CA chains, live delegation, scope/ceiling/mode primitives, key-approved browser/API binding and token-recovery validation, transaction-bound resource authorization | `security/authentication.py`, `certificates.py`, `policy.py`, `sessions.py` | +| `msg-storage` | Transaction-bound authority reads; SQLite/PostgreSQL writers and typed records, savepoints/rollback effects; native Git/blob content, ranges, pins and revision trees | `storage/sqlite.py`, `storage/postgres.py`, `storage/git.py` and their record/session modules | + +Authentication validates service/digest/expiry, current credentials, SHA-256 +bearer verifiers with constant-time comparison, actor/subject bindings, permitted +entry, custodial limits, attached certificates, archive/quarantine state and a +latched recovery generation. A changed generation stays stale until the service +is reconstructed. Missing, revoked or expired authority fails closed. + +Certificate checks include pinned root bytes/public key, parent signatures and +revocation, current signing keys, scope/grant/constraint narrowing, issuance TTL, +delegation depth, the absolute CA depth limit, restricted online-CA issuance and +live source ownership/revision/credential ceilings. CSR publication checks retain +the original signed bytes, digest and possession proof. A trusted capability +registry projection is injected locally; requests cannot supply authority rules. +Only the existing network-constraint schema is implemented, not a generic schema +engine or plugin loader. + +Key-approved session validation follows current signing parents, captured/current +ceilings, auth versions, nested session revocation and custodial vault status. +The current `credential_states` schema is used by both native stores. Historical +`t_oauth_`/`t_browser_` credentials and login bindings remain rejected even if +old rows are restored. This is **binding validation**, not credential issuance +or an OAuth/MCP server; no removed module, OAuth scope or Bearer authority is revived. The helper's +operation/capability declarations are synthetic test configuration, not a +production registry adapter. + +## Admission is not authorization + +`Principal` has private fields and no deserializer. Request JSON cannot mint one. +`Admission::Fresh` supplies an authenticated identity only. Resource mode, +certificate-gate, traversal, membership, sharing/DM/tool and operation-specific +checks still have to be integrated before executing any operation or exposing +private data. The pure mode/scope helpers do not grant permission on their own. + +`Admission::Replay` contains an already committed result and **no fresh +principal**. An old rotated token or exhausted delegated credential can only +retrieve its exact eligible result; a changed request ID, digest, revoked +successor or wrong result namespace cannot authorize a new handler execution. +Consumed token recovery has the same replay-only boundary. Recovery validation +does not issue credentials or consume a delivery record. + +## Storage safety boundary + +`SqliteAuthority` remains read-only. `SqliteWriter` separately opens an existing, +Python-migrated WAL database. Neither API creates or migrates a database. The +writer checks schema version, required tables/columns, primary and unique keys, +foreign keys, the one-root index and append-only audit guards. It rejects database +symlink/hard-link aliases; all processes must use the same configured file. + +Read snapshots and write sessions share `AuthorityStore` queries inside their +own transaction. JSON columns remain bounded in SQL; secret vault ciphertext is +not selected. Sessions are borrowed and non-Send. Callers have no SQL, COMMIT or +ROLLBACK handle. Only a successful store-owned COMMIT can return a success value. + +The Unix writer uses the same `.writer.lock`/`flock` process fence as Python, +with no-follow opens, mode 0600 on creation and a bounded acquisition deadline. +The fence outlives SQL rollback and every reverse-order rollback effect. Nested +savepoints retain successful inner effects until the outer commit; failed inner +units undo their own data/effects before the caller can recover. An ignored write +error poisons its unit rather than allowing a partial commit or an autocommit +write after SQLite has aborted the transaction. + +Implemented mutation ports: resource insertion and generation-checked replacement +(including immutable creation facts, parent cycles, aliases, tags and authorization +epoch), revisions/relations, settings, result rows with subject/digest binding and +the existing 100,000-row cap, events and the existing canonical audit hash chain. +These ports are **trusted internal storage APIs**, not authorization decisions. +`save_result` does not sign receipts, redact secrets or execute handlers. + +Cleanup errors preserve the primary failure and disable that writer instance. +A failed COMMIT with no live SQLite transaction is treated conservatively as +`commit_outcome_uncertain`: no destructive compensation runs, and the instance +refuses further writes. The future service owner must stop admission, inspect the +result ledger and reconcile external work before reopening. Reopening alone is +not recovery. Durable service-wide quarantine and external-work reconciliation +are not implemented by this slice. + +The earlier process-kill tests cover **SQL metadata**, not a complete operation. +The content tests above additionally cover standalone blob publication. Neither +rollback callbacks nor private staging implement a durable cross-store recovery +journal: callbacks do not run after SIGKILL. Identity/session issuance workflows, outbox/lease +execution and complete operation recovery still need to be ported. PostgreSQL +metadata primitives are already implemented above. Do not introduce dual writes. + +## Reproducible validation + +From the repository root, with Python 3.15 and Rust 1.85.0: + +```sh +python -m pip install cryptography==46.0.3 jsonschema==4.25.1 ruff==0.16.9 +cargo +1.85.0 fmt --manifest-path rust/Cargo.toml --all -- --check +python -m ruff check rust/tests +python -m ruff format --check rust/tests +cargo +1.85.0 test --locked --manifest-path rust/Cargo.toml --workspace --all-targets +cargo +1.85.0 test --locked --manifest-path rust/Cargo.toml --workspace --doc +cargo +1.85.0 clippy --locked --manifest-path rust/Cargo.toml --workspace --all-targets -- -D warnings +cargo +1.85.0 build --locked --manifest-path rust/Cargo.toml --workspace --examples +PYTHONPATH=src python rust/tests/test_python_parity.py \ + --binary rust/target/debug/examples/wire_check --report rust/artifacts/parity.json +PYTHONPATH=src python rust/tests/test_identity_parity.py \ + --binary rust/target/debug/examples/authority_check --report rust/artifacts/identity-parity.json +PYTHONPATH=src python rust/tests/test_storage_parity.py \ + --binary rust/target/debug/examples/writer_check --report rust/artifacts/storage-parity.json +PYTHONPATH=src python rust/tests/test_content_parity.py \ + rust/target/debug/examples/content_check --report rust/artifacts/content-debug.json +``` + +Repeat the build with `--release` and the same suites against +`rust/target/release/examples/`. The opt-in conformance workflow performs both profiles and records exact source +SHA/toolchains plus `Cargo.lock`. Daily identity/storage/session changes are +validated locally; no automatic CI acceptance is claimed. Use the pinned compiler +and current Python reference, never a compatibility shim for a removed auth API. + +The wire suite covers canonical floating-point boundaries, Unicode, large +integers, malformed packets and bidirectional signatures. The authority suite +creates disposable databases using the actual Python `SqliteMetadataStore`, then +compares native and Python decisions over the same records. It covers live +revocation, CA constraints, key-approved session/source changes, rejected legacy authority, recovery, exact retries and a +real concurrent WAL snapshot test. It also checks all 4096 modes across three +classes and five permissions: **61,440 simple mode decisions**, not 61,440 +independent identity scenarios. Report groups separate these from other cases. + +The storage suite compares complete stored rows, canonical bytes, responses, +rollback order and unchanged schemas against the actual Python storage methods. +Separate safety/process groups cover swallowed errors, implicit rollback, failed +COMMIT, result caps, real process kills and Python/Rust fence interoperability. +Native tests additionally inject panics, cleanup failures and an uncertain commit +result. The last case is deterministic error injection, not a physical disk test. + +`wire_check`, `authority_check`, `writer_check`, `pg_writer_check` and +`content_check` are local JSONL **test examples**, never HTTP or MCP endpoints. They use only deterministic public test keys and disposable data. +The writer helper deliberately includes unauthenticated storage orchestration +(`once`) and failure/checkpoint controls only for disposable tests. It is not a +production executor and must never be exposed as an endpoint. These examples +do not execute business handlers, make network calls, launch Python from +Rust, or access production credentials. The Rust crates forbid unsafe code; +cryptographic and database primitives are delegated to pinned libraries. + +## Explicit compatibility and cutover gates + +- Parser limit: 1 MiB/64 levels. Model timestamps currently accept RFC3339 UTC + `Z` forms; Python accepts additional aliases. Model version/depth fields have + fixed-width Rust limits, unlike Python arbitrary integers. These need an + explicit compatibility/version decision before cutover. +- Credential JSON must contain required nullable fields. Malformed typed records + use a unified `invalid_identity_record` code instead of Python's field-specific + decoder errors. Corrupt result body/namespace mismatches are explicitly rejected. +- Ancestor traversal is capped at 1024, certificate traversal at 32 with a shared + 4096-work budget, and session source chains at 32. Cycles/oversized restored + authority fail closed. These defensive limits are not silently described as + accepting every Python input. +- Dalek strict verification rejects weak-key forgeries. Historical key/signature + compatibility still needs characterization before routing production traffic. +- Storage row decoding uses `invalid_storage_record` rather than every Python + field-specific error. Domain validation (notably Unicode tag normalization) remains pending. + Resource policy and PostgreSQL hexadecimal aliases are implemented above. Native storage takes + canonical IDs/validated domain records; its DTOs are not public request schemas. + The stricter poisoned-write and malformed-result checks are explicit safety + differences, not assertions of identical behavior for every Python input. +- Operation handlers, mutation-side key-approved identity/session + issuance/recovery, encrypted-key wrapping and remaining content stores, + native `serve`/HTTP/MCP, `worker`, packaging and deployment remain pending. +- No RSS, memory-reduction or throughput result is claimed. Benchmark equivalent + functionality, data, concurrency and complete process trees before comparison. + +## Next implementation slices + +1. Integrate the real operation/capability registry and complete resource + authorization, including private resources, sharing, memberships and tools. +2. Complete remaining workflow mutations, LFS and durable cross-store recovery; + connect the writer to the real authorized executor, receipt signing and secret + delivery. Retain Python migration/administration scripts. Extend the current + metadata crash/retry checks to complete operation effects. +3. Integrate a bounded native executor and HTTP/MCP adapters, then worker + claiming/leases/retries and side effects. Empty loops or Python subprocesses + do not count as a server port. +4. Rehearse isolated deployments and rollback, then measure equivalent workloads. + Keep the current Python service until explicit cutover acceptance. + +## Coordination + +Repository `AGENTS.md`, entry skill and deployed `/AGENTS.md` were read. No +selected MSG signing identity was available in the editor, so no authorized +private coordination thread could be created. No new production identity or +public coordination post was created. Work is recorded in the migration +branch/PR; use an existing authorized identity when private coordination resumes. +The temporary offline-input export workflow is removed by the fifth slice. +No selected production MSG identity was invented or registered for testing. diff --git a/rust/crates/msg-core/Cargo.toml b/rust/crates/msg-core/Cargo.toml new file mode 100644 index 00000000..f62fdc30 --- /dev/null +++ b/rust/crates/msg-core/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "msg-core" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +publish.workspace = true + +[dependencies] +base64.workspace = true +ryu.workspace = true +serde.workspace = true +serde_json.workspace = true +sha2.workspace = true + +[lints] +workspace = true diff --git a/rust/crates/msg-core/src/lib.rs b/rust/crates/msg-core/src/lib.rs new file mode 100644 index 00000000..f805cad3 --- /dev/null +++ b/rust/crates/msg-core/src/lib.rs @@ -0,0 +1,372 @@ +//! MSG canonical-v1 bytes. This is deliberately not RFC 8785 / JCS. +use std::collections::BTreeMap; +use std::fmt; + +use base64::{engine::general_purpose::URL_SAFE_NO_PAD, Engine}; +use serde::de::{self, MapAccess, Visitor}; +use serde::{Deserialize, Deserializer, Serialize, Serializer}; +use serde_json::value::RawValue; +use sha2::{Digest, Sha256}; + +pub const MAX_BYTES: usize = 1_048_576; +pub const MAX_DEPTH: usize = 64; + +/// Errors contain stable codes only; never input, keys, tokens or parser excerpts. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Error(pub &'static str); + +impl fmt::Display for Error { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(self.0) + } +} +impl std::error::Error for Error {} +pub type Result = std::result::Result; + +/// Private representation prevents callers from constructing non-finite values. +#[derive(Clone, PartialEq)] +pub struct Json(Value); + +#[derive(Clone, PartialEq)] +enum Value { + Null, + Bool(bool), + Integer(String), + Float(f64), + String(String), + Array(Vec), + Object(BTreeMap), +} + +// Preserve canonical-v1 numbers when typed records contain arbitrary JSON. +// Do not round through serde_json::Value's fixed-width number representation. +impl Serialize for Json { + fn serialize(&self, serializer: S) -> std::result::Result { + let text = self.canonical().map_err(serde::ser::Error::custom)?; + RawValue::from_string(text) + .map_err(serde::ser::Error::custom)? + .serialize(serializer) + } +} +impl<'de> Deserialize<'de> for Json { + fn deserialize>(deserializer: D) -> std::result::Result { + let raw = Box::::deserialize(deserializer)?; + Self::parse(raw.get()).map_err(serde::de::Error::custom) + } +} + +impl Json { + pub fn as_bool(&self) -> Result { + match self.0 { + Value::Bool(v) => Ok(v), + _ => Err(Error("invalid_type")), + } + } + pub fn parse(input: &str) -> Result { + if input.len() > MAX_BYTES { + return Err(Error("request_too_large")); + } + let raw: Box = serde_json::from_str(input).map_err(|_| Error("invalid_json"))?; + Self::from_raw(&raw, 0) + } + + fn from_raw(raw: &RawValue, depth: usize) -> Result { + if depth > MAX_DEPTH { + return Err(Error("json_too_deep")); + } + let text = raw.get(); + let invalid = |_| Error("invalid_json"); + let value = match text.as_bytes().first() { + Some(b'n') => Value::Null, + Some(b't') => Value::Bool(true), + Some(b'f') => Value::Bool(false), + Some(b'"') => Value::String(serde_json::from_str(text).map_err(invalid)?), + Some(b'[') => { + let values: Vec> = serde_json::from_str(text).map_err(invalid)?; + Value::Array( + values + .iter() + .map(|v| Self::from_raw(v, depth + 1)) + .collect::>()?, + ) + } + Some(b'{') => { + let members: Members = serde_json::from_str(text).map_err(|e| { + if e.to_string().starts_with("duplicate_json_key") { + Error("duplicate_json_key") + } else { + Error("invalid_json") + } + })?; + Value::Object( + members + .0 + .into_iter() + .map(|(k, v)| Ok((k, Self::from_raw(&v, depth + 1)?))) + .collect::>()?, + ) + } + _ if text.contains(['.', 'e', 'E']) => { + let number: f64 = text.parse().map_err(|_| Error("invalid_json"))?; + if !number.is_finite() { + return Err(Error("non_finite_float")); + } + Value::Float(number) + } + _ => { + // CPython's default integer conversion limit. Do not round through f64/u64. + if text.trim_start_matches('-').len() > 4300 { + return Err(Error("invalid_json")); + } + Value::Integer(if text == "-0" { "0" } else { text }.to_owned()) + } + }; + Ok(Self(value)) + } + + pub fn string(value: impl Into) -> Self { + Self(Value::String(value.into())) + } + pub fn empty_array() -> Self { + Self(Value::Array(Vec::new())) + } + pub fn as_object(&self) -> Result<&BTreeMap> { + match &self.0 { + Value::Object(v) => Ok(v), + _ => Err(Error("invalid_type")), + } + } + pub fn into_object(self) -> Result> { + match self.0 { + Value::Object(v) => Ok(v), + _ => Err(Error("invalid_type")), + } + } + pub fn from_object(value: BTreeMap) -> Self { + Self(Value::Object(value)) + } + pub fn as_array(&self) -> Result<&[Json]> { + match &self.0 { + Value::Array(v) => Ok(v), + _ => Err(Error("invalid_type")), + } + } + pub fn as_str(&self) -> Result<&str> { + match &self.0 { + Value::String(v) => Ok(v), + _ => Err(Error("invalid_type")), + } + } + pub fn as_integer(&self) -> Result<&str> { + match &self.0 { + Value::Integer(v) => Ok(v), + _ => Err(Error("invalid_type")), + } + } + pub fn is_null(&self) -> bool { + matches!(self.0, Value::Null) + } + + pub fn canonical(&self) -> Result { + let mut output = String::new(); + self.write(&mut output, 0)?; + Ok(output) + } + + fn write(&self, out: &mut String, depth: usize) -> Result<()> { + if depth > MAX_DEPTH { + return Err(Error("json_too_deep")); + } + match &self.0 { + Value::Null => out.push_str("null"), + Value::Bool(v) => out.push_str(if *v { "true" } else { "false" }), + Value::Integer(v) => out.push_str(v), + Value::Float(v) => out.push_str(&python_float(*v)), + Value::String(v) => { + out.push_str(&serde_json::to_string(v).map_err(|_| Error("invalid_json_value"))?) + } + Value::Array(values) => { + out.push('['); + for (i, value) in values.iter().enumerate() { + if i != 0 { + out.push(','); + } + value.write(out, depth + 1)?; + } + out.push(']'); + } + Value::Object(values) => { + out.push('{'); + for (i, (key, value)) in values.iter().enumerate() { + if i != 0 { + out.push(','); + } + out.push_str( + &serde_json::to_string(key).map_err(|_| Error("invalid_json_value"))?, + ); + out.push(':'); + value.write(out, depth + 1)?; + } + out.push('}'); + } + } + if out.len() > MAX_BYTES { + return Err(Error("request_too_large")); + } + Ok(()) + } +} + +struct Members(BTreeMap>); +impl<'de> Deserialize<'de> for Members { + fn deserialize>(deserializer: D) -> std::result::Result { + struct Unique; + impl<'de> Visitor<'de> for Unique { + type Value = Members; + fn expecting(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str("an object without duplicate keys") + } + fn visit_map>( + self, + mut map: M, + ) -> std::result::Result { + let mut values = BTreeMap::new(); + while let Some((key, value)) = map.next_entry::>()? { + if values.insert(key, value).is_some() { + return Err(de::Error::custom("duplicate_json_key")); + } + } + Ok(Members(values)) + } + } + deserializer.deserialize_map(Unique) + } +} + +/// Ryu supplies shortest round-trip digits; Python supplies the notation rules. +/// Keep float/integer distinction, -0.0, exponent sign and two-digit exponent. +fn python_float(value: f64) -> String { + if value == 0.0 { + return if value.is_sign_negative() { + "-0.0" + } else { + "0.0" + } + .to_owned(); + } + let mut buffer = ryu::Buffer::new(); + let text = buffer.format_finite(value.abs()); + let (mantissa, exponent) = text.split_once('e').unwrap_or((text, "0")); + let base: i32 = exponent.parse().unwrap_or(0); + let point = mantissa.find('.').unwrap_or(mantissa.len()) as i32; + let raw_digits = mantissa.replace('.', ""); + let leading = raw_digits.len() - raw_digits.trim_start_matches('0').len(); + let digits = raw_digits.trim_start_matches('0').trim_end_matches('0'); + let scientific = base + point - leading as i32 - 1; + let mut output = if value.is_sign_negative() { "-" } else { "" }.to_owned(); + if !(-4..16).contains(&scientific) { + output.push_str(&digits[..1]); + if digits.len() > 1 { + output.push('.'); + output.push_str(&digits[1..]); + } + output.push('e'); + output.push(if scientific < 0 { '-' } else { '+' }); + output.push_str(&format!("{:02}", scientific.unsigned_abs())); + } else { + let position = scientific + 1; + if position <= 0 { + output.push_str("0."); + output.push_str(&"0".repeat((-position) as usize)); + output.push_str(digits); + } else if position as usize >= digits.len() { + output.push_str(digits); + output.push_str(&"0".repeat(position as usize - digits.len())); + output.push_str(".0"); + } else { + let position = position as usize; + output.push_str(&digits[..position]); + output.push('.'); + output.push_str(&digits[position..]); + } + } + output +} + +pub fn digest(bytes: &[u8]) -> String { + format!("sha256:{:x}", Sha256::digest(bytes)) +} +pub fn b64(bytes: &[u8]) -> String { + URL_SAFE_NO_PAD.encode(bytes) +} +pub fn unb64(value: &str, limit: usize) -> Result> { + if value.len() > limit.saturating_mul(4) / 3 + 4 { + return Err(Error("invalid_base64")); + } + let bytes = URL_SAFE_NO_PAD + .decode(value) + .map_err(|_| Error("invalid_base64"))?; + if bytes.len() > limit || b64(&bytes) != value { + return Err(Error("invalid_base64")); + } + Ok(bytes) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn canonical_preserves_python_numbers() { + for (raw, expected) in [ + ("1E20", "1e+20"), + ("1e-7", "1e-07"), + ("1e15", "1000000000000000.0"), + ("1e-4", "0.0001"), + ("-0.0", "-0.0"), + ("-0", "0"), + ("18446744073709551616000", "18446744073709551616000"), + ("5e-324", "5e-324"), + ("1.7976931348623157e308", "1.7976931348623157e+308"), + ] { + assert_eq!(Json::parse(raw).unwrap().canonical().unwrap(), expected); + } + } + + #[test] + fn rejects_ambiguous_or_unbounded_inputs() { + for raw in [ + r#"{"a":1,"\u0061":2}"#, + r#"[{"x":0,"x":1}]"#, + "NaN", + "Infinity", + "1e999", + r#""\ud800""#, + "01", + "[1,]", + ] { + assert!( + Json::parse(raw).and_then(|v| v.canonical()).is_err(), + "{raw}" + ); + } + assert!(Json::parse(&" ".repeat(MAX_BYTES + 1)).is_err()); + assert!(Json::parse(&format!("{}0{}", "[".repeat(66), "]".repeat(66))).is_err()); + } + + #[test] + fn unicode_and_base64_are_canonical() { + assert_eq!( + Json::parse(r#"{"😀":1,"\ue000":2,"a":"中文\n"}"#) + .unwrap() + .canonical() + .unwrap(), + "{\"a\":\"中文\\n\",\"\u{e000}\":2,\"😀\":1}" + ); + for bad in ["YQ=", "YQ==", "YR", "+/", "Y Q", "a"] { + assert!(unb64(bad, 10).is_err()); + } + assert_eq!(unb64("YQ", 1).unwrap(), b"a"); + assert!(unb64("YQ", 0).is_err()); + } +} diff --git a/rust/crates/msg-crypto/Cargo.toml b/rust/crates/msg-crypto/Cargo.toml new file mode 100644 index 00000000..a013d7dd --- /dev/null +++ b/rust/crates/msg-crypto/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "msg-crypto" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +publish.workspace = true + +[dependencies] +ed25519-dalek.workspace = true +serde.workspace = true +sha2.workspace = true +msg-core = { path = "../msg-core" } + +[lints] +workspace = true diff --git a/rust/crates/msg-crypto/src/lib.rs b/rust/crates/msg-crypto/src/lib.rs new file mode 100644 index 00000000..ee4b82a8 --- /dev/null +++ b/rust/crates/msg-crypto/src/lib.rs @@ -0,0 +1,101 @@ +//! Native signature primitives, not authentication or certificate authorization. +use ed25519_dalek::{Signer as _, SigningKey, VerifyingKey}; +use msg_core::{b64, unb64, Error, Result}; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; + +#[derive(Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Signature { + pub key_id: String, + pub algorithm: String, + pub value: String, +} + +pub fn key_id(public: &[u8]) -> String { + format!("k_{:x}", Sha256::digest(public)) +} +pub fn subject_id(public: &[u8]) -> String { + let mut digest = Sha256::new(); + digest.update(b"msg-subject-v1\0"); + digest.update(public); + format!("u_{}", &format!("{:x}", digest.finalize())[..32]) +} +pub fn framed(payload: &[u8], purpose: &str) -> Result> { + if purpose.is_empty() || !purpose.is_ascii() || purpose.contains('\0') { + return Err(Error("invalid_signature_purpose")); + } + let mut message = Vec::with_capacity(17 + purpose.len() + payload.len()); + message.extend_from_slice(b"msg.lmm.best/v1/"); + message.extend_from_slice(purpose.as_bytes()); + message.push(0); + message.extend_from_slice(payload); + Ok(message) +} + +/// Intentionally no Debug/Serialize/Clone or private-key export. +/// ed25519-dalek's default zeroize feature clears the owned key on drop. +pub struct Ed25519Signer(SigningKey); +impl Ed25519Signer { + pub fn from_seed(seed: &[u8; 32]) -> Self { + Self(SigningKey::from_bytes(seed)) + } + pub fn public_key(&self) -> [u8; 32] { + self.0.verifying_key().to_bytes() + } + pub fn sign(&self, payload: &[u8], purpose: &str) -> Result { + Ok(Signature { + key_id: key_id(&self.public_key()), + algorithm: "ed25519".to_owned(), + value: b64(&self.0.sign(&framed(payload, purpose)?).to_bytes()), + }) + } +} + +/// Verifies bytes only. Callers must separately enforce credential state, +/// subject binding, expiry, service origin, replay rules and CA authorization. +pub fn verify(public: &[u8], payload: &[u8], signature: &Signature, purpose: &str) -> Result<()> { + let invalid = |_| Error("invalid_signature"); + if signature.algorithm != "ed25519" || signature.key_id != key_id(public) { + return Err(Error("invalid_signature")); + } + let public: &[u8; 32] = public.try_into().map_err(invalid)?; + let key = VerifyingKey::from_bytes(public).map_err(|_| Error("invalid_signature"))?; + let bytes = unb64(&signature.value, 64).map_err(|_| Error("invalid_signature"))?; + let signature = + ed25519_dalek::Signature::from_slice(&bytes).map_err(|_| Error("invalid_signature"))?; + key.verify_strict(&framed(payload, purpose)?, &signature) + .map_err(|_| Error("invalid_signature")) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn domains_keys_and_message_are_bound() { + let signer = Ed25519Signer::from_seed(&[42; 32]); + let signature = signer.sign(b"hello", "request").unwrap(); + assert!(verify(&signer.public_key(), b"hello", &signature, "request").is_ok()); + assert!(verify(&signer.public_key(), b"hello", &signature, "receipt").is_err()); + assert!(verify(&signer.public_key(), b"other", &signature, "request").is_err()); + assert!(verify(&[0; 32], b"hello", &signature, "request").is_err()); + assert!(framed(b"x", "").is_err()); + assert!(framed(b"x", "bad\0purpose").is_err()); + assert!(framed(b"x", "中文").is_err()); + } + + #[test] + fn rejects_weak_key_identity_forgery() { + let mut identity = [0; 32]; + identity[0] = 1; + let mut forged = [0; 64]; + forged[0] = 1; + let signature = Signature { + key_id: key_id(&identity), + algorithm: "ed25519".to_owned(), + value: b64(&forged), + }; + assert!(verify(&identity, b"forgery", &signature, "request").is_err()); + } +} diff --git a/rust/crates/msg-identity/Cargo.toml b/rust/crates/msg-identity/Cargo.toml new file mode 100644 index 00000000..f6402dfc --- /dev/null +++ b/rust/crates/msg-identity/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "msg-identity" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +publish.workspace = true + +[dependencies] +msg-core = { path = "../msg-core" } +msg-crypto = { path = "../msg-crypto" } +msg-protocol = { path = "../msg-protocol" } +chrono.workspace = true +serde.workspace = true +serde_json.workspace = true +sha2.workspace = true +subtle.workspace = true + +[lints] +workspace = true diff --git a/rust/crates/msg-identity/src/authentication.rs b/rust/crates/msg-identity/src/authentication.rs new file mode 100644 index 00000000..6c01e7ea --- /dev/null +++ b/rust/crates/msg-identity/src/authentication.rs @@ -0,0 +1,585 @@ +//! Authentication admission only. Full resource checks must follow a fresh admission. +use crate::{certificates::CertificateValidator, models::*, require, store::*}; +use chrono::Duration; +use msg_core::{unb64, Error, Json, Result, MAX_BYTES}; +use msg_crypto::{key_id, subject_id, Signature}; +use msg_protocol::Request; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; +use std::collections::BTreeSet; +use subtle::ConstantTimeEq; + +/// Supplied by the trusted operation registry, never by a request envelope. +#[derive(Deserialize)] +pub struct OperationPolicy { + pub name: String, + pub version: u32, + pub effect: String, + pub entries: BTreeSet, + pub require_signature: bool, + pub anonymous_only: bool, +} +#[derive(Clone, Copy)] +pub enum Entry { + Network, + LocalAdmin, + Worker, +} +impl Entry { + fn name(self) -> &'static str { + match self { + Self::Network => "network", + Self::LocalAdmin => "local_admin", + Self::Worker => "worker", + } + } +} + +/// Fields are private and there is no Deserialize implementation. +/// Request data cannot assert an actor or mint a principal. +/// +/// ```compile_fail +/// let _ = serde_json::from_str::("{}"); +/// ``` +#[derive(Serialize)] +pub struct Principal { + actor: Option, + subject: Option, + credential_id: Option, + method: &'static str, + certificates: Vec, + ceiling: Vec, +} +impl Principal { + pub fn actor(&self) -> Option<&str> { + self.actor.as_deref() + } + pub fn subject(&self) -> Option<&str> { + self.subject.as_deref() + } + pub fn credential_id(&self) -> Option<&str> { + self.credential_id.as_deref() + } + pub fn certificates(&self) -> &[String] { + &self.certificates + } + pub fn ceiling(&self) -> &[Grant] { + &self.ceiling + } + pub fn method(&self) -> &str { + self.method + } +} + +pub enum Admission { + /// Authenticated identity only, not permission to execute. + Fresh(Principal), + /// Revoked/exhausted credentials receive no fresh principal. + /// Return the already committed result; never invoke a handler with this branch. + Replay(Json), +} + +pub struct AuthenticationService { + certificates: CertificateValidator, + runtime: RuntimeGeneration, + primary_ceiling: Vec, + temporary_ceiling: Vec, + browser_policy: Option, +} +impl AuthenticationService { + pub fn new( + certificates: CertificateValidator, + store: &dyn AuthorityStore, + primary_ceiling: Vec, + temporary_ceiling: Vec, + ) -> Result { + Ok(Self { + certificates, + runtime: RuntimeGeneration::capture(store)?, + primary_ceiling, + temporary_ceiling, + browser_policy: None, + }) + } + pub fn with_browser_policy(mut self, policy: crate::sessions::BrowserPolicy) -> Self { + self.browser_policy = Some(policy); + self + } + pub fn authenticate( + &mut self, + request: &Request, + spec: &OperationPolicy, + store: &dyn AuthorityStore, + entry: Entry, + now: Timestamp, + ) -> Result { + let requested_subject = optional_string(request.field("subject")?)?; + check_archived(store, requested_subject)?; + self.runtime.require_current(store)?; + let anchor = store.setting("active_root_certificate")?; + require( + anchor + .as_ref() + .map(|v| v.as_str()) + .transpose()? + .unwrap_or(self.certificates.root_id()) + == self.certificates.root_id(), + "service_restart_required", + )?; + require( + request.field("operation")?.as_str()? == spec.name + && request.field("contract_version")?.as_integer()? == spec.version.to_string(), + "operation_policy_mismatch", + )?; + if store.has_setting("recovery_quarantine")? { + return Err(Error(if spec.effect == "read" { + "recovery_quarantined" + } else { + "writes_paused" + })); + } + require( + request.field("target_service")?.as_str()? == self.certificates.service(), + "wrong_service", + )?; + request.verify_payload_digest()?; + let proof = request.field("proof")?; + if spec.effect != "read" || !proof.is_null() { + let expiry = + optional_string(request.field("expires_at")?)?.ok_or(Error("request_expired"))?; + let expiry = Timestamp::parse(expiry)?; + require(now < expiry, "request_expired")?; + require( + expiry.0 - now.0 <= Duration::seconds(300), + "request_expiry_too_long", + )?; + } + let mut replay = None; + let principal = match spec.name.as_str() { + "identity.register" => { + let signed = proof.as_object().map_err(|_| Error("proof_required"))?; + require(signed.contains_key("signature"), "proof_required")?; + let public = unb64(argument(request, "public_key")?.as_str()?, 32)?; + let uid = subject_id(&public); + require( + !setting_truthy(store, &format!("delegated_identity:{uid}"))?, + "delegated_identity_not_upgradable", + )?; + require( + public.len() == 32 && requested_subject == Some(uid.as_str()), + "subject_mismatch", + )?; + request.verify_signature_bytes(&public)?; + Principal { + actor: Some(uid.clone()), + subject: Some(uid), + credential_id: Some(key_id(&public)), + method: "signature", + certificates: vec![], + ceiling: self.primary_ceiling.clone(), + } + } + "identity.temporary" | "identity.custodial_create" => { + require( + proof.is_null() && requested_subject.is_none(), + "invalid_bootstrap", + )?; + if let Some(public) = request.field("arguments")?.as_object()?.get("public_key") { + if truthy(public)? { + let uid = subject_id(&unb64(public.as_str()?, 32)?); + require( + !setting_truthy(store, &format!("delegated_identity:{uid}"))?, + "delegated_identity_not_upgradable", + )?; + } + } + let nonce = unb64(argument(request, "nonce")?.as_str()?, 64)?; + require(nonce.len() >= 24, "invalid_bootstrap_nonce")?; + let prefix = if spec.name == "identity.custodial_create" { + "u_cust_" + } else { + "u_tmp_" + }; + let uid = format!( + "{}{}", + prefix, + &format!("{:x}", Sha256::digest(&nonce))[..32] + ); + Principal { + credential_id: Some(format!("t_{}", &uid[2..])), + actor: Some(uid.clone()), + subject: Some(uid), + method: "token", + certificates: vec![], + ceiling: self.temporary_ceiling.clone(), + } + } + "identity.token_recover" => { + require( + proof.is_null() && requested_subject.is_some(), + "invalid_recovery_proof", + )?; + let old = store.credential(argument(request, "credential_id")?.as_str()?)?; + crate::sessions::require_binding( + store, + &old, + now, + self.browser_policy.as_ref(), + &self.temporary_ceiling, + )?; + require( + old.kind == CredentialKind::Token && requested_subject == Some(&old.subject_id), + "recovery_unavailable", + )?; + let actor = store.subject(&old.subject_id)?; + require( + !actor.local_only && actor.resource_id != "u_root", + "local_only", + )?; + let secret = unb64(argument(request, "recovery_secret")?.as_str()?, 64)?; + require(secret.len() >= 32, "invalid_recovery_secret")?; + let mut hasher = Sha256::new(); + hasher.update(b"token-recovery-v1\0"); + hasher.update(&secret); + let verifier = format!("{:x}", hasher.finalize()); + let delivery = store + .recovery_delivery(&old.id, &actor.resource_id)? + .ok_or(Error("recovery_unavailable"))?; + require( + bool::from(delivery.verifier.as_bytes().ct_eq(verifier.as_bytes())) + && delivery.expires_at > now + && delivery.request_id + == argument(request, "original_request_id")?.as_str()?, + "recovery_unavailable", + )?; + if delivery.consumed { + replay = store.request_result( + Some(&actor.resource_id), + request.field("request_id")?.as_str()?, + request.field("payload_digest")?.as_str()?, + )?; + let previous = replay.as_ref().ok_or(Error("recovery_unavailable"))?; + let data = previous + .as_object()? + .get("data") + .ok_or(Error("recovery_unavailable"))? + .as_object()?; + require( + data.get("previous_credential") + .and_then(|v| v.as_str().ok()) + == Some(&old.id), + "recovery_unavailable", + )?; + } else { + require( + old.revoked_at.is_none() && old.expires_at.is_some_and(|v| v > now), + "recovery_unavailable", + )?; + } + let mut ceiling = Vec::new(); + for grant in &self.primary_ceiling { + if grant.capability == "identity.basic" + && grant.operations.contains("identity.token_recover@1") + { + let mut grant = grant.clone(); + grant.operations = BTreeSet::from(["identity.token_recover@1".to_owned()]); + ceiling.push(grant); + } + } + require(!ceiling.is_empty(), "recovery_unavailable")?; + Principal { + actor: Some(actor.resource_id.clone()), + subject: Some(actor.resource_id), + credential_id: Some(old.id), + method: "recovery", + certificates: vec![], + ceiling, + } + } + _ if proof.is_null() => { + let anonymous_write = matches!( + spec.name.as_str(), + "communication.internet_receive" | "identity.link_claim" + ) && spec.version == 1 + && spec.anonymous_only + && spec.effect == "transaction"; + require( + requested_subject.is_none() && (spec.effect == "read" || anonymous_write), + "authentication_required", + )?; + Principal { + actor: None, + subject: None, + credential_id: None, + method: "anonymous", + certificates: vec![], + ceiling: vec![], + } + } + _ => { + let proof = proof.as_object()?; + let signature: Option = + proof.get("signature").map(decode).transpose()?; + let cid = match &signature { + Some(s) => s.key_id.as_str(), + None => proof + .get("credential_id") + .ok_or(Error("invalid_proof"))? + .as_str()?, + }; + let credential = store.credential(cid)?; + let actor = store.subject(&credential.subject_id)?; + require( + credential.not_before <= now && credential.expires_at.is_none_or(|v| now < v), + "credential_expired", + )?; + let (method, ids): (&str, Vec) = match &signature { + Some(signature) => { + require( + credential.kind == CredentialKind::SigningKey, + "wrong_credential_kind", + )?; + msg_crypto::verify( + &credential.verifier.0, + &request.signing_bytes()?, + signature, + "request", + )?; + ( + "signature", + decode(proof.get("certificates").ok_or(Error("invalid_proof"))?)?, + ) + } + None => { + require( + credential.kind == CredentialKind::Token, + "wrong_credential_kind", + )?; + let token = unb64( + proof.get("token").ok_or(Error("invalid_proof"))?.as_str()?, + MAX_BYTES, + )?; + let hash = Sha256::digest(&token); + require( + bool::from(credential.verifier.0.as_slice().ct_eq(hash.as_slice())), + "invalid_token", + )?; + crate::sessions::require_binding( + store, + &credential, + now, + self.browser_policy.as_ref(), + &self.temporary_ceiling, + )?; + ("token", vec![]) + } + }; + if credential.revoked_at.is_some() { + if method == "token" + && matches!( + spec.name.as_str(), + "identity.token_rotate" | "identity.custodial_upgrade_finish" + ) + { + replay = store.request_result( + Some(&actor.resource_id), + request.field("request_id")?.as_str()?, + request.field("payload_digest")?.as_str()?, + )?; + } + let previous = replay.as_ref().ok_or(Error("credential_revoked"))?; + let data = previous + .as_object()? + .get("data") + .ok_or(Error("credential_revoked"))? + .as_object()?; + require( + data.get("previous_credential") + .and_then(|v| v.as_str().ok()) + == Some(cid), + "credential_revoked", + )?; + let successor = store.credential( + data.get("credential_id") + .ok_or(Error("credential_revoked"))? + .as_str()?, + )?; + require( + successor.subject_id == actor.resource_id + && successor.revoked_at.is_none() + && successor.expires_at.is_none_or(|v| v > now), + "credential_revoked", + )?; + } + let subject = store.subject( + requested_subject + .filter(|v| !v.is_empty()) + .unwrap_or(&actor.resource_id), + )?; + if let Some(binding) = + store.setting(&format!("delegated_identity:{}", actor.resource_id))? + { + let binding: DelegatedIdentity = decode(&binding)?; + require( + subject.resource_id == binding.grantor, + "delegated_subject_required", + )?; + require(ids.contains(&binding.certificate_id), "delegation_required")?; + if binding.max_uses.is_some_and(|max| binding.uses >= max) { + let previous = store + .request_result( + Some(&subject.resource_id), + request.field("request_id")?.as_str()?, + request.field("payload_digest")?.as_str()?, + )? + .ok_or(Error("delegation_exhausted"))?; + require( + previous + .as_object()? + .get("actor") + .and_then(|v| v.as_str().ok()) + == Some(actor.resource_id.as_str()), + "delegation_exhausted", + )?; + replay = Some(previous); + } + } + require( + !(actor.local_only + || subject.local_only + || actor.resource_id == "u_root" + || subject.resource_id == "u_root") + || matches!(entry, Entry::LocalAdmin), + "local_only", + )?; + require(spec.entries.contains(entry.name()), "entry_not_allowed")?; + let operation = format!("{}@{}", spec.name, spec.version); + require( + credential + .ceiling + .iter() + .any(|g| g.operations.contains(&operation)), + "credential_ceiling", + )?; + if actor.kind == SubjectKind::Custodial + && method == "token" + && spec.effect != "read" + { + let download = spec.name == "transfer.open" + && argument(request, "direction").and_then(Json::as_str).ok() + == Some("download"); + require( + custodial_write(&spec.name) || download, + "custodial_operation_not_supported", + )?; + } + let confirmation = actor.kind == SubjectKind::Custodial + && method == "token" + && matches!( + spec.name.as_str(), + "achievement.finish" + | "achievement.pin" + | "achievement.unpin" + | "achievement.reorder" + ); + require( + !spec.require_signature || method == "signature" || confirmation, + "signature_required", + )?; + let mut validated = Vec::new(); + let mut delegated = false; + for id in ids { + let cert = self.certificates.validate(&id, store, now)?; + require( + cert.subject_id == actor.resource_id && cert.key_id == credential.id, + "certificate_subject_mismatch", + )?; + if cert.kind == CertificateKind::Delegation { + for source in &cert.authority_sources { + if let Some(fact) = + store.setting(&format!("delegation:{}", source.id))? + { + delegated |= fact + .as_object()? + .get("grantor") + .and_then(|v| v.as_str().ok()) + == Some(subject.resource_id.as_str()); + } + } + } + validated.push(cert.resource_id); + } + require( + actor.resource_id == subject.resource_id || delegated, + "delegation_required", + )?; + Principal { + actor: Some(actor.resource_id), + subject: Some(subject.resource_id), + credential_id: Some(credential.id), + method, + certificates: validated, + ceiling: credential.ceiling, + } + } + }; + check_archived(store, principal.actor())?; + check_archived(store, principal.subject())?; + Ok(match replay { + Some(result) => Admission::Replay(result), + None => Admission::Fresh(principal), + }) + } +} +fn optional_string(value: &Json) -> Result> { + if value.is_null() { + Ok(None) + } else { + value.as_str().map(Some) + } +} +fn argument<'a>(request: &'a Request, name: &str) -> Result<&'a Json> { + request + .field("arguments")? + .as_object()? + .get(name) + .ok_or(Error("missing_field")) +} +fn check_archived(store: &dyn AuthorityStore, subject: Option<&str>) -> Result<()> { + require( + !setting_truthy( + store, + &format!("identity_archived:{}", subject.unwrap_or("None")), + )?, + "account_archived", + ) +} +fn custodial_write(name: &str) -> bool { + matches!( + name, + "identity.rename" + | "content.post_create" + | "content.post_edit" + | "content.file_put" + | "content.attach" + | "file.create" + | "file.copy" + | "file.write" + | "file.patch" + | "discussion.reply" + | "discussion.quote" + | "discussion.repost" + | "identity.token_rotate" + | "identity.custodial_upgrade_start" + | "identity.custodial_upgrade_finish" + | "identity.custodial_rewrap_entry" + | "identity.custodial_rewrap_revision" + | "identity.custodial_rewrap_ack" + | "achievement.start" + | "achievement.answer" + | "achievement.finish" + | "achievement.pin" + | "achievement.unpin" + | "achievement.reorder" + ) +} diff --git a/rust/crates/msg-identity/src/authorization.rs b/rust/crates/msg-identity/src/authorization.rs new file mode 100644 index 00000000..ac10bdb4 --- /dev/null +++ b/rust/crates/msg-identity/src/authorization.rs @@ -0,0 +1,942 @@ +//! Current-state resource authorization. Authentication is necessary, not sufficient. +//! +//! All facts are read through the same transaction as the operation. A request +//! cannot construct `Principal`, supply a registry, or bypass this policy with a +//! replayed result. The caller must check each replayed resource again. +use crate::{ + authentication::{Entry, Principal}, + certificates::CertificateValidator, + models::{ + decode, CertificateKind, CredentialKind, DelegationFact, Grant, ResourceAuthority, + ResourceRef, Timestamp, + }, + policy::{allows, grant_covers, parse_mode, scope_contains, CERTGATE}, + require, + store::{setting_truthy, truthy, AuthorityStore, Record, RuntimeGeneration}, +}; +use msg_core::{Error, Json, Result}; +use serde::Deserialize; +use std::collections::{BTreeMap, BTreeSet}; + +const ROOT: &str = "u_root"; +const TOOLS: &str = "t_tools"; + +/// Policy metadata is read from stored resource rows, never a caller's arguments. +#[derive(Clone, Deserialize)] +pub struct PolicyResource { + #[serde(flatten)] + pub authority: ResourceAuthority, + pub name: String, + pub type_version: u32, +} +impl std::ops::Deref for PolicyResource { + type Target = ResourceAuthority; + fn deref(&self) -> &Self::Target { + &self.authority + } +} + +#[derive(Clone)] +pub struct DirectConversation { + pub resource_id: String, + pub participant_a: String, + pub participant_b: String, + pub state: String, +} +#[derive(Clone)] +pub struct LegacyShare { + pub grantor: String, + pub expires_at: String, + pub created_at: String, +} +#[derive(Clone)] +pub struct ShareSource { + pub resource_id: String, + pub grantor: String, + pub grantee: String, + pub grantee_kind: String, + pub parent_id: Option, + pub operations: String, + pub constraints: String, + pub allow_reshare: bool, + pub expires_at: String, + pub revoked_at: Option, + pub created_at: String, +} + +/// The complete read boundary used by resource policy. Implementations must not +/// silently turn storage errors into absent facts or ignore unreadable rows. +pub trait AuthorizationStore: AuthorityStore { + fn authority(&self) -> &dyn AuthorityStore; + fn memberships(&self, subject: &str) -> Result>; + fn organization_exists(&self, id: &str) -> Result; + fn topic_admin(&self, topic: &str, subject: &str) -> Result; + fn topic_ban(&self, topic: &str, subject: &str) -> Result>>; + fn direct_conversation(&self, resource: &str) -> Result>; + fn direct_blocked(&self, a: &str, b: &str) -> Result; + fn legacy_share(&self, resource: &str, grantee: &str) -> Result>; + fn share_candidates(&self, resource: &str, now: &str) -> Result>; + fn share_source(&self, id: &str) -> Result>; + fn csr_issuer(&self, id: &str) -> Result; + fn policy_resource(&self, id: &str) -> Result { + let resource: PolicyResource = decode(&self.record(Record::Resource, id)?)?; + require(resource.id == id, "storage_record_mismatch")?; + Ok(resource) + } +} + +/// Constructed by trusted handlers. This is not a deserializable network proof. +pub struct AccessRequirement { + pub resource_id: String, + pub operation: String, + pub check: Check, +} +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Check { + Read, + List, + Traverse, + Write, + Create, + Remove, + Chmod, + Chgrp, + Chown, + Manage, + Certgate, + ToolUse, + Purge, +} +impl Check { + pub fn parse(value: &str) -> Result { + Ok(match value { + "read" => Self::Read, + "list" => Self::List, + "traverse" => Self::Traverse, + "write" => Self::Write, + "create" => Self::Create, + "remove" => Self::Remove, + "chmod" => Self::Chmod, + "chgrp" => Self::Chgrp, + "chown" => Self::Chown, + "manage" => Self::Manage, + "certgate" => Self::Certgate, + "tool_use" => Self::ToolUse, + "purge" => Self::Purge, + _ => return Err(Error("invalid_permission")), + }) + } + fn write(self) -> bool { + !matches!(self, Self::Read | Self::List | Self::Traverse) + } + fn override_capability(self) -> Option<&'static str> { + match self { + Self::Read | Self::List | Self::Traverse => Some("resource.read_override"), + Self::Write => Some("resource.write_override"), + Self::Chmod => Some("resource.chmod_override"), + Self::Chgrp => Some("resource.chgrp_override"), + Self::Chown => Some("resource.chown"), + Self::Purge => Some("resource.purge"), + Self::ToolUse => Some("tool.use"), + _ => None, + } + } +} + +/// Only the container flag is needed; type/version lookup is still mandatory. +#[derive(Clone, Deserialize)] +pub struct ResourceTypePolicy { + pub name: String, + pub version: u32, + pub container: bool, +} + +pub struct AuthorizationService { + certificates: CertificateValidator, + runtime: RuntimeGeneration, + resource_types: BTreeMap<(String, u32), bool>, + base_families: BTreeSet, +} +impl AuthorizationService { + pub fn new( + certificates: CertificateValidator, + store: &dyn AuthorityStore, + resource_types: Vec, + base_families: BTreeSet, + ) -> Result { + let mut types = BTreeMap::new(); + for spec in resource_types { + require( + types + .insert((spec.name, spec.version), spec.container) + .is_none(), + "duplicate_resource_type", + )?; + } + Ok(Self { + certificates, + runtime: RuntimeGeneration::capture(store)?, + resource_types: types, + base_families, + }) + } + fn live(&mut self, store: &dyn AuthorityStore) -> Result<()> { + self.runtime.require_current(store)?; + require( + !store.has_setting("recovery_quarantine")?, + "recovery_quarantined", + ) + } + pub fn grants( + &mut self, + principal: &Principal, + store: &dyn AuthorityStore, + now: Timestamp, + ) -> Result> { + self.live(store)?; + let mut grants = Vec::new(); + for id in principal.certificates() { + let certificate = self.certificates.validate(id, store, now)?; + require( + Some(certificate.key_id.as_str()) == principal.credential_id() + && Some(certificate.subject_id.as_str()) == principal.actor(), + "certificate_subject_mismatch", + )?; + grants.extend(certificate.grants); + } + Ok(grants) + } + pub fn has( + &mut self, + principal: &Principal, + capability: &str, + operation: &str, + resource: &str, + store: &dyn AuthorityStore, + now: Timestamp, + ) -> Result { + self.live(store)?; + if principal.method() == "local" && principal.subject() == Some(ROOT) { + return Ok(true); + } + let mut ceiling = false; + for grant in principal.ceiling() { + ceiling |= grant_covers(grant, capability, operation, resource, store)?; + } + if !ceiling { + return Ok(false); + } + for grant in self.grants(principal, store, now)? { + if grant_covers(&grant, capability, operation, resource, store)? { + return Ok(true); + } + } + Ok(false) + } + pub fn ceiling( + &mut self, + principal: &Principal, + operation: &str, + resource: &str, + store: &dyn AuthorityStore, + now: Timestamp, + ) -> Result<()> { + self.live(store)?; + if principal.method() == "anonymous" { + return Ok(()); + } + let mut allowed = false; + for grant in principal.ceiling() { + allowed |= grant.operations.contains(operation) + && scope_contains(&grant.scope, resource, store)?; + } + if !allowed + && matches!( + operation, + "discovery.get@1" + | "discovery.raw@1" + | "job.get@1" + | "transfer.open@1" + | "transfer.part_get@1" + | "transfer.status@1" + | "transfer.seal@1" + | "transfer.cancel@1" + ) + { + allowed = self.tool_output(principal, operation, resource, store, now)?; + } + require(allowed, "credential_ceiling")?; + if principal.actor() != principal.subject() { + let mut delegated = false; + for id in principal.certificates() { + let certificate = self.certificates.validate(id, store, now)?; + if certificate.kind != CertificateKind::Delegation { + continue; + } + // validate() rechecks every authority source under this exact snapshot. + for source in &certificate.authority_sources { + let fact: DelegationFact = decode( + &store + .setting(&format!("delegation:{}", source.id))? + .ok_or(Error("authority_source_invalid"))?, + )?; + if Some(fact.grantor.as_str()) == principal.subject() { + for grant in &certificate.grants { + delegated |= grant.operations.contains(operation) + && scope_contains(&grant.scope, resource, store)?; + } + } + } + } + require(delegated, "delegation_scope")?; + } + Ok(()) + } + fn tool_output( + &mut self, + principal: &Principal, + operation: &str, + resource: &str, + store: &dyn AuthorityStore, + now: Timestamp, + ) -> Result { + if let Some(output) = store.setting(&format!("tool_output:{resource}"))? { + if truthy(&output)? { + let object = output.as_object()?; + if object.get("subject").and_then(|v| v.as_str().ok()) == principal.subject() { + return self.has( + principal, + "tool.use", + operation, + object + .get("tool_id") + .ok_or(Error("invalid_storage_record"))? + .as_str()?, + store, + now, + ); + } + } + } + Ok(false) + } + pub fn ordinary( + &self, + principal: &Principal, + operation: &str, + resource: &str, + store: &dyn AuthorityStore, + ) -> Result { + if matches!(principal.method(), "anonymous" | "local") { + return Ok(true); + } + for grant in principal.ceiling() { + if self.base_families.contains(&grant.capability) + && grant.operations.contains(operation) + && scope_contains(&grant.scope, resource, store)? + { + return Ok(true); + } + } + Ok(false) + } + pub fn require_base( + &mut self, + principal: &Principal, + operation: &str, + resource: &str, + store: &dyn AuthorityStore, + now: Timestamp, + ) -> Result<()> { + self.ceiling(principal, operation, resource, store, now)?; + require( + self.ordinary(principal, operation, resource, store)?, + "credential_ceiling", + ) + } + fn chain( + &self, + resource: &PolicyResource, + store: &dyn AuthorizationStore, + ) -> Result> { + let mut chain = Vec::new(); + for item in store.ancestors(&resource.id)? { + chain.push(store.policy_resource(&item.id)?); + } + chain.push(resource.clone()); + Ok(chain) + } + pub fn share_target_error( + &self, + resource: &PolicyResource, + chain: &[PolicyResource], + store: &dyn AuthorizationStore, + ) -> Result> { + if resource.state != "active" || chain.iter().any(|r| r.state != "active") { + return Ok(Some("share_forbidden_resource")); + } + if *self + .resource_types + .get(&(resource.kind.clone(), resource.type_version)) + .ok_or(Error("unknown_resource_type"))? + { + return Ok(Some("share_container_forbidden")); + } + if chain.iter().any(|r| { + matches!(r.id.as_str(), "r_agents" | "r_rules" | "t_last_will") + || matches!( + r.kind.as_str(), + "tool" + | "csr" + | "certificate" + | "credential" + | "legacy_directive" + | "dm_conversation" + ) + }) || chain.windows(2).any(|p| { + p[0].kind == "user" && matches!(p[1].name.as_str(), "SOUL.md" | "AGENTS.md" | "todos") + }) { + return Ok(Some("share_forbidden_resource")); + } + for item in chain { + if setting_truthy(store.authority(), &format!("hosting_preview:{}", item.id))? + || setting_truthy( + store.authority(), + &format!("hosting_preview_file:{}", item.id), + )? + || store.direct_conversation(&item.id)?.is_some() + { + return Ok(Some("share_forbidden_resource")); + } + } + Ok(None) + } + pub fn share_source_active( + &self, + resource: &PolicyResource, + grant_id: &str, + subject: &str, + now: Timestamp, + store: &dyn AuthorizationStore, + mut reshare: bool, + ) -> Result { + let chain = self.chain(resource, store)?; + if self.share_target_error(resource, &chain, store)?.is_some() { + return Ok(false); + } + let (mut id, mut subject) = (grant_id.to_owned(), subject.to_owned()); + let mut seen = BTreeSet::new(); + let mut child_expiry = None; + loop { + if seen.len() >= 16 || !seen.insert(id.clone()) { + return Ok(false); + } + let Some(row) = store.share_source(&id)? else { + return Ok(false); + }; + if row.resource_id != resource.id || row.revoked_at.is_some() { + return Ok(false); + } + let (Ok(expires), Ok(created), Ok(operations), Ok(constraints)) = ( + Timestamp::parse(&row.expires_at), + Timestamp::parse(&row.created_at), + Json::parse(&row.operations), + Json::parse(&row.constraints), + ) else { + return Ok(false); + }; + let supported = operations + .as_array() + .is_ok_and(|a| a.len() == 1 && a[0].as_str().ok() == Some("read")) + && constraints.as_object().is_ok_and(|a| a.is_empty()); + if !supported + || created > now + || expires <= now + || expires <= created + || child_expiry.is_some_and(|child| child > expires) + || (reshare && !row.allow_reshare) + { + return Ok(false); + } + match row.grantee_kind.as_str() { + "user" if row.grantee == subject => (), + "group" => { + let group = match store.policy_resource(&row.grantee) { + Ok(group) => group, + Err(Error("not_found")) => return Ok(false), + Err(error) => return Err(error), + }; + if group.state != "active" + || !store.organization_exists(&row.grantee)? + || !store.memberships(&subject)?.contains(&row.grantee) + { + return Ok(false); + } + } + _ => return Ok(false), + } + let Some(parent) = row.parent_id else { + return Ok(row.grantor == resource.owner); + }; + subject = row.grantor; + id = parent; + child_expiry = Some(expires); + reshare = true; + } + } + pub fn shared_read( + &self, + principal: &Principal, + resource: &PolicyResource, + chain: &[PolicyResource], + now: Timestamp, + store: &dyn AuthorizationStore, + ) -> Result { + let Some(subject) = principal.subject() else { + return Ok(false); + }; + if principal.actor() != Some(subject) + || self.share_target_error(resource, chain, store)?.is_some() + { + return Ok(false); + } + if let Some(row) = store.legacy_share(&resource.id, subject)? { + if row.grantor == resource.owner && resource.state == "active" { + if let (Ok(expires), Ok(created)) = ( + Timestamp::parse(&row.expires_at), + Timestamp::parse(&row.created_at), + ) { + if created <= now && now < expires { + return Ok(true); + } + } + } + } + if resource.state != "active" { + return Ok(false); + } + let time = now.0.format("%Y-%m-%dT%H:%M:%S%.6fZ").to_string(); + for candidate in store.share_candidates(&resource.id, &time)? { + if self.share_source_active(resource, &candidate, subject, now, store, false)? { + return Ok(true); + } + } + Ok(false) + } + pub fn share_link_read_allowed( + &mut self, + grantor: &str, + credential_id: &str, + resource: &PolicyResource, + chain: &[PolicyResource], + now: Timestamp, + store: &dyn AuthorizationStore, + ) -> Result { + self.runtime.require_current(store.authority())?; + if store.has_setting("recovery_quarantine")? + || resource.owner != grantor + || self.share_target_error(resource, chain, store)?.is_some() + { + return Ok(false); + } + let memberships = store.memberships(grantor)?; + if !allows(resource, Some(grantor), &memberships, "read")? { + return Ok(false); + } + for ancestor in chain.iter().take(chain.len().saturating_sub(1)) { + if !allows(ancestor, Some(grantor), &memberships, "traverse")? { + return Ok(false); + } + } + let credential = match store.credential(credential_id) { + Ok(credential) => credential, + Err(Error("credential_not_found")) => return Ok(false), + Err(error) => return Err(error), + }; + if credential.subject_id != grantor + || credential.kind != CredentialKind::SigningKey + || !credential.current_at(now) + { + return Ok(false); + } + for grant in &credential.ceiling { + if grant.operations.contains("sharing.link_create@1") + && scope_contains(&grant.scope, &resource.id, store.authority())? + { + return Ok(true); + } + } + Ok(false) + } + /// Every check runs in order. `request_name` is only the literal `name` + /// argument needed for topic presentation policy; it grants no identity. + pub fn require( + &mut self, + principal: &Principal, + entry: Entry, + now: Timestamp, + checks: &[AccessRequirement], + request_name: Option<&str>, + store: &dyn AuthorizationStore, + ) -> Result> { + self.live(store.authority())?; + if principal.subject() == Some(ROOT) || principal.actor() == Some(ROOT) { + require( + matches!(entry, Entry::LocalAdmin) && principal.method() == "local", + "local_only", + )?; + } + let memberships = principal + .subject() + .map(|s| store.memberships(s)) + .transpose()? + .unwrap_or_default(); + for check in checks { + let resource = store.policy_resource(&check.resource_id)?; + let operation = check.operation.as_str(); + require(resource.kind != "watch", "permission_denied")?; + if resource.kind == "saved_query" { + require( + matches!(operation, "query.save@1" | "query.saved_archive@1") + && principal.subject() == principal.actor() + && principal.subject() == Some(&resource.owner), + "permission_denied", + )?; + } + self.ceiling(principal, operation, &resource.id, store.authority(), now)?; + let chain = self.chain(&resource, store)?; + let mut presentation = None; + if resource.kind == "file" + && matches!(resource.name.as_str(), "ABOUT.md" | "HEADER.svg") + { + if let Some(parent) = &resource.parent { + let parent = store.policy_resource(parent)?; + if parent.kind == "topic" { + presentation = Some(parent); + } + } + } + if resource.kind == "topic" + && check.check == Check::Create + && matches!(request_name, Some("ABOUT.md" | "HEADER.svg")) + { + presentation = Some(resource.clone()); + } + let presentation_edit = presentation.is_some() && check.check.write(); + if presentation_edit { + let topic = presentation + .as_ref() + .ok_or(Error("invalid_storage_record"))?; + let mut can_edit = false; + if let Some(subject) = principal.subject() { + can_edit = subject == topic.owner + || subject == ROOT + || store.topic_admin(&topic.id, subject)?; + } + require(can_edit, "topic_admin_required")?; + } + if check.check.write() && principal.subject() != Some(ROOT) { + protect_wiki(&chain, operation)?; + } + 'preview: for item in &chain { + for prefix in ["hosting_preview:", "hosting_preview_file:"] { + if let Some(marker) = store.setting(&format!("{prefix}{}", item.id))? { + if truthy(&marker)? { + let owner = marker + .as_object()? + .get("owner") + .ok_or(Error("invalid_storage_record"))? + .as_str()?; + require( + principal.subject() == Some(owner) + || (principal.subject() == Some(ROOT) + && matches!(entry, Entry::LocalAdmin) + && principal.method() == "local"), + "permission_denied", + )?; + break 'preview; + } + } + } + } + if check.check.write() { + require( + !chain + .iter() + .any(|r| matches!(r.id.as_str(), "r_agents" | "r_rules")), + "system_managed_resource", + )?; + if !presentation_edit && chain.iter().any(|r| r.id == "t_last_will") { + require( + matches!( + operation, + "identity.legacy_put@1" + | "identity.legacy_put@2" + | "identity.legacy_archive@1" + ) && principal.subject() == Some(&resource.owner), + "legacy_directive_only", + )?; + } + let managed: Vec<_> = chain + .windows(2) + .filter(|p| p[0].kind == "user") + .map(|p| p[1].name.as_str()) + .collect(); + if managed + .iter() + .any(|name| matches!(*name, "SOUL.md" | "AGENTS.md" | "notes" | "todos")) + { + let sharing_notes = matches!( + operation, + "sharing.grant@1" + | "sharing.grant@2" + | "sharing.revoke@1" + | "sharing.revoke@2" + | "sharing.link_create@1" + | "sharing.link_revoke@1" + ) && managed + .iter() + .all(|name| !matches!(*name, "SOUL.md" | "AGENTS.md" | "todos")); + require( + (sharing_notes + || matches!( + operation, + "identity.personal_put@1" + | "identity.note_put@1" + | "identity.personal_put@2" + | "identity.note_put@2" + | "identity.soul_visibility@1" + | "identity.note_archive@1" + | "identity.note_restore@1" + | "identity.todo_put@1" + | "identity.todo_put@2" + | "identity.todo_archive@1" + | "identity.todo_restore@1" + )) + && principal.subject() == Some(&resource.owner), + "personal_managed_resource", + )?; + } + if let Some(subject) = principal.subject() { + for ancestor in chain.iter().rev().filter(|r| r.kind == "topic") { + if let Some(expiry) = store.topic_ban(&ancestor.id, subject)? { + require( + expiry + .as_ref() + .map(|t| Timestamp::parse(t)) + .transpose()? + .is_some_and(|time| time <= now), + "topic_banned", + )?; + } + } + } + } + let mut direct = None; + for ancestor in &chain { + direct = store.direct_conversation(&ancestor.id)?; + if direct.is_some() { + break; + } + } + if let Some(direct) = direct { + require( + principal + .subject() + .is_some_and(|s| s == direct.participant_a || s == direct.participant_b), + "permission_denied", + )?; + if matches!(check.check, Check::Read | Check::List | Check::Traverse) { + continue; + } + if check.check == Check::Create + && resource.id == direct.resource_id + && matches!( + operation, + "communication.dm_send@1" | "communication.dm_send@2" + ) + && direct.state == "active" + { + require( + !store.direct_blocked(&direct.participant_a, &direct.participant_b)?, + "dm_blocked", + )?; + continue; + } + if check.check == Check::Write + && resource.kind == "post" + && principal.subject() == Some(&resource.owner) + && operation == "content.post_edit@1" + { + require( + direct.state == "active" + && !store + .direct_blocked(&direct.participant_a, &direct.participant_b)?, + "dm_blocked", + )?; + continue; + } + return Err(Error("dm_controlled_resource")); + } + let tool_access = resource.kind == "tool" + && self.has( + principal, + "tool.use", + operation, + &resource.id, + store.authority(), + now, + )?; + if resource.kind == "tool" { + require(tool_access, "tool_certificate_required")?; + } + let shared_read = check.check == Check::Read + && self.shared_read(principal, &resource, &chain, now, store)?; + for ancestor in chain.iter().take(chain.len().saturating_sub(1)) { + let mut readable = allows(ancestor, principal.subject(), &memberships, "traverse")?; + let minimal_tool = tool_access && ancestor.id == TOOLS; + if !(readable || minimal_tool || shared_read) { + readable = self.has( + principal, + "resource.read_override", + operation, + &ancestor.id, + store.authority(), + now, + )?; + } + require(readable || minimal_tool || shared_read, "permission_denied")?; + require(ancestor.state == "active", "ancestor_inactive")?; + } + if check.check.write() { + require(principal.subject().is_some(), "authentication_required")?; + for ancestor in &chain { + if parse_mode(&ancestor.mode)? & CERTGATE != 0 { + require( + self.has( + principal, + "resource.certified_write", + operation, + &resource.id, + store.authority(), + now, + )?, + "certificate_gate", + )?; + } + } + } + if resource.kind == "tool" { + require( + matches!(check.check, Check::Read | Check::ToolUse), + "tool_read_only", + )?; + continue; + } + if resource.kind == "csr" + && check.check == Check::Read + && principal.subject() != Some(&resource.owner) + { + require( + principal.subject() == Some(store.csr_issuer(&resource.id)?.as_str()) + && self.has( + principal, + "cert.issue", + operation, + &resource.id, + store.authority(), + now, + )?, + "permission_denied", + )?; + continue; + } + let mut allowed = match check.check { + Check::Certgate => self.has( + principal, + "resource.certified_write", + operation, + &resource.id, + store.authority(), + now, + )?, + Check::Chmod | Check::Chgrp | Check::Manage => { + principal.subject() == Some(&resource.owner) + } + Check::Create | Check::Remove => { + allows(&resource, principal.subject(), &memberships, "write")? + && allows(&resource, principal.subject(), &memberships, "traverse")? + } + Check::Chown | Check::Purge | Check::ToolUse => false, + Check::Read => { + allows(&resource, principal.subject(), &memberships, "read")? || shared_read + } + Check::List => allows(&resource, principal.subject(), &memberships, "list")?, + Check::Traverse => { + allows(&resource, principal.subject(), &memberships, "traverse")? + } + Check::Write => allows(&resource, principal.subject(), &memberships, "write")?, + }; + let mut ordinary = + self.ordinary(principal, operation, &resource.id, store.authority())?; + if !ordinary && check.check == Check::Read { + ordinary = + self.tool_output(principal, operation, &resource.id, store.authority(), now)?; + } + if presentation_edit && matches!(check.check, Check::Write | Check::Create) { + allowed = true; + } + allowed &= ordinary; + if !allowed { + if let Some(capability) = check.check.override_capability() { + allowed = self.has( + principal, + capability, + operation, + &resource.id, + store.authority(), + now, + )?; + } + } + require(allowed, "permission_denied")?; + } + Ok(principal + .certificates() + .iter() + .map(|id| ResourceRef { + id: id.clone(), + revision: None, + }) + .collect()) + } +} +fn protect_wiki(chain: &[PolicyResource], operation: &str) -> Result<()> { + if !chain.iter().any(|r| r.id == "t_wiki") { + return Ok(()); + } + let name = operation.split('@').next().unwrap_or(operation); + require( + !matches!( + name, + "content.move" + | "content.archive" + | "content.restore" + | "content.purge" + | "content.chmod" + | "content.chgrp" + | "content.chown" + | "content.topic_configure" + | "content.topic_policy_set" + | "file.delete" + | "content.topic_remove" + | "content.topic_promote" + | "content.topic_demote" + | "content.topic_ban" + | "content.topic_unban" + | "content.topic_invite" + | "content.topic_approve" + ) && !name.starts_with("content.topic_member_"), + "wiki_shared_resource", + ) +} diff --git a/rust/crates/msg-identity/src/certificates.rs b/rust/crates/msg-identity/src/certificates.rs new file mode 100644 index 00000000..896d4a24 --- /dev/null +++ b/rust/crates/msg-identity/src/certificates.rs @@ -0,0 +1,422 @@ +//! Current-state CA chains, issuance limits and live delegation sources. +use crate::{ + models::*, + policy::*, + require, + store::{setting_truthy, AuthorityStore}, +}; +use msg_core::{digest, Error, Result}; +use std::collections::BTreeSet; + +const ROOT: &str = "u_root"; +const ONLINE_CA: &str = "u_online_ca"; +const ONLINE_ISSUABLE: &[&str] = &[ + "identity.basic", + "resource.basic", + "discussion.basic", + "communication.basic", + "discovery.basic", + "transfer.basic", + "group.basic", + "cert.request", + "git.basic", + "hosting.basic", + "keystore.basic", + "batch.basic", + "sharing.basic", + "money.basic", + "store.basic", + "bounty.basic", + "orders.basic", + "delivery.basic", +]; + +pub struct CertificateValidator { + registry: Registry, + root: Certificate, + root_bytes: String, + root_public_key: [u8; 32], + service: String, +} +impl CertificateValidator { + /// The anchor and registry must come from trusted local configuration. + pub fn new( + registry: Registry, + root: Certificate, + root_public_key: [u8; 32], + service: String, + ) -> Result { + let root_bytes = wire(&root)?.canonical()?; + Ok(Self { + registry, + root, + root_bytes, + root_public_key, + service, + }) + } + pub fn root_id(&self) -> &str { + &self.root.resource_id + } + pub fn service(&self) -> &str { + &self.service + } + pub fn validate( + &self, + id: &str, + store: &dyn AuthorityStore, + now: Timestamp, + ) -> Result { + Validation { + validator: self, + store, + now, + work: 0, + } + .walk(id, &BTreeSet::new(), None, None) + } + pub fn validate_publication( + &self, + cert: &Certificate, + csr: &CertificateRequest, + store: &dyn AuthorityStore, + now: Timestamp, + ) -> Result { + let bytes = csr.signing_bytes()?; + require(csr.request_digest == digest(&bytes), "csr_digest_mismatch")?; + msg_crypto::verify(&csr.public_key.0, &bytes, &csr.possession_proof, "csr")?; + require( + cert.subject_id == csr.subject_id + && cert.issuer_id == csr.requested_issuer + && cert.key_id == msg_crypto::key_id(&csr.public_key.0) + && cert.kind == csr.kind + && wire(&cert.grants)? == wire(&csr.grants)? + && wire(&cert.issuance)? == wire(&csr.issuance)? + && wire(&cert.authority_sources)? == wire(&csr.authority_sources)? + && cert.delegation_depth == csr.delegation_depth + && cert.target_service == csr.target_service, + "csr_certificate_mismatch", + )?; + require( + duration_within(cert, csr.requested_ttl_seconds), + "csr_ttl_exceeded", + )?; + Validation { + validator: self, + store, + now, + work: 0, + } + .walk( + &cert.resource_id, + &BTreeSet::new(), + Some(cert), + Some(&csr.resource_id), + ) + } +} +fn duration_within(cert: &Certificate, seconds: u64) -> bool { + let duration = cert.expires_at.0 - cert.not_before.0; + duration + .num_microseconds() + .is_some_and(|micros| micros >= 0 && (micros as u128) <= u128::from(seconds) * 1_000_000) +} + +struct Validation<'a> { + validator: &'a CertificateValidator, + store: &'a dyn AuthorityStore, + now: Timestamp, + work: usize, +} +impl Validation<'_> { + fn spend(&mut self) -> Result<()> { + self.work += 1; + require(self.work <= 4096, "authority_work_limit") + } + fn allowed_issuance(&self, grant: &Grant, policy: &IssuancePolicy) -> Result { + self.validator.registry.validate_grant(grant, self.store)?; + for allowed in &policy.issue_grants { + if grant_subset(grant, allowed, self.store)? { + return Ok(true); + } + } + Ok(false) + } + fn can_issue(&self, parent: &Certificate, capability: &str, resource: &str) -> Result { + for grant in &parent.grants { + if grant_covers(grant, capability, "cert.publish@1", resource, self.store)? { + return Ok(true); + } + } + Ok(false) + } + fn walk( + &mut self, + id: &str, + seen: &BTreeSet, + proposed: Option<&Certificate>, + issuance_resource: Option<&str>, + ) -> Result { + self.spend()?; + require(!seen.contains(id) && seen.len() < 32, "certificate_cycle")?; + let mut seen = seen.clone(); + seen.insert(id.to_owned()); + let cert = match proposed { + Some(cert) => cert.clone(), + None => self.store.certificate(id)?, + }; + require(cert.resource_id == id, "storage_record_mismatch")?; + require( + cert.target_service == self.validator.service, + "wrong_service", + )?; + require( + cert.not_before <= self.now && self.now < cert.expires_at, + "certificate_expired", + )?; + if proposed.is_none() { + require(!self.store.certificate_revoked(id)?, "certificate_revoked")?; + } + if id == self.validator.root_id() { + require( + wire(&cert)?.canonical()? == self.validator.root_bytes, + "trust_anchor_mismatch", + )?; + require( + cert.subject_id == ROOT + && cert.issuer_id == ROOT + && cert.parent_certificate_id.is_none() + && cert.kind == CertificateKind::Ca, + "invalid_root_certificate", + )?; + msg_crypto::verify( + &self.validator.root_public_key, + &cert.signing_bytes()?, + &cert.signature, + "certificate", + )?; + return Ok(cert); + } + require( + cert.subject_id != ROOT && cert.parent_certificate_id.is_some(), + "local_only", + )?; + let parent = self.walk( + cert.parent_certificate_id + .as_deref() + .ok_or(Error("local_only"))?, + &seen, + None, + None, + )?; + require( + parent.kind == CertificateKind::Ca && parent.issuance.is_some(), + "issuer_not_ca", + )?; + let policy = parent.issuance.as_ref().ok_or(Error("issuer_not_ca"))?; + require(cert.issuer_id == parent.subject_id, "issuer_mismatch")?; + let key = self.store.credential(&parent.key_id)?; + require( + key.subject_id == parent.subject_id && key.revoked_at.is_none(), + "issuer_key_revoked", + )?; + msg_crypto::verify( + &key.verifier.0, + &cert.signing_bytes()?, + &cert.signature, + "certificate", + )?; + let subject_key = self.store.credential(&cert.key_id)?; + require( + subject_key.subject_id == cert.subject_id, + "certificate_key_mismatch", + )?; + require(subject_key.current_at(self.now), "certificate_key_revoked")?; + require( + parent.not_before <= cert.not_before && cert.expires_at <= parent.expires_at, + "certificate_validity_escalation", + )?; + require( + duration_within(&cert, policy.max_cert_ttl_seconds), + "certificate_ttl_escalation", + )?; + require( + cert.delegation_depth <= policy.max_delegation_depth, + "delegation_depth_exceeded", + )?; + let resource = match issuance_resource { + Some(id) => id.to_owned(), + None => self + .store + .setting(&format!("certificate_request:{}", cert.resource_id))? + .map(|v| v.as_str().map(str::to_owned)) + .transpose()? + .unwrap_or_else(|| cert.resource_id.clone()), + }; + require( + self.can_issue(&parent, "cert.issue", &resource)?, + "issuer_cannot_issue", + )?; + for grant in &cert.grants { + require( + self.allowed_issuance(grant, policy)?, + "issuance_scope_exceeded", + )?; + } + if cert.subject_id == ONLINE_CA { + require( + parent.resource_id == self.validator.root_id() + && cert.kind == CertificateKind::Ca + && cert + .issuance + .as_ref() + .is_some_and(|p| p.max_child_ca_depth == 0) + && cert.grants.len() == 1 + && cert.grants[0].capability == "cert.issue" + && cert.grants[0].operations.contains("cert.publish@1"), + "online_ca_policy_exceeded", + )?; + for grant in &cert + .issuance + .as_ref() + .ok_or(Error("online_ca_policy_exceeded"))? + .issue_grants + { + require( + ONLINE_ISSUABLE.contains(&grant.capability.as_str()) + && !self + .validator + .registry + .capability(&grant.capability, grant.version)? + .ca_only, + "online_ca_policy_exceeded", + )?; + } + } + if parent.subject_id == ONLINE_CA { + require( + matches!( + cert.kind, + CertificateKind::Identity | CertificateKind::Delegation + ) && (cert.kind != CertificateKind::Delegation + || !cert.authority_sources.is_empty()) + && cert + .grants + .iter() + .all(|g| ONLINE_ISSUABLE.contains(&g.capability.as_str())), + "online_ca_policy_exceeded", + )?; + } + if cert.kind == CertificateKind::Ca { + require( + cert.issuance.is_some() && policy.max_child_ca_depth > 0, + "ca_depth_exceeded", + )?; + let child = cert.issuance.as_ref().ok_or(Error("ca_depth_exceeded"))?; + require( + self.can_issue(&parent, "cert.ca.issue", &resource)?, + "issuer_cannot_issue_ca", + )?; + require( + child.max_child_ca_depth < policy.max_child_ca_depth, + "ca_depth_exceeded", + )?; + let mut level = 1; + let mut ancestor = parent.clone(); + while ancestor.resource_id != self.validator.root_id() { + self.spend()?; + level += 1; + require(level <= 3, "ca_depth_exceeded")?; + ancestor = self.store.certificate( + ancestor + .parent_certificate_id + .as_deref() + .ok_or(Error("issuer_not_ca"))?, + )?; + } + require(child.max_child_ca_depth <= 3 - level, "ca_depth_exceeded")?; + require( + child.max_cert_ttl_seconds <= policy.max_cert_ttl_seconds + && child.max_delegation_depth <= policy.max_delegation_depth, + "issuance_policy_escalation", + )?; + for grant in &child.issue_grants { + require( + self.allowed_issuance(grant, policy)?, + "issuance_scope_exceeded", + )?; + } + } else { + require(cert.issuance.is_none(), "unexpected_issuance_policy")?; + } + for source in &cert.authority_sources { + self.authority(source, &cert, &seen)?; + } + Ok(cert) + } + fn authority( + &mut self, + source: &ResourceRef, + cert: &Certificate, + seen: &BTreeSet, + ) -> Result<()> { + self.spend()?; + let resource = self.store.resource(&source.id)?; + require( + resource.kind == "delegation" && resource.state == "active", + "authority_source_inactive", + )?; + require( + source.revision == resource.revision, + "authority_source_changed", + )?; + let fact: DelegationFact = decode( + &self + .store + .setting(&format!("delegation:{}", source.id))? + .ok_or(Error("authority_source_invalid"))?, + )?; + require(fact.grantee == cert.subject_id, "authority_source_invalid")?; + require(fact.expires_at > self.now, "authority_source_expired")?; + let key = fact + .grantor_credential_id + .as_ref() + .map(|id| self.store.credential(id)) + .transpose()?; + if let Some(key) = &key { + require(key.current_at(self.now), "authority_source_inactive")?; + require( + !setting_truthy(self.store, &format!("identity_archived:{}", key.subject_id))?, + "authority_source_inactive", + )?; + } + for grant in &fact.grants { + require( + self.store.resource(&grant.scope.resource_id)?.owner == fact.grantor, + "authority_source_lost", + )?; + if let Some(key) = &key { + let mut allowed = false; + for parent in &key.ceiling { + allowed |= grant_subset(grant, parent, self.store)?; + } + require(allowed, "authority_source_lost")?; + } + } + for grant in cert + .grants + .iter() + .chain(cert.issuance.iter().flat_map(|p| &p.issue_grants)) + { + let mut allowed = false; + for parent in &fact.grants { + allowed |= grant_subset(grant, parent, self.store)?; + } + require(allowed, "delegation_scope_exceeded")?; + } + if let Some(parent) = &fact.parent_certificate { + self.walk(parent, seen, None, None)?; + } + Ok(()) + } +} diff --git a/rust/crates/msg-identity/src/lib.rs b/rust/crates/msg-identity/src/lib.rs new file mode 100644 index 00000000..481a3907 --- /dev/null +++ b/rust/crates/msg-identity/src/lib.rs @@ -0,0 +1,21 @@ +//! Native identity verification against one current authority snapshot. +//! +//! An authenticated principal is NOT authorization to execute an operation. +//! Full resource authorization and the daemon are separate cutover gates. +pub mod authentication; +pub mod authorization; +pub mod certificates; +pub mod models; +pub mod policy; +pub mod sessions; +pub mod store; + +use msg_core::{Error, Result}; + +pub(crate) fn require(condition: bool, code: &'static str) -> Result<()> { + if condition { + Ok(()) + } else { + Err(Error(code)) + } +} diff --git a/rust/crates/msg-identity/src/models.rs b/rust/crates/msg-identity/src/models.rs new file mode 100644 index 00000000..fa88ebe4 --- /dev/null +++ b/rust/crates/msg-identity/src/models.rs @@ -0,0 +1,292 @@ +//! Typed Python wire records. Credentials and signatures deliberately lack Debug. +use chrono::{DateTime, Datelike, Timelike, Utc}; +use msg_core::{b64, unb64, Error, Json, Result}; +use msg_crypto::Signature; +use serde::{de::DeserializeOwned, Deserialize, Deserializer, Serialize, Serializer}; +use std::collections::{BTreeMap, BTreeSet}; + +pub fn decode(value: &Json) -> Result { + serde_json::from_str(&value.canonical()?).map_err(|_| Error("invalid_identity_record")) +} +pub fn wire(value: &T) -> Result { + Json::parse(&serde_json::to_string(value).map_err(|_| Error("invalid_identity_record"))?) +} + +// Serde normally makes every Option field optional. Python requires fields +// without a dataclass default, even when their explicit value may be null. +fn required_nullable<'de, D, T>(d: D) -> std::result::Result, D::Error> +where + D: Deserializer<'de>, + T: Deserialize<'de>, +{ + Option::::deserialize(d) +} +fn positive_u32<'de, D: Deserializer<'de>>(d: D) -> std::result::Result { + std::num::NonZeroU32::deserialize(d).map(std::num::NonZeroU32::get) +} +fn positive_u64<'de, D: Deserializer<'de>>(d: D) -> std::result::Result { + std::num::NonZeroU64::deserialize(d).map(std::num::NonZeroU64::get) +} + +#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +pub struct Timestamp(pub DateTime); +impl Timestamp { + pub fn parse(value: &str) -> Result { + if !value.ends_with('Z') { + return Err(Error("invalid_datetime")); + } + let time = DateTime::parse_from_rfc3339(value) + .map_err(|_| Error("invalid_datetime"))? + .with_timezone(&Utc); + if !(1..=9999).contains(&time.year()) || time.nanosecond() >= 1_000_000_000 { + return Err(Error("invalid_datetime")); + } + // Python truncates sub-microsecond digits, never rounds them. + Ok(Self( + time.with_nanosecond(time.nanosecond() / 1000 * 1000) + .ok_or(Error("invalid_datetime"))?, + )) + } +} +impl Serialize for Timestamp { + fn serialize(&self, s: S) -> std::result::Result { + s.serialize_str(&self.0.format("%Y-%m-%dT%H:%M:%S%.6fZ").to_string()) + } +} +impl<'de> Deserialize<'de> for Timestamp { + fn deserialize>(d: D) -> std::result::Result { + Self::parse(&String::deserialize(d)?).map_err(serde::de::Error::custom) + } +} + +#[derive(Clone)] +pub struct Verifier(pub Vec); +impl Serialize for Verifier { + fn serialize(&self, s: S) -> std::result::Result { + s.serialize_str(&b64(&self.0)) + } +} +impl<'de> Deserialize<'de> for Verifier { + fn deserialize>(d: D) -> std::result::Result { + unb64(&String::deserialize(d)?, 4096) + .map(Self) + .map_err(serde::de::Error::custom) + } +} + +#[derive(Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ResourceRef { + pub id: String, + #[serde(default)] + pub revision: Option, +} +#[derive(Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Scope { + pub resource_id: String, + #[serde(default)] + pub descendants: bool, +} +#[derive(Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Grant { + pub capability: String, + #[serde(deserialize_with = "positive_u32")] + pub version: u32, + pub scope: Scope, + pub operations: BTreeSet, + pub constraints: BTreeMap, +} +#[derive(Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct IssuancePolicy { + pub issue_grants: Vec, + #[serde(deserialize_with = "positive_u64")] + pub max_cert_ttl_seconds: u64, + pub max_child_ca_depth: u32, + pub max_delegation_depth: u32, +} +#[derive(Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum SubjectKind { + Temporary, + Registered, + Custodial, + System, +} +#[derive(Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Subject { + pub resource_id: String, + pub kind: SubjectKind, + pub primary_group: String, + pub auth_version: u64, + #[serde(default)] + pub local_only: bool, +} +#[derive(Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum CredentialKind { + SigningKey, + SshKey, + Token, +} +#[derive(Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Credential { + pub id: String, + pub subject_id: String, + pub kind: CredentialKind, + pub verifier: Verifier, + pub ceiling: Vec, + pub not_before: Timestamp, + #[serde(deserialize_with = "required_nullable")] + pub expires_at: Option, + #[serde(deserialize_with = "required_nullable")] + pub revoked_at: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub source_credential_id: Option, +} +impl Credential { + pub fn current_at(&self, now: Timestamp) -> bool { + self.revoked_at.is_none() + && self.not_before <= now + && self.expires_at.is_none_or(|expires| now < expires) + } +} +#[derive(Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum CertificateKind { + Identity, + Delegation, + Capability, + Ca, +} +#[derive(Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Certificate { + pub resource_id: String, + pub serial: String, + pub subject_id: String, + pub key_id: String, + pub issuer_id: String, + #[serde(deserialize_with = "required_nullable")] + pub parent_certificate_id: Option, + pub authority_sources: Vec, + pub kind: CertificateKind, + pub grants: Vec, + pub not_before: Timestamp, + pub expires_at: Timestamp, + pub target_service: String, + pub delegation_depth: u32, + #[serde(deserialize_with = "required_nullable")] + pub issuance: Option, + pub signature: Signature, +} +impl Certificate { + pub fn signing_bytes(&self) -> Result> { + let mut fields = wire(self)?.into_object()?; + fields.remove("signature"); + Ok(Json::from_object(fields).canonical()?.into_bytes()) + } +} +#[derive(Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct CertificateRequest { + pub resource_id: String, + pub applicant: String, + pub subject_id: String, + pub requested_issuer: String, + pub public_key: Verifier, + pub kind: CertificateKind, + pub grants: Vec, + #[serde(deserialize_with = "required_nullable")] + pub issuance: Option, + #[serde(deserialize_with = "positive_u64")] + pub requested_ttl_seconds: u64, + pub target_service: String, + pub delegation_depth: u32, + pub authority_sources: Vec, + pub request_digest: String, + pub possession_proof: Signature, +} +impl CertificateRequest { + pub fn signing_bytes(&self) -> Result> { + let mut fields = wire(self)?.into_object()?; + for key in ["resource_id", "request_digest", "possession_proof"] { + fields.remove(key); + } + Ok(Json::from_object(fields).canonical()?.into_bytes()) + } +} + +/// Only authority-relevant fields of a resource are exposed by the store. +/// The original body remains private to storage; it is not a public read response. +#[derive(Clone, Deserialize)] +pub struct ResourceAuthority { + pub id: String, + #[serde(rename = "type")] + pub kind: String, + #[serde(deserialize_with = "required_nullable")] + pub parent: Option, + pub owner: String, + pub group: String, + pub mode: String, + pub state: String, + #[serde(deserialize_with = "required_nullable")] + pub revision: Option, +} +#[derive(Clone, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct DelegationFact { + pub grantor: String, + pub grantee: String, + pub grants: Vec, + pub expires_at: Timestamp, + #[serde(deserialize_with = "required_nullable")] + pub parent_certificate: Option, + #[serde(default)] + pub grantor_credential_id: Option, +} +#[derive(Clone, Deserialize)] +pub struct DelegatedIdentity { + pub grantor: String, + pub certificate_id: String, + pub max_uses: Option, + pub uses: u64, +} + +#[cfg(test)] +mod tests { + use super::*; + #[test] + fn required_null_is_not_a_missing_field() { + let value = Json::parse(r#"{"id":"k","subject_id":"u","kind":"token","verifier":"","ceiling":[],"not_before":"2026-01-01T00:00:00Z","expires_at":null,"revoked_at":null}"#).unwrap(); + assert!(decode::(&value).is_ok()); + for field in ["expires_at", "revoked_at"] { + let mut fields = value.clone().into_object().unwrap(); + fields.remove(field); + assert!(decode::(&Json::from_object(fields)).is_err()); + } + } + #[test] + fn grant_versions_and_issuance_ttls_are_positive() { + let value = Json::parse(r#"{"capability":"x","version":0,"scope":{"resource_id":"r"},"operations":[],"constraints":{}}"#).unwrap(); + assert!(decode::(&value).is_err()); + let value = Json::parse(r#"{"issue_grants":[],"max_cert_ttl_seconds":0,"max_child_ca_depth":0,"max_delegation_depth":0}"#).unwrap(); + assert!(decode::(&value).is_err()); + } + #[test] + fn json_records_keep_large_integers_and_reject_duplicate_members() { + let value = + Json::parse(r#"{"value":1234567890123456789012345678901234567890,"float":1.0}"#) + .unwrap(); + let decoded: BTreeMap = decode(&value).unwrap(); + assert_eq!( + wire(&decoded).unwrap().canonical().unwrap(), + value.canonical().unwrap() + ); + assert!(serde_json::from_str::(r#"{"value":1,"value":2}"#).is_err()); + } +} diff --git a/rust/crates/msg-identity/src/policy.rs b/rust/crates/msg-identity/src/policy.rs new file mode 100644 index 00000000..cb37c205 --- /dev/null +++ b/rust/crates/msg-identity/src/policy.rs @@ -0,0 +1,241 @@ +//! Pure mode/scope/ceiling rules. None of these alone authorizes an operation. +use crate::{models::*, require, store::AuthorityStore}; +use msg_core::{Error, Json, Result}; +use serde::Deserialize; +use std::collections::{BTreeMap, BTreeSet}; + +pub const CERTGATE: u16 = 0o4000; +pub const SETGID: u16 = 0o2000; +pub const STICKY: u16 = 0o1000; + +pub fn parse_mode(value: &str) -> Result { + require( + value.len() == 4 && value.bytes().all(|b| (b'0'..=b'7').contains(&b)), + "invalid_mode", + )?; + u16::from_str_radix(value, 8).map_err(|_| Error("invalid_mode")) +} +pub fn class_bits(mode: u16, class: &str) -> Result { + require(mode <= 0o7777, "invalid_mode")?; + let shift = match class { + "owner" => 6, + "group" => 3, + "other" => 0, + _ => return Err(Error("invalid_permission_class")), + }; + Ok((mode >> shift) & 7) +} +pub fn allows( + resource: &ResourceAuthority, + subject: Option<&str>, + memberships: &BTreeSet, + permission: &str, +) -> Result { + let class = if subject == Some(&resource.owner) { + "owner" + } else if subject.is_some() && memberships.contains(&resource.group) { + "group" + } else { + "other" + }; + let mask = match permission { + "read" | "list" => 4, + "write" => 2, + "execute" | "traverse" => 1, + _ => return Err(Error("invalid_permission")), + }; + Ok(class_bits(parse_mode(&resource.mode)?, class)? & mask == mask) +} +pub fn scope_contains(scope: &Scope, id: &str, store: &dyn AuthorityStore) -> Result { + if scope.resource_id == id { + return Ok(true); + } + if !scope.descendants { + return Ok(false); + } + Ok(store + .ancestors(id)? + .iter() + .any(|r| r.id == scope.resource_id)) +} +pub fn scope_subset(child: &Scope, parent: &Scope, store: &dyn AuthorityStore) -> Result { + if child.resource_id == parent.resource_id { + return Ok(!child.descendants || parent.descendants); + } + Ok(parent.descendants && scope_contains(parent, &child.resource_id, store)?) +} +pub fn grant_covers( + grant: &Grant, + capability: &str, + operation: &str, + id: &str, + store: &dyn AuthorityStore, +) -> Result { + Ok(grant.capability == capability + && grant.version == 1 + && grant.operations.contains(operation) + && scope_contains(&grant.scope, id, store)?) +} +pub fn constraints_subset( + child: &BTreeMap, + parent: &BTreeMap, +) -> Result { + for (name, value) in parent { + let Some(proposed) = child.get(name) else { + return Ok(false); + }; + match name.as_str() { + "hosts" | "methods" | "ports" | "schemes" => { + let allowed = value.as_array()?; + if !proposed + .as_array()? + .iter() + .all(|v| allowed.iter().any(|a| python_equal(a, v))) + { + return Ok(false); + } + } + "max_response_bytes" | "timeout_ms" | "max_redirects" => { + let Ok(p) = proposed.as_integer() else { + return Ok(false); + }; + let q = integral_number(value).ok_or(Error("schema_validation"))?; + if compare_integers(p, &q).is_gt() { + return Ok(false); + } + } + _ if !python_equal(proposed, value) => return Ok(false), + _ => (), + } + } + Ok(true) +} +pub fn grant_subset(child: &Grant, parent: &Grant, store: &dyn AuthorityStore) -> Result { + Ok(child.capability == parent.capability + && child.version == parent.version + && child.operations.is_subset(&parent.operations) + && scope_subset(&child.scope, &parent.scope, store)? + && constraints_subset(&child.constraints, &parent.constraints)?) +} + +/// Trusted registry projection, NOT request data or a certificate-supplied schema. +#[derive(Clone, Deserialize)] +pub struct CapabilityPolicy { + pub name: String, + pub version: u32, + pub scope_types: BTreeSet, + pub operations: BTreeSet, + pub ca_only: bool, + pub constraints_schema: Option, +} +pub struct Registry { + capabilities: BTreeMap<(String, u32), CapabilityPolicy>, +} +impl Registry { + pub fn new(specs: Vec) -> Result { + let mut capabilities = BTreeMap::new(); + for spec in specs { + require( + capabilities + .insert((spec.name.clone(), spec.version), spec) + .is_none(), + "duplicate_capability", + )?; + } + Ok(Self { capabilities }) + } + pub fn capability(&self, name: &str, version: u32) -> Result<&CapabilityPolicy> { + self.capabilities + .get(&(name.to_owned(), version)) + .ok_or(Error("unknown_capability")) + } + pub fn validate_grant(&self, grant: &Grant, store: &dyn AuthorityStore) -> Result<()> { + let spec = self.capability(&grant.capability, grant.version)?; + let resource = store.resource(&grant.scope.resource_id)?; + require( + spec.scope_types.contains(&resource.kind), + "invalid_scope_type", + )?; + require( + !grant.operations.is_empty() && grant.operations.is_subset(&spec.operations), + "invalid_grant_operations", + )?; + match &spec.constraints_schema { + None => require(grant.constraints.is_empty(), "unknown_grant_constraint"), + Some(schema) if schema.id == "schema:network-constraints" => { + validate_network(&grant.constraints) + } + _ => Err(Error("unsupported_constraints_schema")), + } + } +} +// Compare exact integer values, never round a large integer through f64. +fn compare_integers(left: &str, right: &str) -> std::cmp::Ordering { + let (ln, rn) = (left.starts_with('-'), right.starts_with('-')); + if ln != rn { + return rn.cmp(&ln); + } + let (left, right) = (left.trim_start_matches('-'), right.trim_start_matches('-')); + let order = left.len().cmp(&right.len()).then_with(|| left.cmp(right)); + if ln { + order.reverse() + } else { + order + } +} +fn integral_number(value: &Json) -> Option { + if let Ok(integer) = value.as_integer() { + return Some(integer.to_owned()); + } + // JSON Schema considers 1.0 an integer; Python's pure subset rule still + // requires a literal int for the proposed upper bound, as checked above. + let text = value.canonical().ok()?; + let float: f64 = text.parse().ok()?; + if !float.is_finite() || float.fract() != 0.0 { + return None; + } + Some(if float == 0.0 { + "0".to_owned() + } else { + format!("{float:.0}") + }) +} +fn python_equal(left: &Json, right: &Json) -> bool { + if left == right { + return true; + } + if let (Some(left), Some(right)) = (integral_number(left), integral_number(right)) { + return left == right; + } + false +} +fn validate_network(values: &BTreeMap) -> Result<()> { + for (key, value) in values { + let valid = match key.as_str() { + "hosts" | "methods" | "schemes" => value.as_array().is_ok_and(|values| { + values.iter().all(|v| { + v.as_str().is_ok_and(|v| match key.as_str() { + "schemes" => matches!(v, "http" | "https"), + "methods" => matches!( + v, + "GET" | "HEAD" | "POST" | "PUT" | "PATCH" | "DELETE" | "OPTIONS" + ), + _ => true, + }) + }) + }), + "ports" => value.as_array().is_ok_and(|values| { + values.iter().all(|v| { + integral_number(v) + .and_then(|v| v.parse::().ok()) + .is_some_and(|v| v > 0) + }) + }), + "max_redirects" | "timeout_ms" | "max_response_bytes" => integral_number(value) + .is_some_and(|v| !v.starts_with('-') && (key == "max_redirects" || v != "0")), + _ => false, + }; + require(valid, "schema_validation")?; + } + Ok(()) +} diff --git a/rust/crates/msg-identity/src/sessions.rs b/rust/crates/msg-identity/src/sessions.rs new file mode 100644 index 00000000..945ce9a1 --- /dev/null +++ b/rust/crates/msg-identity/src/sessions.rs @@ -0,0 +1,170 @@ +//! Current key-approved browser/API bindings. No OAuth/Bearer revival or issuance. +use crate::{ + models::*, + policy::grant_subset, + require, + store::{truthy, AuthorityStore}, +}; +use msg_core::{Error, Json, Result}; +use serde::Deserialize; +use std::collections::BTreeSet; + +#[derive(Clone, Deserialize)] +pub struct BrowserPolicy { + pub enabled: bool, +} +#[derive(Clone)] +pub struct CredentialState { + pub expires_at: Timestamp, + pub body: Json, +} +#[derive(Deserialize)] +struct Source { + parent: String, + subject: String, + auth_version: u64, + #[serde(default)] + custodial: bool, + #[serde(default)] + ceiling: Vec, + #[serde(default)] + session: Option, +} +fn rejected_family(id: &str) -> bool { + id.starts_with("t_oauth_") || id.starts_with("t_browser_") +} +fn revoked(body: &Json) -> Result { + body.as_object()? + .get("revoked") + .map(truthy) + .transpose() + .map(|v| v.unwrap_or(false)) +} +fn login_binding(body: &Json) -> Result { + body.as_object()? + .get("login_binding") + .map(truthy) + .transpose() + .map(|v| v.unwrap_or(false)) +} +fn get(store: &dyn AuthorityStore, id: &str, now: Timestamp) -> Result { + let state = store.credential_state(id)?.ok_or(Error("invalid_grant"))?; + require(state.expires_at > now, "invalid_grant")?; + Ok(state.body) +} +fn require_ceiling(store: &dyn AuthorityStore, ceiling: &[Grant], parent: &[Grant]) -> Result<()> { + for grant in ceiling { + let mut allowed = false; + for source in parent { + allowed |= grant_subset(grant, source, store)?; + } + require(allowed, "invalid_grant")?; + } + Ok(()) +} +fn require_source( + store: &dyn AuthorityStore, + body: &Json, + now: Timestamp, + custodial_ceiling: &[Grant], + seen: &mut BTreeSet, +) -> Result<()> { + let source: Source = decode(body).map_err(|_| Error("invalid_grant"))?; + let parent = store.credential(&source.parent)?; + let subject = store.subject(&source.subject)?; + require( + parent.subject_id == subject.resource_id + && !rejected_family(&parent.id) + && !subject.local_only + && subject.auth_version == source.auth_version + && parent.current_at(now) + && !login_binding(body)?, + "invalid_grant", + )?; + if source.custodial { + let vault = store.custodial_binding(&subject.resource_id)?; + require( + subject.kind == SubjectKind::Custodial + && vault + .as_ref() + .is_some_and(|(key, status)| key == &parent.id && status == "active"), + "invalid_grant", + )?; + } + require_ceiling( + store, + &source.ceiling, + if source.custodial { + custodial_ceiling + } else { + &parent.ceiling + }, + )?; + if let Some(session) = source.session.filter(|s| !s.is_empty()) { + // Corrupt restored cycles fail closed; they never exhaust a native stack. + require( + seen.len() < 32 && seen.insert(session.clone()), + "invalid_grant", + )?; + let body = get(store, &session, now)?; + require(!revoked(&body)?, "invalid_grant")?; + require_source(store, &body, now, custodial_ceiling, seen)?; + } + Ok(()) +} +pub fn require_binding( + store: &dyn AuthorityStore, + credential: &Credential, + now: Timestamp, + policy: Option<&BrowserPolicy>, + custodial_ceiling: &[Grant], +) -> Result<()> { + // Restoring historical access/family rows must not reactivate those protocols. + require(!rejected_family(&credential.id), "invalid_grant")?; + if let Some(browser) = store.credential_state(&format!("browser:{}", credential.id))? { + require( + policy.is_some_and(|p| p.enabled), + "browser_sessions_disabled", + )?; + let source: Source = decode(&browser.body).map_err(|_| Error("invalid_grant"))?; + require( + credential.id.starts_with("t_session_") + && credential.source_credential_id.as_deref() == Some(&source.parent) + && credential.subject_id == source.subject + && !source.custodial + && !login_binding(&browser.body)?, + "invalid_grant", + )?; + require_source( + store, + &browser.body, + now, + custodial_ceiling, + &mut BTreeSet::new(), + )?; + return require_ceiling(store, &credential.ceiling, &source.ceiling); + } + if let Some(api) = store.credential_state(&format!("api:{}", credential.id))? { + let source: Source = decode(&api.body).map_err(|_| Error("invalid_grant"))?; + require( + credential.source_credential_id.as_deref() == Some(&source.parent), + "invalid_grant", + )?; + require_source( + store, + &api.body, + now, + custodial_ceiling, + &mut BTreeSet::new(), + )?; + return require_ceiling( + store, + &credential.ceiling, + &store.credential(&source.parent)?.ceiling, + ); + } + require( + !credential.id.starts_with("t_session_") && credential.source_credential_id.is_none(), + "invalid_grant", + ) +} diff --git a/rust/crates/msg-identity/src/store.rs b/rust/crates/msg-identity/src/store.rs new file mode 100644 index 00000000..39fe9571 --- /dev/null +++ b/rust/crates/msg-identity/src/store.rs @@ -0,0 +1,132 @@ +//! Narrow current-state read port. A snapshot must not mix database generations. +use crate::{models::*, require}; +use msg_core::{Error, Json, Result}; +use std::collections::BTreeSet; + +pub struct RecoveryDelivery { + pub verifier: String, + pub expires_at: Timestamp, + pub consumed: bool, + pub request_id: String, +} + +#[derive(Clone, Copy)] +pub enum Record { + Subject, + Credential, + Certificate, + Resource, +} + +pub trait AuthorityStore { + fn record(&self, kind: Record, id: &str) -> Result; + fn setting(&self, key: &str) -> Result>; + /// Presence, not decoded truthiness: even `false`/`null` quarantine denies. + fn has_setting(&self, key: &str) -> Result; + fn certificate_revoked(&self, id: &str) -> Result; + fn request_result(&self, subject: Option<&str>, id: &str, digest: &str) + -> Result>; + fn credential_state(&self, id: &str) -> Result>; + fn custodial_binding(&self, subject: &str) -> Result>; + fn recovery_delivery( + &self, + credential: &str, + subject: &str, + ) -> Result>; + + fn subject(&self, id: &str) -> Result { + let value: Subject = decode(&self.record(Record::Subject, id)?)?; + require(value.resource_id == id, "storage_record_mismatch")?; + Ok(value) + } + fn credential(&self, id: &str) -> Result { + let value: Credential = decode(&self.record(Record::Credential, id)?)?; + require(value.id == id, "storage_record_mismatch")?; + Ok(value) + } + fn certificate(&self, id: &str) -> Result { + let value: Certificate = decode(&self.record(Record::Certificate, id)?)?; + require(value.resource_id == id, "storage_record_mismatch")?; + Ok(value) + } + fn resource(&self, id: &str) -> Result { + let value: ResourceAuthority = decode(&self.record(Record::Resource, id)?)?; + require(value.id == id, "storage_record_mismatch")?; + Ok(value) + } + fn ancestors(&self, id: &str) -> Result> { + let mut seen = BTreeSet::new(); + let mut result = Vec::new(); + let mut current = self.resource(id)?; + while let Some(parent) = ¤t.parent { + require(seen.insert(current.id.clone()), "parent_cycle")?; + require(seen.len() <= 1024, "authority_work_limit")?; + current = self.resource(parent)?; + result.push(current.clone()); + } + result.reverse(); + Ok(result) + } +} + +pub fn truthy(value: &Json) -> Result { + if value.is_null() { + return Ok(false); + } + if let Ok(value) = value.as_bool() { + return Ok(value); + } + if let Ok(value) = value.as_str() { + return Ok(!value.is_empty()); + } + if let Ok(value) = value.as_array() { + return Ok(!value.is_empty()); + } + if let Ok(value) = value.as_object() { + return Ok(!value.is_empty()); + } + // Arbitrary precision integers are not converted through f64. + let encoded = value.canonical()?; + Ok(!matches!(encoded.as_str(), "0" | "0.0" | "-0.0")) +} + +pub fn setting_truthy(store: &dyn AuthorityStore, key: &str) -> Result { + store + .setting(key)? + .as_ref() + .map(truthy) + .transpose() + .map(|v| v.unwrap_or(false)) +} + +/// Once a runtime generation is stale it cannot recover without a new instance. +/// No public reset function, and a malformed generation latches the same fence. +pub struct RuntimeGeneration { + generation: Option, + stale: bool, +} +impl RuntimeGeneration { + pub fn capture(store: &dyn AuthorityStore) -> Result { + Ok(Self { + generation: Self::read(store)?, + stale: false, + }) + } + fn read(store: &dyn AuthorityStore) -> Result> { + store + .setting("recovery_runtime_generation")? + .map(|v| { + let s = v.as_str().map_err(|_| Error("recovery_runtime_stale"))?; + require(!s.is_empty(), "recovery_runtime_stale")?; + Ok(s.to_owned()) + }) + .transpose() + } + pub fn require_current(&mut self, store: &dyn AuthorityStore) -> Result<()> { + match Self::read(store) { + Ok(value) if value == self.generation => (), + _ => self.stale = true, + } + require(!self.stale, "recovery_runtime_stale") + } +} diff --git a/rust/crates/msg-identity/tests/sessions.rs b/rust/crates/msg-identity/tests/sessions.rs new file mode 100644 index 00000000..1e1510ff --- /dev/null +++ b/rust/crates/msg-identity/tests/sessions.rs @@ -0,0 +1,141 @@ +use msg_core::{Error, Json, Result}; +use msg_identity::{ + models::*, + sessions::{require_binding, BrowserPolicy, CredentialState}, + store::{AuthorityStore, Record, RecoveryDelivery}, +}; +use std::collections::BTreeMap; + +struct Memory { + states: BTreeMap, +} +fn now() -> Timestamp { + Timestamp::parse("2026-10-05T00:00:00Z").unwrap() +} +fn credential(id: &str, parent: Option<&str>) -> Credential { + Credential { + id: id.into(), + subject_id: "u_fixture".into(), + kind: CredentialKind::Token, + verifier: Verifier(vec![0; 32]), + ceiling: vec![], + not_before: now(), + expires_at: Some(Timestamp::parse("2030-01-01T00:00:00Z").unwrap()), + revoked_at: None, + source_credential_id: parent.map(String::from), + } +} +fn source(session: Option<&str>) -> CredentialState { + let body = serde_json::json!({"parent":"t_parent","subject":"u_fixture","auth_version":1,"ceiling":[],"session":session}); + CredentialState { + expires_at: Timestamp::parse("2030-01-01T00:00:00Z").unwrap(), + body: Json::parse(&body.to_string()).unwrap(), + } +} +impl AuthorityStore for Memory { + fn record(&self, kind: Record, _id: &str) -> Result { + match kind { + Record::Subject => wire(&Subject { + resource_id: "u_fixture".into(), + kind: SubjectKind::Registered, + primary_group: "g_fixture".into(), + auth_version: 1, + local_only: false, + }), + Record::Credential => wire(&credential("t_parent", None)), + _ => Err(Error("not_found")), + } + } + fn setting(&self, _key: &str) -> Result> { + Ok(None) + } + fn has_setting(&self, _key: &str) -> Result { + Ok(false) + } + fn certificate_revoked(&self, _id: &str) -> Result { + Ok(false) + } + fn request_result( + &self, + _subject: Option<&str>, + _id: &str, + _digest: &str, + ) -> Result> { + Ok(None) + } + fn credential_state(&self, id: &str) -> Result> { + Ok(self.states.get(id).cloned()) + } + fn custodial_binding(&self, _subject: &str) -> Result> { + Ok(None) + } + fn recovery_delivery( + &self, + _credential: &str, + _subject: &str, + ) -> Result> { + Ok(None) + } +} +#[test] +fn browser_disabled_does_not_accept_a_valid_stored_binding() { + let memory = Memory { + states: BTreeMap::from([("browser:t_session_fixture".into(), source(None))]), + }; + let c = credential("t_session_fixture", Some("t_parent")); + for policy in [None, Some(BrowserPolicy { enabled: false })] { + assert!(matches!( + require_binding(&memory, &c, now(), policy.as_ref(), &[]), + Err(Error("browser_sessions_disabled")) + )); + } + require_binding( + &memory, + &c, + now(), + Some(&BrowserPolicy { enabled: true }), + &[], + ) + .unwrap(); +} +#[test] +fn restored_session_cycle_is_bounded_and_denied() { + let memory = Memory { + states: BTreeMap::from([ + ("api:t_api_fixture".into(), source(Some("session:cycle"))), + ("session:cycle".into(), source(Some("session:cycle"))), + ]), + }; + assert!(matches!( + require_binding( + &memory, + &credential("t_api_fixture", Some("t_parent")), + now(), + None, + &[] + ), + Err(Error("invalid_grant")) + )); +} +#[test] +fn restored_excessive_source_depth_is_bounded_and_denied() { + let mut memory = Memory { + states: BTreeMap::from([("api:t_api_fixture".into(), source(Some("session:0")))]), + }; + for i in 0..33 { + let next = (i < 32).then(|| format!("session:{}", i + 1)); + memory + .states + .insert(format!("session:{i}"), source(next.as_deref())); + } + assert!(matches!( + require_binding( + &memory, + &credential("t_api_fixture", Some("t_parent")), + now(), + None, + &[] + ), + Err(Error("invalid_grant")) + )); +} diff --git a/rust/crates/msg-protocol/Cargo.toml b/rust/crates/msg-protocol/Cargo.toml new file mode 100644 index 00000000..df689549 --- /dev/null +++ b/rust/crates/msg-protocol/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "msg-protocol" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +publish.workspace = true + +[dependencies] +chrono.workspace = true +serde_json.workspace = true +msg-core = { path = "../msg-core" } +msg-crypto = { path = "../msg-crypto" } + +[lints] +workspace = true diff --git a/rust/crates/msg-protocol/examples/wire_check.rs b/rust/crates/msg-protocol/examples/wire_check.rs new file mode 100644 index 00000000..3cd0a3d3 --- /dev/null +++ b/rust/crates/msg-protocol/examples/wire_check.rs @@ -0,0 +1,90 @@ +//! JSONL conformance driver. Not shipped as msg/msgd; no network or storage. +//! The deterministic signer is public test material, never a service identity. +use std::io::{self, BufRead, Read, Write}; + +use msg_core::{b64, digest, unb64, Error, Json, MAX_BYTES}; +use msg_crypto::{key_id, subject_id, Ed25519Signer, Signature}; +use msg_protocol::Request; +use serde_json::{json, Value}; + +fn text<'a>(value: &'a Value, key: &str) -> Result<&'a str, Error> { + value + .get(key) + .and_then(Value::as_str) + .ok_or(Error("invalid_test_input")) +} +fn run(value: Value) -> Result { + match text(&value, "action")? { + "canonical" => { + let canonical = Json::parse(text(&value, "raw")?)?.canonical()?; + Ok(json!({"canonical":canonical, "digest":digest(canonical.as_bytes())})) + } + "base64" => { + let bytes = unb64(text(&value, "raw")?, MAX_BYTES)?; + Ok(json!({"encoded":b64(&bytes), "digest":digest(&bytes)})) + } + "request" => { + let request = Request::parse(text(&value, "raw")?)?; + let verified = match value.get("public").and_then(Value::as_str) { + Some(public) => Some(request.verify_signature_bytes(&unb64(public, 32)?).is_ok()), + None => None, + }; + Ok( + json!({"canonical":request.canonical()?, "payload_digest":request.payload_digest()?, "signing":b64(&request.signing_bytes()?), "digest_valid":request.verify_payload_digest().is_ok(), "verified":verified}), + ) + } + "sign" => { + let signer = Ed25519Signer::from_seed(&std::array::from_fn(|i| i as u8)); + let payload = unb64(text(&value, "payload")?, MAX_BYTES)?; + let signature = signer.sign(&payload, text(&value, "purpose")?)?; + Ok( + json!({"public":b64(&signer.public_key()), "key_id":key_id(&signer.public_key()), "subject_id":subject_id(&signer.public_key()), "signature":signature}), + ) + } + "verify" => { + let signature: Signature = serde_json::from_value( + value + .get("signature") + .cloned() + .ok_or(Error("invalid_test_input"))?, + ) + .map_err(|_| Error("invalid_test_input"))?; + msg_crypto::verify( + &unb64(text(&value, "public")?, 32)?, + &unb64(text(&value, "payload")?, MAX_BYTES)?, + &signature, + text(&value, "purpose")?, + )?; + Ok(json!({"verified":true})) + } + _ => Err(Error("unknown_test_action")), + } +} +fn main() -> io::Result<()> { + let stdin = io::stdin(); + let mut reader = stdin.lock(); + let mut stdout = io::BufWriter::new(io::stdout().lock()); + loop { + let mut raw = String::new(); + // Bound the test transport too, before allocating an unbounded line. + let read = (&mut reader) + .take((MAX_BYTES * 8 + 1) as u64) + .read_line(&mut raw)?; + if read == 0 { + break; + } + if raw.len() > MAX_BYTES * 8 { + return Err(io::Error::other("test_input_too_large")); + } + let result = serde_json::from_str(&raw) + .map_err(|_| Error("invalid_test_input")) + .and_then(run); + let output = match result { + Ok(data) => json!({"ok":true, "data":data}), + Err(error) => json!({"ok":false, "code":error.0}), + }; + writeln!(stdout, "{output}")?; + stdout.flush()?; + } + Ok(()) +} diff --git a/rust/crates/msg-protocol/src/lib.rs b/rust/crates/msg-protocol/src/lib.rs new file mode 100644 index 00000000..0e708eb7 --- /dev/null +++ b/rust/crates/msg-protocol/src/lib.rs @@ -0,0 +1,276 @@ +//! Transport-neutral envelope decoding and signed bytes, without an executor. +//! Verifying a signature here does not authorize any resource operation. +use std::collections::{BTreeMap, BTreeSet}; + +use chrono::{DateTime, Datelike, Timelike, Utc}; +use msg_core::{digest, unb64, Error, Json, Result, MAX_BYTES}; +use msg_crypto::Signature; + +const REQUIRED: [&str; 11] = [ + "request_id", + "protocol_version", + "operation", + "contract_version", + "target_service", + "subject", + "arguments", + "expected_generations", + "expires_at", + "payload_digest", + "proof", +]; + +/// No Debug or public field access: proofs can contain bearer credentials. +pub struct Request { + fields: BTreeMap, +} + +fn check(condition: bool) -> Result<()> { + if condition { + Ok(()) + } else { + Err(Error("invalid_request_envelope")) + } +} +fn string(value: &Json, min: usize, max: usize) -> Result<&str> { + let text = value.as_str()?; + check((min..=max).contains(&text.chars().count()))?; + Ok(text) +} +fn nonnegative(value: &Json, positive: bool) -> Result<()> { + let integer = value.as_integer()?; + check(!integer.starts_with('-') && (!positive || integer != "0")) +} +fn exact_fields(fields: &BTreeMap, names: &[&str]) -> Result<()> { + check(fields.len() == names.len() && names.iter().all(|name| fields.contains_key(*name))) +} +fn strings(value: &Json, max_items: usize, min_length: usize, max_length: usize) -> Result<()> { + let items = value.as_array()?; + check(items.len() <= max_items)?; + for item in items { + string(item, min_length, max_length)?; + } + Ok(()) +} +fn signature_shape(value: &Json) -> Result<()> { + let signature = value.as_object()?; + exact_fields(signature, &["key_id", "algorithm", "value"])?; + string(&signature["key_id"], 1, 160)?; + check(signature["algorithm"].as_str()? == "ed25519")?; + unb64(signature["value"].as_str()?, MAX_BYTES)?; + Ok(()) +} +fn proof_shape(value: &Json) -> Result<()> { + if value.is_null() { + return Ok(()); + } + let proof = value.as_object()?; + if proof.contains_key("signature") { + exact_fields(proof, &["signature", "certificates"])?; + signature_shape(&proof["signature"])?; + strings(&proof["certificates"], 16, 1, 160) + } else { + exact_fields(proof, &["credential_id", "token"])?; + string(&proof["credential_id"], 1, 160)?; + unb64(proof["token"].as_str()?, MAX_BYTES)?; + Ok(()) + } +} + +impl Request { + pub fn parse(raw: &str) -> Result { + let fields = Json::parse(raw)?.into_object()?; + let result = Self::decode(fields); + // Do not expose parser internals or rejected request/proof contents. + result.map_err(|error| match error.0 { + "invalid_type" => Error("invalid_request_envelope"), + _ => error, + }) + } + + fn decode(mut fields: BTreeMap) -> Result { + check(REQUIRED.iter().all(|name| fields.contains_key(*name)))?; + check(fields.keys().all(|name| { + REQUIRED.contains(&name.as_str()) || matches!(name.as_str(), "source" | "return_fields") + }))?; + fields + .entry("return_fields".to_owned()) + .or_insert_with(Json::empty_array); + fields + .entry("source".to_owned()) + .or_insert_with(|| Json::string("unknown")); + string(&fields["request_id"], 1, 128)?; + check(fields["protocol_version"].as_integer()? == "1")?; + nonnegative(&fields["contract_version"], true)?; + string(&fields["target_service"], 0, 2048)?; + if !fields["subject"].is_null() { + string(&fields["subject"], 0, 160)?; + } + fields["arguments"].as_object()?; + let operation = string(&fields["operation"], 1, 128)?; + check( + operation.contains('.') + && operation.split('.').all(|part| { + let mut bytes = part.bytes(); + bytes.next().is_some_and(|first| first.is_ascii_lowercase()) + && bytes.all(|byte| { + byte.is_ascii_lowercase() || byte.is_ascii_digit() || byte == b'_' + }) + }), + )?; + let expected = fields["expected_generations"].as_array()?; + check(expected.len() <= 64)?; + let mut seen = BTreeSet::new(); + for item in expected { + let pair = item.as_array()?; + check(pair.len() == 2)?; + let id = string(&pair[0], 1, 160)?; + nonnegative(&pair[1], false)?; + if !seen.insert(id) { + return Err(Error("duplicate_expected_generation")); + } + } + let payload_digest = fields["payload_digest"].as_str()?; + check( + payload_digest.len() == 71 + && payload_digest.starts_with("sha256:") + && payload_digest[7..] + .bytes() + .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte)), + )?; + check(matches!( + fields["source"].as_str()?, + "msg" | "manual" | "mcp" | "unknown" + ))?; + strings(&fields["return_fields"], 30, 0, MAX_BYTES)?; + proof_shape(&fields["proof"])?; + if !fields["expires_at"].is_null() { + let text = string(&fields["expires_at"], 1, 40)?; + check(text.ends_with('Z'))?; + // The rollout profile accepts RFC3339 UTC. Python CLI emits this + // form. Non-RFC3339 datetime.fromisoformat aliases remain gated. + let time = DateTime::parse_from_rfc3339(text).map_err(|_| Error("invalid_datetime"))?; + check((1..=9999).contains(&time.year()) && time.nanosecond() < 1_000_000_000)?; + let normalized = time + .with_timezone(&Utc) + .format("%Y-%m-%dT%H:%M:%S%.6fZ") + .to_string(); + fields.insert("expires_at".to_owned(), Json::string(normalized)); + } + let request = Self { fields }; + request.canonical()?; + Ok(request) + } + + /// Immutable decoded fields, for trusted native server layers only. + /// No mutable accessor: validation cannot be invalidated after construction. + pub fn field(&self, name: &str) -> Result<&Json> { + self.fields.get(name).ok_or(Error("missing_field")) + } + + pub fn canonical(&self) -> Result { + Json::from_object(self.fields.clone()).canonical() + } + pub fn payload_digest(&self) -> Result { + let mut payload = self.fields.clone(); + for field in ["proof", "payload_digest", "source", "expires_at"] { + payload.remove(field); + } + Ok(digest(Json::from_object(payload).canonical()?.as_bytes())) + } + pub fn signing_bytes(&self) -> Result> { + let mut signed = self.fields.clone(); + signed.remove("proof"); + Ok(Json::from_object(signed).canonical()?.into_bytes()) + } + pub fn verify_payload_digest(&self) -> Result<()> { + if self.fields["payload_digest"].as_str()? == self.payload_digest()? { + Ok(()) + } else { + Err(Error("payload_digest_mismatch")) + } + } + pub fn verify_signature_bytes(&self, public: &[u8]) -> Result<()> { + self.verify_payload_digest()?; + let proof = self.fields["proof"] + .as_object() + .map_err(|_| Error("signature_required"))?; + let raw = proof + .get("signature") + .ok_or(Error("signature_required"))? + .canonical()?; + let signature: Signature = + serde_json::from_str(&raw).map_err(|_| Error("invalid_signature"))?; + msg_crypto::verify(public, &self.signing_bytes()?, &signature, "request") + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn packet() -> serde_json::Value { + serde_json::json!({ + "protocol_version":1, "request_id":"test-1", "operation":"discovery.get", + "contract_version":1, "target_service":"https://msg.lmm.best", "subject":null, + "arguments":{"id":"r_agents"}, "expected_generations":[], + "expires_at":"2026-10-05T00:00:00Z", "payload_digest":format!("sha256:{}", "0".repeat(64)), "proof":null + }) + } + + #[test] + fn retry_digest_excludes_only_transport_metadata() { + let mut value = packet(); + let first = Request::parse(&value.to_string()).unwrap(); + value["expires_at"] = "2026-10-05T00:02:00Z".into(); + value["source"] = "mcp".into(); + let refreshed = Request::parse(&value.to_string()).unwrap(); + assert_eq!( + first.payload_digest().unwrap(), + refreshed.payload_digest().unwrap() + ); + assert_ne!( + first.signing_bytes().unwrap(), + refreshed.signing_bytes().unwrap() + ); + value["request_id"] = "other".into(); + assert_ne!( + first.payload_digest().unwrap(), + Request::parse(&value.to_string()) + .unwrap() + .payload_digest() + .unwrap() + ); + } + + #[test] + fn defaults_and_timestamp_match_python_record() { + let request = Request::parse(&packet().to_string()).unwrap(); + let canonical = request.canonical().unwrap(); + assert!(canonical.contains("\"source\":\"unknown\"")); + assert!(canonical.contains("\"return_fields\":[]")); + assert!(canonical.contains("2026-10-05T00:00:00.000000Z")); + } + + #[test] + fn rejects_unknown_fields_and_bool_integers() { + for (key, bad) in [ + ("extra", serde_json::json!(0)), + ("protocol_version", serde_json::json!(true)), + ("contract_version", serde_json::json!(1.0)), + ("expected_generations", serde_json::json!([["r", true]])), + ( + "expected_generations", + serde_json::json!([["r", 0], ["r", 1]]), + ), + ( + "proof", + serde_json::json!({"credential_id":"c", "token":"YQ=="}), + ), + ] { + let mut value = packet(); + value[key] = bad; + assert!(Request::parse(&value.to_string()).is_err()); + } + } +} diff --git a/rust/crates/msg-storage/Cargo.toml b/rust/crates/msg-storage/Cargo.toml new file mode 100644 index 00000000..b0722f12 --- /dev/null +++ b/rust/crates/msg-storage/Cargo.toml @@ -0,0 +1,31 @@ +[package] +name = "msg-storage" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +publish.workspace = true + +[dependencies] +msg-core = { path = "../msg-core" } +msg-identity = { path = "../msg-identity" } +rusqlite.workspace = true +serde.workspace = true +serde_json.workspace = true +msg-crypto = { path = "../msg-crypto" } + +postgres.workspace = true +postgres-native-tls.workspace = true +native-tls.workspace = true +bytes.workspace = true +sha2.workspace = true + +[target.'cfg(unix)'.dependencies] +fs2.workspace = true +libc.workspace = true + +[dev-dependencies] +msg-protocol = { path = "../msg-protocol" } + +[lints] +workspace = true diff --git a/rust/crates/msg-storage/examples/authority_check.rs b/rust/crates/msg-storage/examples/authority_check.rs new file mode 100644 index 00000000..af6480c7 --- /dev/null +++ b/rust/crates/msg-storage/examples/authority_check.rs @@ -0,0 +1,338 @@ +//! Local JSONL differential-test helper. Never register this as an HTTP/MCP tool. +//! Inputs are synthetic public fixtures; errors contain codes, never proof bytes. +use msg_core::{digest, unb64, Error, Json, Result, MAX_BYTES}; +use msg_identity::{ + authentication::{Admission, AuthenticationService, Entry, OperationPolicy}, + authorization::{ + AccessRequirement, AuthorizationService, AuthorizationStore, Check, ResourceTypePolicy, + }, + certificates::CertificateValidator, + models::*, + policy::*, + store::Record, +}; +use msg_protocol::Request; +use msg_storage::{postgres::PgStore, SqliteAuthority}; +use serde::Deserialize; +use serde_json::{json, Value}; +use std::{ + collections::BTreeSet, + io::{self, BufRead, Read, Write}, + path::Path, +}; + +#[derive(Deserialize)] +struct Context { + root: Certificate, + root_public: String, + service: String, + capabilities: Vec, + primary_ceiling: Vec, + temporary_ceiling: Vec, + browser: msg_identity::sessions::BrowserPolicy, + #[serde(default)] + resource_types: Vec, + #[serde(default)] + base_families: BTreeSet, +} +impl Context { + fn validator(&self) -> Result { + let public: [u8; 32] = unb64(&self.root_public, 32)? + .try_into() + .map_err(|_| Error("invalid_test_input"))?; + CertificateValidator::new( + Registry::new(self.capabilities.clone())?, + self.root.clone(), + public, + self.service.clone(), + ) + } +} +fn text<'a>(input: &'a Value, key: &str) -> Result<&'a str> { + input + .get(key) + .and_then(Value::as_str) + .ok_or(Error("invalid_test_input")) +} +fn value(input: &Value, key: &str) -> Result { + Json::parse( + &input + .get(key) + .ok_or(Error("invalid_test_input"))? + .to_string(), + ) +} +fn run( + input: Value, + context: &Context, + auth: &mut AuthenticationService, + authorization: &mut AuthorizationService, + session: &dyn AuthorizationStore, +) -> Result { + let store = session.authority(); + let now = || Timestamp::parse(text(&input, "now")?); + match text(&input, "action")? { + "authenticate" => { + let request = Request::parse(text(&input, "raw")?)?; + let policy: OperationPolicy = decode(&value(&input, "policy")?)?; + let entry = match text(&input, "entry")? { + "network" => Entry::Network, + "local_admin" => Entry::LocalAdmin, + "worker" => Entry::Worker, + _ => return Err(Error("invalid_test_input")), + }; + match auth.authenticate(&request, &policy, store, entry, now()?)? { + Admission::Fresh(p) => Ok(json!({"kind":"fresh", "principal":p})), + Admission::Replay(result) => Ok(json!({"kind":"replay", "result":result})), + } + } + "authorize" | "base" | "has" => { + let request = Request::parse(text(&input, "raw")?)?; + let policy: OperationPolicy = decode(&value(&input, "policy")?)?; + let entry = match text(&input, "entry")? { + "network" => Entry::Network, + "local_admin" => Entry::LocalAdmin, + "worker" => Entry::Worker, + _ => return Err(Error("invalid_test_input")), + }; + let Admission::Fresh(principal) = + auth.authenticate(&request, &policy, store, entry, now()?)? + else { + return Err(Error("test_fresh_principal_required")); + }; + match text(&input, "action")? { + "base" => { + authorization.require_base( + &principal, + text(&input, "operation")?, + text(&input, "id")?, + store, + now()?, + )?; + Ok(json!({"allowed":true})) + } + "has" => Ok( + json!({"allowed":authorization.has(&principal, text(&input,"capability")?, text(&input,"operation")?, text(&input,"id")?, store, now()?)?}), + ), + _ => { + let mut checks = Vec::new(); + for item in input + .get("checks") + .and_then(Value::as_array) + .ok_or(Error("invalid_test_input"))? + { + checks.push(AccessRequirement { + resource_id: text(item, "resource_id")?.to_owned(), + operation: text(item, "operation")?.to_owned(), + check: Check::parse(text(item, "check")?)?, + }); + } + let refs = authorization.require( + &principal, + entry, + now()?, + &checks, + input.get("request_name").and_then(Value::as_str), + session, + )?; + Ok(json!({"refs":refs})) + } + } + } + "memberships" => Ok(json!({"groups":session.memberships(text(&input,"subject")?)?})), + "share_source" => { + let resource = session.policy_resource(text(&input, "id")?)?; + Ok( + json!({"allowed":authorization.share_source_active(&resource,text(&input,"grant")?,text(&input,"subject")?,now()?,session,input.get("reshare").and_then(Value::as_bool).unwrap_or(false))?}), + ) + } + "share_link" => { + let resource = session.policy_resource(text(&input, "id")?)?; + let mut chain = store + .ancestors(&resource.id)? + .iter() + .map(|r| session.policy_resource(&r.id)) + .collect::>>()?; + chain.push(resource.clone()); + Ok( + json!({"allowed":authorization.share_link_read_allowed(text(&input,"grantor")?,text(&input,"credential")?,&resource,&chain,now()?,session)?}), + ) + } + "certificate" => { + let cert = context + .validator()? + .validate(text(&input, "id")?, store, now()?)?; + Ok( + json!({"canonical":wire(&cert)?.canonical()?, "signing_digest":digest(&cert.signing_bytes()?)}), + ) + } + "publication" => { + let cert: Certificate = decode(&value(&input, "certificate")?)?; + let csr: CertificateRequest = decode(&value(&input, "csr")?)?; + let validated = + context + .validator()? + .validate_publication(&cert, &csr, store, now()?)?; + Ok(json!({"canonical":wire(&validated)?.canonical()?})) + } + "result" => Ok( + json!({"result":store.request_result(input.get("subject").and_then(Value::as_str), text(&input,"id")?, text(&input,"digest")?)?}), + ), + "setting" => Ok( + json!({"exists":store.has_setting(text(&input,"key")?)?, "value":store.setting(text(&input,"key")?)?}), + ), + "record_digest" => { + let kind = match text(&input, "kind")? { + "subject" => Record::Subject, + "credential" => Record::Credential, + "certificate" => Record::Certificate, + "resource" => Record::Resource, + _ => return Err(Error("invalid_test_input")), + }; + let raw = store.record(kind, text(&input, "id")?)?; + let normalized = match kind { + Record::Subject => wire(&decode::(&raw)?)?, + Record::Credential => wire(&decode::(&raw)?)?, + Record::Certificate => wire(&decode::(&raw)?)?, + Record::Resource => raw, + }; + Ok(json!({"digest":digest(normalized.canonical()?.as_bytes())})) + } + "scope" => Ok( + json!({"contains":scope_contains(&decode(&value(&input,"scope")?)?, text(&input,"id")?, store)?}), + ), + "subset" => Ok( + json!({"subset":grant_subset(&decode(&value(&input,"child")?)?, &decode(&value(&input,"parent")?)?, store)?}), + ), + "modes" => { + let mut decisions = Vec::new(); + let mut resource = ResourceAuthority { + id: "r".into(), + kind: "file".into(), + parent: None, + owner: "owner".into(), + group: "group".into(), + mode: String::new(), + state: "active".into(), + revision: None, + }; + let group = BTreeSet::from(["group".to_owned()]); + for mode in 0..=0o7777 { + resource.mode = format!("{mode:04o}"); + for subject in [Some("owner"), Some("member"), None] { + for permission in ["read", "write", "execute", "list", "traverse"] { + decisions.push(allows(&resource, subject, &group, permission)?); + } + } + } + Ok( + json!({"count":decisions.len(), "digest":digest(wire(&decisions)?.canonical()?.as_bytes())}), + ) + } + _ => Err(Error("unknown_test_action")), + } +} +fn read_input(reader: &mut impl BufRead) -> io::Result> { + let mut raw = String::new(); + if reader + .take((MAX_BYTES * 8 + 1) as u64) + .read_line(&mut raw)? + == 0 + { + return Ok(None); + } + if raw.len() > MAX_BYTES * 8 { + return Err(io::Error::other("test_input_too_large")); + } + serde_json::from_str(&raw) + .map(Some) + .map_err(|_| io::Error::other("invalid_test_input")) +} +fn output(writer: &mut impl Write, result: Result) -> io::Result<()> { + let value = match result { + Ok(data) => json!({"ok":true,"data":data}), + Err(e) => json!({"ok":false,"code":e.0}), + }; + writeln!(writer, "{value}")?; + writer.flush() +} +fn main() -> io::Result<()> { + let path = std::env::args() + .nth(1) + .ok_or(io::Error::other("test_database_required"))?; + let mut reader = io::stdin().lock(); + let mut writer = io::BufWriter::new(io::stdout().lock()); + let input = read_input(&mut reader)?.ok_or(io::Error::other("test_context_required"))?; + let context: Context = + serde_json::from_value(input).map_err(|_| io::Error::other("invalid_test_context"))?; + enum Backend { + Sqlite(SqliteAuthority), + Postgres(Box), + } + impl Backend { + fn read( + &mut self, + body: impl FnOnce(&dyn AuthorizationStore) -> Result, + ) -> Result { + match self { + Self::Sqlite(store) => body(&store.snapshot()?), + Self::Postgres(store) => store + .transaction(false, |tx| body(tx)) + .map_err(|e| Error(e.code())), + } + } + } + let mut store = match std::env::var("MSG_PARITY_POSTGRES_DSN") { + Ok(dsn) => PgStore::open_existing(&dsn).map(|store| Backend::Postgres(Box::new(store))), + Err(_) => SqliteAuthority::open_existing(Path::new(&path)).map(Backend::Sqlite), + }; + let store = match &mut store { + Ok(store) => store, + Err(e) => { + output(&mut writer, Err(*e))?; + return Ok(()); + } + }; + let (mut auth, mut authorization) = store + .read(|snapshot| { + let auth = AuthenticationService::new( + context.validator()?, + snapshot.authority(), + context.primary_ceiling.clone(), + context.temporary_ceiling.clone(), + )? + .with_browser_policy(context.browser.clone()); + let authorization = AuthorizationService::new( + context.validator()?, + snapshot.authority(), + context.resource_types.clone(), + context.base_families.clone(), + )?; + Ok((auth, authorization)) + }) + .map_err(io::Error::other)?; + output(&mut writer, Ok(json!({"ready":true})))?; + while let Some(input) = read_input(&mut reader)? { + if input.get("action").and_then(Value::as_str) == Some("hold_snapshot") { + let result = store.read(|snapshot| { + output( + &mut writer, + snapshot + .setting("snapshot_probe") + .map(|v| json!({"before":v})), + ) + .map_err(|_| Error("test_io_error"))?; + read_input(&mut reader) + .map_err(|_| Error("test_io_error"))? + .ok_or(Error("test_resume_required"))?; + Ok(json!({"after":snapshot.setting("snapshot_probe")?})) + }); + output(&mut writer, result)?; + } else { + let result = store + .read(|snapshot| run(input, &context, &mut auth, &mut authorization, snapshot)); + output(&mut writer, result)?; + } + } + Ok(()) +} diff --git a/rust/crates/msg-storage/examples/content_check.rs b/rust/crates/msg-storage/examples/content_check.rs new file mode 100644 index 00000000..8ca58491 --- /dev/null +++ b/rust/crates/msg-storage/examples/content_check.rs @@ -0,0 +1,156 @@ +//! Disposable JSONL parity/crash driver, NOT an authenticated operation API. +//! The root must contain an explicit test marker. No listener or Python adapter. +use msg_core::{b64, unb64, Error, Json, Result}; +use msg_storage::{ + content::{ContentOptions, GitContentStore}, + records::{decode, encode, BlobRef}, +}; +use serde::de::DeserializeOwned; +use sha2::{Digest, Sha256}; +use std::{ + io::{self, Read, Write}, + path::Path, +}; +fn get(v: &Json, key: &str) -> Result { + decode(v.as_object()?.get(key).ok_or(Error("fixture_field"))?) +} +fn emit(value: &Json) -> Result<()> { + println!("{}", value.canonical()?); + io::stdout().flush().map_err(|_| Error("fixture_output")) +} +fn hold() -> Result<()> { + emit(&Json::parse(r#"{"checkpoint":"held"}"#)?)?; + let mut input = String::new(); + io::stdin() + .read_line(&mut input) + .map_err(|_| Error("fixture_input"))?; + if input.trim() != "continue" { + return Err(Error("fixture_input")); + } + Ok(()) +} +struct Generated { + remaining: u64, + pause: bool, + started: bool, +} +impl Read for Generated { + fn read(&mut self, bytes: &mut [u8]) -> io::Result { + if self.pause && self.started { + self.pause = false; + hold().map_err(io::Error::other)?; + } + self.started = true; + let length = bytes.len().min(self.remaining as usize); + bytes[..length].fill(b'x'); + self.remaining -= length as u64; + Ok(length) + } +} +struct HashSink { + hash: Sha256, + count: u64, +} +impl Write for HashSink { + fn write(&mut self, bytes: &[u8]) -> io::Result { + self.hash.update(bytes); + self.count += bytes.len() as u64; + Ok(bytes.len()) + } + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } +} +fn dispatch(store: &GitContentStore, value: &Json) -> Result { + match get::(value, "action")?.as_str() { + "put" => { + let bytes = unb64(&get::(value, "bytes")?, 524_288)?; + let expected: Option = get(value, "expected")?; + encode(&store.put_bytes( + &bytes, + &get::(value, "media_type")?, + expected.as_deref(), + )?) + } + "generated" => { + let length: u64 = get(value, "length")?; + if length > 16_777_216 { + return Err(Error("fixture_limit")); + } + let mut input = Generated { + remaining: length, + pause: get(value, "hold_during")?, + started: false, + }; + let blob = store.put( + &mut input, + &get::(value, "media_type")?, + None, + length, + )?; + if get::(value, "hold_after")? { + hold()?; + } + encode(&blob) + } + "read" => { + let blob: BlobRef = get(value, "blob")?; + let range: Option<(u64, u64)> = get(value, "range")?; + if range + .map(|(a, b)| b.saturating_sub(a)) + .unwrap_or(blob.size.max(0) as u64) + > 524_288 + { + return Err(Error("fixture_limit")); + } + let mut bytes = Vec::new(); + store.read_to(&blob, range, &mut bytes)?; + Ok(Json::string(b64(&bytes))) + } + "hash" => { + let mut sink = HashSink { + hash: Sha256::new(), + count: 0, + }; + store.read_to(&get(value, "blob")?, None, &mut sink)?; + encode( + &serde_json::json!({"digest":format!("sha256:{:x}",sink.hash.finalize()),"size":sink.count}), + ) + } + "pin" | "unpin" | "pinned" => { + let blob = get(value, "blob")?; + let lease: String = get(value, "lease")?; + match get::(value, "action")?.as_str() { + "pin" => store.pin(&blob, &lease)?, + "unpin" => store.unpin(&blob, &lease)?, + _ => (), + } + encode(&store.pinned(&blob, &lease)?) + } + "revision" => encode( + &store.commit_revision(&get::(value, "topic")?, &get(value, "revision")?)?, + ), + _ => Err(Error("fixture_action")), + } +} +fn main() -> std::result::Result<(), Box> { + let root = std::env::args().nth(1).ok_or("fixture_root")?; + let root = Path::new(&root); + if std::fs::read(root.join(".msg-native-content-test"))? != b"disposable test content\n" { + return Err("fixture_marker".into()); + } + let store = GitContentStore::open(root, ContentOptions::default())?; + loop { + let mut line = String::new(); + if io::stdin().read_line(&mut line)? == 0 { + break; + } + let result = Json::parse(line.trim_end()).and_then(|value| dispatch(&store, &value)); + let output = match result { + Ok(data) => encode(&serde_json::json!({"data":data}))?, + Err(e) => encode(&serde_json::json!({"error":e.0}))?, + }; + emit(&output)?; + } + Ok(()) +} diff --git a/rust/crates/msg-storage/examples/pg_writer_check.rs b/rust/crates/msg-storage/examples/pg_writer_check.rs new file mode 100644 index 00000000..15784893 --- /dev/null +++ b/rust/crates/msg-storage/examples/pg_writer_check.rs @@ -0,0 +1,217 @@ +//! Disposable PostgreSQL failure-injection driver, NOT an authenticated executor. +//! No network listener, arbitrary SQL, production credentials or Python process. +use msg_core::{Error, Json, Result}; +use msg_identity::store::{AuthorityStore, Record}; +use msg_storage::{ + postgres::{PgFailure, PgSession, PgStore}, + records::{decode, encode, IdentityRecord}, +}; +use serde::de::DeserializeOwned; +use std::io::{self, Write}; +fn field<'a>(value: &'a Json, key: &str) -> Result<&'a Json> { + value.as_object()?.get(key).ok_or(Error("fixture_field")) +} +fn get(value: &Json, key: &str) -> Result { + decode(field(value, key)?) +} +fn flag(value: &Json, key: &str) -> Result { + value + .as_object()? + .get(key) + .map(Json::as_bool) + .transpose() + .map(|v| v.unwrap_or(false)) +} +fn emit(value: &Json) -> Result<()> { + println!("{}", value.canonical()?); + io::stdout().flush().map_err(|_| Error("fixture_output")) +} +fn hold(name: &str) -> Result<()> { + emit(&encode(&serde_json::json!({"checkpoint":name}))?)?; + let mut line = String::new(); + io::stdin() + .read_line(&mut line) + .map_err(|_| Error("fixture_input"))?; + if line.trim() != "continue" { + return Err(Error("fixture_input")); + } + Ok(()) +} +fn steps(tx: &PgSession, values: &Json) -> Result> { + let mut output = Vec::new(); + for step in values.as_array()? { + match field(step, "action")?.as_str()? { + "set" => tx.set_setting(&get::(step, "key")?, field(step, "value")?)?, + "read" => output.push( + tx.setting(&get::(step, "key")?)? + .unwrap_or(Json::parse("null")?), + ), + "resource" => output.push(encode(&tx.resource_record(&get::(step, "id")?)?)?), + "identity" => { + let expected = get(step, "expected")?; + match get::(step, "record_kind")?.as_str() { + "subject" => tx.update_identity( + IdentityRecord::Subject(&get(step, "record")?), + expected, + )?, + "organization" => tx.update_identity( + IdentityRecord::Organization(&get(step, "record")?), + expected, + )?, + "membership" => tx.update_identity( + IdentityRecord::Membership(&get(step, "record")?), + expected, + )?, + "email" => { + tx.update_identity(IdentityRecord::Email(&get(step, "record")?), expected)? + } + _ => return Err(Error("fixture_action")), + } + } + "subject" => output.push(tx.record(Record::Subject, &get::(step, "id")?)?), + "credential" => tx.save_credential(&get(step, "record")?, get(step, "expected")?)?, + "get_credential" => { + output.push(tx.record(Record::Credential, &get::(step, "id")?)?) + } + "csr" => tx.save_csr(&get(step, "record")?)?, + "csr_state" => output.push(encode(&tx.csr_state(&get::(step, "id")?)?)?), + "transition_csr" => tx.transition_csr(&get(step, "record")?, get(step, "expected")?)?, + "certificate" => { + let id: Option = get(step, "csr")?; + tx.register_certificate( + &get(step, "record")?, + id.as_deref() + .map(|id| Ok((id, get(step, "expected")?))) + .transpose()?, + )?; + } + "revoke_certificate" => { + tx.revoke_certificate(&get::(step, "id")?, &get(step, "event")?)? + } + "get_certificate" => { + output.push(tx.record(Record::Certificate, &get::(step, "id")?)?) + } + "transfer" => tx.save_transfer(&get(step, "record")?, get(step, "expected")?)?, + "get_transfer" => output.push(encode(&tx.transfer(&get::(step, "id")?)?)?), + "chunk" => tx.put_chunk(&get(step, "record")?)?, + "missing" => output.push(encode(&tx.missing_ranges( + &get::(step, "id")?, + get::>(step, "cursor")?.as_deref(), + get(step, "limit")?, + )?)?), + "enqueue" => tx.enqueue(&get(step, "record")?)?, + "job" => output.push(encode(&tx.job(&get::(step, "id")?)?)?), + "save_job" => tx.save_job(&get(step, "record")?)?, + "path" => output.push(Json::string(tx.resource_path(&get::(step, "id")?)?)), + "resolve" => output.push(Json::string( + tx.resolve(&get::(step, "path")?, flag(step, "migrated")?)?, + )), + "children" => output.push(encode(&tx.children( + &get::(step, "id")?, + get::>(step, "cursor")?.as_deref(), + get(step, "limit")?, + )?)?), + "history" => output.push(encode(&tx.history( + &get::(step, "id")?, + get::>(step, "cursor")?.as_deref(), + get(step, "limit")?, + )?)?), + "get_revision" => output.push(encode(&tx.revision(&get(step, "reference")?)?)?), + "insert" => tx.insert_resource(&get(step, "record")?)?, + "replace" => tx.replace_resource(&get(step, "record")?, get(step, "expected")?)?, + "revision" => tx.append_revision(&get(step, "record")?)?, + "result" => tx.save_result( + get::>(step, "subject")?.as_deref(), + &get::(step, "digest")?, + &get(step, "record")?, + )?, + "lookup" => output.push( + tx.request_result( + get::>(step, "subject")?.as_deref(), + &get::(step, "id")?, + &get::(step, "digest")?, + )? + .unwrap_or(Json::parse("null")?), + ), + "event" => tx.append_event(&get(step, "record")?)?, + "audit" => tx.append_audit(&get(step, "record")?)?, + "savepoint" => match tx.savepoint(|tx| steps(tx, field(step, "steps")?)) { + Ok(values) => output.push(encode(&values)?), + Err(error) if flag(step, "recover")? => { + output.push(encode(&serde_json::json!({"error":error.0}))?) + } + Err(error) => return Err(error), + }, + "ignore" => { + let _ = steps(tx, field(step, "steps")?); + } + "effect" => { + let name: String = get(step, "name")?; + let fail = flag(step, "fail")?; + let panic = flag(step, "panic")?; + let pause = flag(step, "hold")?; + tx.on_rollback(move || { + emit(&encode(&serde_json::json!({"effect":name}))?)?; + if pause { + hold("compensation")?; + } + assert!(!panic, "fixture cleanup panic"); + if fail { + return Err(Error("fixture_cleanup")); + } + Ok(()) + })?; + } + "hold" => hold("transaction")?, + "fail" => return Err(Error("fixture_failure")), + "panic" => panic!("fixture handler panic"), + // This is only a test of the storage building blocks. There is NO + // authentication or authorization here; never expose it as an API. + "once" => { + let subject: Option = get(step, "subject")?; + let digest: String = get(step, "digest")?; + let record: msg_storage::records::OperationResult = get(step, "record")?; + if let Some(result) = + tx.request_result(subject.as_deref(), &record.request_id, &digest)? + { + output.push(result); + } else { + steps(tx, field(step, "steps")?)?; + tx.save_result(subject.as_deref(), &digest, &record)?; + output.push(encode(&record)?); + } + } + _ => return Err(Error("fixture_action")), + } + } + Ok(output) +} +fn run(case: &Json) -> std::result::Result, PgFailure> { + let dsn = std::env::var("MSG_PARITY_POSTGRES_DSN").map_err(|_| Error("fixture_dsn"))?; + let store = PgStore::open_existing(&dsn)?; + let output = store.transaction(!flag(case, "read_only")?, |tx| { + steps(tx, field(case, "steps")?) + })?; + if flag(case, "hold_after_commit")? { + hold("committed")?; + } + Ok(output) +} +fn main() { + let mut line = String::new(); + let result = (|| { + io::stdin() + .read_line(&mut line) + .map_err(|_| Error("fixture_input"))?; + run(&Json::parse(&line)?) + })(); + let output = match result { + Ok(values) => serde_json::json!({"status":"ok","values":values}), + Err(error) => { + serde_json::json!({"status":"error","code":error.code(),"cleanup":if matches!(error,PgFailure::Rejected{cleanup_failed:true,..}) {vec!["storage_cleanup_failed"]} else {vec![]},"uncertain":matches!(error,PgFailure::Uncertain{..})}) + } + }; + if emit(&encode(&output).expect("fixture result is bounded")).is_err() { + std::process::exit(1); + } +} diff --git a/rust/crates/msg-storage/examples/writer_check.rs b/rust/crates/msg-storage/examples/writer_check.rs new file mode 100644 index 00000000..3d1265c0 --- /dev/null +++ b/rust/crates/msg-storage/examples/writer_check.rs @@ -0,0 +1,155 @@ +//! Disposable local failure-injection driver, NOT an authenticated executor. +//! No network listener, arbitrary SQL, production credentials or Python process. +use msg_core::{Error, Json, Result}; +use msg_identity::store::{AuthorityStore, Record}; +use msg_storage::{ + records::{decode, encode}, + SqliteWriter, WriteFailure, WriteSession, +}; +use serde::de::DeserializeOwned; +use std::{ + io::{self, Write}, + path::Path, + time::Duration, +}; +fn field<'a>(value: &'a Json, key: &str) -> Result<&'a Json> { + value.as_object()?.get(key).ok_or(Error("fixture_field")) +} +fn get(value: &Json, key: &str) -> Result { + decode(field(value, key)?) +} +fn flag(value: &Json, key: &str) -> Result { + value + .as_object()? + .get(key) + .map(Json::as_bool) + .transpose() + .map(|v| v.unwrap_or(false)) +} +fn emit(value: &Json) -> Result<()> { + println!("{}", value.canonical()?); + io::stdout().flush().map_err(|_| Error("fixture_output")) +} +fn hold(name: &str) -> Result<()> { + emit(&encode(&serde_json::json!({"checkpoint":name}))?)?; + let mut line = String::new(); + io::stdin() + .read_line(&mut line) + .map_err(|_| Error("fixture_input"))?; + if line.trim() != "continue" { + return Err(Error("fixture_input")); + } + Ok(()) +} +fn steps(tx: &mut WriteSession<'_>, values: &Json) -> Result> { + let mut output = Vec::new(); + for step in values.as_array()? { + match field(step, "action")?.as_str()? { + "set" => tx.set_setting(&get::(step, "key")?, field(step, "value")?)?, + "read" => output.push( + tx.setting(&get::(step, "key")?)? + .unwrap_or(Json::parse("null")?), + ), + "resource" => output.push(tx.record(Record::Resource, &get::(step, "id")?)?), + "insert" => tx.insert_resource(&get(step, "record")?)?, + "replace" => tx.replace_resource(&get(step, "record")?, get(step, "expected")?)?, + "revision" => tx.append_revision(&get(step, "record")?)?, + "result" => tx.save_result( + get::>(step, "subject")?.as_deref(), + &get::(step, "digest")?, + &get(step, "record")?, + )?, + "lookup" => output.push( + tx.request_result( + get::>(step, "subject")?.as_deref(), + &get::(step, "id")?, + &get::(step, "digest")?, + )? + .unwrap_or(Json::parse("null")?), + ), + "event" => tx.append_event(&get(step, "record")?)?, + "audit" => tx.append_audit(&get(step, "record")?)?, + "savepoint" => match tx.savepoint(|tx| steps(tx, field(step, "steps")?)) { + Ok(values) => output.push(encode(&values)?), + Err(error) if flag(step, "recover")? => { + output.push(encode(&serde_json::json!({"error":error.0}))?) + } + Err(error) => return Err(error), + }, + "ignore" => { + let _ = steps(tx, field(step, "steps")?); + } + "effect" => { + let name: String = get(step, "name")?; + let fail = flag(step, "fail")?; + let panic = flag(step, "panic")?; + let pause = flag(step, "hold")?; + tx.on_rollback(move || { + emit(&encode(&serde_json::json!({"effect":name}))?)?; + if pause { + hold("compensation")?; + } + assert!(!panic, "fixture cleanup panic"); + if fail { + return Err(Error("fixture_cleanup")); + } + Ok(()) + })?; + } + "hold" => hold("transaction")?, + "fail" => return Err(Error("fixture_failure")), + "panic" => panic!("fixture handler panic"), + // This is only a test of the storage building blocks. There is NO + // authentication or authorization here; never expose it as an API. + "once" => { + let subject: Option = get(step, "subject")?; + let digest: String = get(step, "digest")?; + let record: msg_storage::records::OperationResult = get(step, "record")?; + if let Some(result) = + tx.request_result(subject.as_deref(), &record.request_id, &digest)? + { + output.push(result); + } else { + steps(tx, field(step, "steps")?)?; + tx.save_result(subject.as_deref(), &digest, &record)?; + output.push(encode(&record)?); + } + } + _ => return Err(Error("fixture_action")), + } + } + Ok(output) +} +fn run(path: &Path, case: &Json) -> std::result::Result, WriteFailure> { + let timeout = case + .as_object()? + .get("timeout_ms") + .map(decode::) + .transpose()? + .unwrap_or(2000); + let mut writer = SqliteWriter::open_existing(path, Duration::from_millis(timeout))?; + let output = writer.transaction(|tx| steps(tx, field(case, "steps")?))?; + if flag(case, "hold_after_commit")? { + hold("committed")?; + } + Ok(output) +} +fn main() { + let mut line = String::new(); + let result = (|| { + let path = std::env::args().nth(1).ok_or(Error("fixture_path"))?; + io::stdin() + .read_line(&mut line) + .map_err(|_| Error("fixture_input"))?; + run(Path::new(&path), &Json::parse(&line)?) + })(); + let output = match result { + Ok(values) => serde_json::json!({"status":"ok","values":values}), + Err(error) => { + serde_json::json!({"status":"error","code":error.cause.0,"cleanup":error.cleanup_errors.iter().map(|e| e.0).collect::>()}) + } + }; + if emit(&encode(&output).expect("fixture result is bounded")).is_err() { + std::process::exit(1); + } +} diff --git a/rust/crates/msg-storage/src/authorization.rs b/rust/crates/msg-storage/src/authorization.rs new file mode 100644 index 00000000..7d6bfdad --- /dev/null +++ b/rust/crates/msg-storage/src/authorization.rs @@ -0,0 +1,138 @@ +//! Transaction-bound authorization facts; no policy or implicit permission here. +use crate::records::{Membership, Organization}; +use crate::{database_error, Session}; +use msg_core::{Error, Json, Result, MAX_BYTES}; +use msg_identity::{ + authorization::{AuthorizationStore, DirectConversation, LegacyShare, ShareSource}, + models::{decode, SubjectKind}, + store::AuthorityStore, +}; +use rusqlite::{params, OptionalExtension, Params, Row}; +use std::collections::BTreeSet; + +const MAX_FACTS: usize = 4096; +fn bounded_json(raw: &str) -> Result { + if raw.len() > MAX_BYTES { + return Err(Error("storage_record_too_large")); + } + Json::parse(raw) +} +impl Session<'_, A> { + fn authorization_rows( + &self, + sql: &'static str, + params: impl Params, + decode_row: impl FnMut(&Row<'_>) -> rusqlite::Result, + ) -> Result> { + self.active()?; + let mut stmt = self + .transaction + .prepare_cached(sql) + .map_err(database_error)?; + let rows = stmt + .query_map(params, decode_row) + .map_err(database_error)? + .collect::, _>>() + .map_err(database_error)?; + if rows.len() > MAX_FACTS { + return Err(Error("authority_work_limit")); + } + Ok(rows) + } +} +impl AuthorizationStore for Session<'_, A> { + fn authority(&self) -> &dyn AuthorityStore { + self + } + fn memberships(&self, subject: &str) -> Result> { + let rows: Vec<(String, String)> = self.authorization_rows( + "SELECT m.org,CAST(substr(CAST(m.body AS BLOB),1,1048577) AS TEXT) FROM memberships m JOIN resources r ON r.id=m.org WHERE m.subject=?1 AND r.type='organization' AND r.state='active' ORDER BY m.org LIMIT 4097", + [subject], |r| Ok((r.get(0)?, r.get(1)?)))?; + let mut groups = BTreeSet::new(); + for (org, body) in rows { + let member: Membership = decode(&bounded_json(&body)?)?; + member.validate()?; + if member.status == "active" + && member.organization_id == org + && member.subject_id == subject + && org != "g_public" + { + groups.insert(org); + } + } + match self.subject(subject) { + Ok(identity) + if matches!( + identity.kind, + SubjectKind::Registered | SubjectKind::Custodial | SubjectKind::System + ) && !identity.local_only => + { + groups.insert("g_public".into()); + } + Ok(_) | Err(Error("subject_not_found")) => (), + Err(error) => return Err(error), + } + Ok(groups) + } + fn organization_exists(&self, id: &str) -> Result { + let value = self.json("SELECT CAST(substr(CAST(body AS BLOB),1,1048577) AS TEXT) FROM identities WHERE id=?1 AND kind='organization'", id)?; + match value { + None => Ok(false), + Some(value) => { + let row: Organization = decode(&value)?; + row.validate()?; + if row.resource_id != id { + return Err(Error("storage_record_mismatch")); + } + Ok(true) + } + } + } + fn topic_admin(&self, topic: &str, subject: &str) -> Result { + self.active()?; + self.transaction.query_row("SELECT EXISTS(SELECT 1 FROM topic_memberships WHERE topic=?1 AND subject=?2 AND role='admin' AND status='active')", + [topic, subject], |r| r.get(0)).map_err(database_error) + } + fn topic_ban(&self, topic: &str, subject: &str) -> Result>> { + self.active()?; + self.transaction.query_row("SELECT expires_at FROM topic_bans WHERE topic=?1 AND subject=?2 AND status='active'", [topic, subject], |r| r.get(0)) + .optional().map_err(database_error) + } + fn direct_conversation(&self, resource: &str) -> Result> { + self.active()?; + self.transaction.query_row("SELECT resource_id,participant_a,participant_b,state FROM dm_conversations WHERE resource_id=?1", [resource], + |r| Ok(DirectConversation { resource_id: r.get(0)?, participant_a: r.get(1)?, participant_b: r.get(2)?, state: r.get(3)? })) + .optional().map_err(database_error) + } + fn direct_blocked(&self, a: &str, b: &str) -> Result { + self.active()?; + self.transaction.query_row("SELECT EXISTS(SELECT 1 FROM dm_blocks WHERE (blocker=?1 AND blocked=?2) OR (blocker=?2 AND blocked=?1))", + [a, b], |r| r.get(0)).map_err(database_error) + } + fn legacy_share(&self, resource: &str, grantee: &str) -> Result> { + self.active()?; + self.transaction.query_row("SELECT grantor,expires_at,created_at FROM share_grants WHERE resource_id=?1 AND grantee=?2 AND revoked_at IS NULL", + [resource, grantee], |r| Ok(LegacyShare { grantor: r.get(0)?, expires_at: r.get(1)?, created_at: r.get(2)? })) + .optional().map_err(database_error) + } + fn share_candidates(&self, resource: &str, now: &str) -> Result> { + self.authorization_rows("SELECT id FROM share_grants_v2 WHERE resource_id=?1 AND revoked_at IS NULL AND expires_at>?2 ORDER BY id LIMIT 4097", + params![resource, now], |r| r.get(0)) + } + fn share_source(&self, id: &str) -> Result> { + self.active()?; + self.transaction.query_row("SELECT resource_id,grantor,grantee,grantee_kind,parent_id,substr(operations,1,1048577),substr(constraints,1,1048577),allow_reshare,expires_at,revoked_at,created_at FROM share_grants_v2 WHERE id=?1", + [id], |r| Ok(ShareSource { resource_id: r.get(0)?, grantor: r.get(1)?, grantee: r.get(2)?, grantee_kind: r.get(3)?, parent_id: r.get(4)?, + operations: r.get(5)?, constraints: r.get(6)?, allow_reshare: r.get::<_, i64>(7)? == 1, expires_at: r.get(8)?, revoked_at: r.get(9)?, created_at: r.get(10)? })) + .optional().map_err(database_error) + } + fn csr_issuer(&self, id: &str) -> Result { + let row = self + .json( + "SELECT CAST(substr(CAST(body AS BLOB),1,1048577) AS TEXT) FROM csrs WHERE id=?1", + id, + )? + .ok_or(Error("csr_not_found"))?; + Ok(decode::(&row)?.requested_issuer) + } +} diff --git a/rust/crates/msg-storage/src/content/files.rs b/rust/crates/msg-storage/src/content/files.rs new file mode 100644 index 00000000..fad2d072 --- /dev/null +++ b/rust/crates/msg-storage/src/content/files.rs @@ -0,0 +1,282 @@ +//! Filesystem primitives for service-owned content directories, never user paths. +use msg_core::{Error, Result}; +use sha2::{Digest, Sha256}; +use std::{ + fs::{self, DirBuilder, File, Metadata, OpenOptions, Permissions}, + io::{Read, Seek, SeekFrom, Write}, + os::unix::fs::{DirBuilderExt, MetadataExt, OpenOptionsExt, PermissionsExt}, + path::{Component, Path, PathBuf}, + sync::atomic::{AtomicU64, Ordering}, +}; + +pub(super) fn io(_: std::io::Error) -> Error { + Error("content_store_error") +} +pub(super) fn hex(data: &[u8]) -> String { + format!("{:x}", Sha256::digest(data)) +} +pub(super) fn key(digest: &str) -> Result<&str> { + let key = digest + .strip_prefix("sha256:") + .ok_or(Error("invalid_digest"))?; + if key.len() != 64 + || !key + .bytes() + .all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)) + { + return Err(Error("invalid_digest")); + } + Ok(key) +} +pub(super) fn oid(value: &str) -> Result<&str> { + if ![40, 64].contains(&value.len()) + || !value + .bytes() + .all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)) + { + return Err(Error("invalid_content_index")); + } + Ok(value) +} +pub(super) fn segment(value: &str) -> Result<&str> { + if value.is_empty() + || value.len() > 160 + || [".", ".."].contains(&value) + || !value + .bytes() + .all(|b| b.is_ascii_alphanumeric() || b"._:-".contains(&b)) + { + return Err(Error("invalid_revision_id")); + } + Ok(value) +} + +/// Reject symlink path components. Directory ownership and protection against +/// hostile same-UID processes remain an administrator responsibility; this is +/// not an openat sandbox. No request may choose these configured directories. +pub(super) fn absolute(path: &Path) -> Result { + let path = if path.is_absolute() { + path.to_path_buf() + } else { + std::env::current_dir().map_err(io)?.join(path) + }; + let mut checked = PathBuf::new(); + for component in path.components() { + match component { + Component::ParentDir => return Err(Error("unsafe_content_path")), + Component::CurDir => continue, + _ => checked.push(component), + } + match fs::symlink_metadata(&checked) { + Ok(meta) if meta.file_type().is_symlink() => return Err(Error("unsafe_content_path")), + Ok(_) => (), + Err(e) if e.kind() == std::io::ErrorKind::NotFound => (), + Err(e) => return Err(io(e)), + } + } + Ok(checked) +} +pub(super) fn directory(path: &Path, group: bool) -> Result<()> { + absolute(path)?; + let created = match fs::symlink_metadata(path) { + Ok(_) => false, + Err(e) if e.kind() == std::io::ErrorKind::NotFound => { + DirBuilder::new().mode(0o700).create(path).map_err(io)?; + true + } + Err(e) => return Err(io(e)), + }; + if created && group { + fs::set_permissions(path, Permissions::from_mode(0o2750)).map_err(io)?; + } + check_directory(path, group)?; + if created { + sync_dir(path)?; + if let Some(parent) = path.parent() { + sync_dir(parent)?; + } + } + Ok(()) +} +pub(super) fn check_directory(path: &Path, group: bool) -> Result<()> { + let meta = fs::symlink_metadata(path).map_err(io)?; + if !meta.is_dir() || meta.file_type().is_symlink() || meta.mode() & 0o022 != 0 { + return Err(Error("unsafe_content_path")); + } + if group && meta.mode() & 0o2077 != 0o2050 { + return Err(Error("content_permission_migration_required")); + } + Ok(()) +} +pub(super) fn sync_dir(path: &Path) -> Result<()> { + OpenOptions::new() + .read(true) + .custom_flags(libc::O_NOFOLLOW | libc::O_DIRECTORY | libc::O_CLOEXEC) + .open(path) + .and_then(|f| f.sync_all()) + .map_err(io) +} +pub(super) fn open(path: &Path) -> Result { + let file = OpenOptions::new() + .read(true) + .custom_flags(libc::O_NOFOLLOW | libc::O_NONBLOCK | libc::O_CLOEXEC) + .open(path) + .map_err(|e| { + if e.kind() == std::io::ErrorKind::NotFound { + Error("content_missing") + } else { + Error("unsafe_content_path") + } + })?; + if !file.metadata().map_err(io)?.is_file() { + return Err(Error("unsafe_content_path")); + } + Ok(file) +} +pub(super) fn bounded(path: &Path, limit: usize) -> Result> { + let file = open(path)?; + if file.metadata().map_err(io)?.len() > limit as u64 { + return Err(Error("invalid_content_index")); + } + let mut bytes = Vec::new(); + file.take(limit as u64 + 1) + .read_to_end(&mut bytes) + .map_err(io)?; + if bytes.len() > limit { + return Err(Error("invalid_content_index")); + } + Ok(bytes) +} +fn unchanged(a: &Metadata, b: &Metadata) -> bool { + ( + a.dev(), + a.ino(), + a.len(), + a.mtime(), + a.mtime_nsec(), + a.ctime(), + a.ctime_nsec(), + ) == ( + b.dev(), + b.ino(), + b.len(), + b.mtime(), + b.mtime_nsec(), + b.ctime(), + b.ctime_nsec(), + ) +} +/// Read and hash with constant memory, stopping at a caller-supplied quota. +pub(super) fn stream( + source: &mut impl Read, + target: &mut impl Write, + limit: u64, +) -> Result<(String, u64)> { + let mut hash = Sha256::new(); + let mut total = 0_u64; + let mut buffer = [0; 65_536]; + loop { + let count = match source.read(&mut buffer) { + Err(e) if e.kind() == std::io::ErrorKind::Interrupted => continue, + other => other.map_err(io)?, + }; + if count == 0 { + break; + } + total = total + .checked_add(count as u64) + .ok_or(Error("content_too_large"))?; + if total > limit.min(i64::MAX as u64) { + return Err(Error("content_too_large")); + } + hash.update(&buffer[..count]); + target.write_all(&buffer[..count]).map_err(io)?; + } + Ok((format!("sha256:{:x}", hash.finalize()), total)) +} +pub(super) fn verified_copy( + source: &mut File, + target: &mut impl Write, + digest: &str, + size: u64, +) -> Result<()> { + let before = source.metadata().map_err(io)?; + if before.len() != size { + return Err(Error("content_size_mismatch")); + } + source.seek(SeekFrom::Start(0)).map_err(io)?; + let (actual, length) = stream(source, target, size)?; + if actual != digest || length != size || !unchanged(&before, &source.metadata().map_err(io)?) { + return Err(Error("content_digest_mismatch")); + } + Ok(()) +} +static NEXT_TEMP: AtomicU64 = AtomicU64::new(0); +pub(super) struct Temporary { + pub path: PathBuf, + pub file: File, +} +impl Temporary { + pub fn new(parent: &Path) -> Result { + check_directory(parent, false)?; + for _ in 0..128 { + let path = parent.join(format!( + ".pending-rust-{}-{}", + std::process::id(), + NEXT_TEMP.fetch_add(1, Ordering::Relaxed) + )); + let result = OpenOptions::new() + .read(true) + .write(true) + .create_new(true) + .mode(0o600) + .custom_flags(libc::O_NOFOLLOW | libc::O_CLOEXEC) + .open(&path); + match result { + Ok(file) => return Ok(Self { path, file }), + Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => continue, + Err(e) => return Err(io(e)), + } + } + Err(Error("content_store_error")) + } + pub fn rewind(&mut self) -> Result<()> { + self.file.seek(SeekFrom::Start(0)).map(|_| ()).map_err(io) + } + pub fn permissions(&self, group: bool) -> Result<()> { + self.file + .set_permissions(Permissions::from_mode(if group { 0o640 } else { 0o600 })) + .map_err(io)?; + self.file.sync_all().map_err(io) + } + /// Publish without overwriting an existing inode (also preserves LFS links). + pub fn install(&self, path: &Path) -> Result { + match fs::hard_link(&self.path, path) { + Ok(()) => { + sync_dir(path.parent().ok_or(Error("unsafe_content_path"))?)?; + Ok(true) + } + Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => Ok(false), + Err(e) => Err(io(e)), + } + } +} +impl Drop for Temporary { + fn drop(&mut self) { + let _ = fs::remove_file(&self.path); + } +} + +pub(super) fn replace(path: &Path, bytes: &[u8], group: bool) -> Result<()> { + if let Ok(meta) = fs::symlink_metadata(path) { + if !meta.is_file() || meta.file_type().is_symlink() { + return Err(Error("unsafe_content_path")); + } + } + let parent = path.parent().ok_or(Error("unsafe_content_path"))?; + let mut temp = Temporary::new(parent)?; + temp.file.write_all(bytes).map_err(io)?; + temp.permissions(group)?; + fs::rename(&temp.path, path).map_err(io)?; + sync_dir(parent) +} diff --git a/rust/crates/msg-storage/src/content/git.rs b/rust/crates/msg-storage/src/content/git.rs new file mode 100644 index 00000000..ccdf2ec6 --- /dev/null +++ b/rust/crates/msg-storage/src/content/git.rs @@ -0,0 +1,108 @@ +//! Direct Git plumbing only: no shell, Python, hooks, inherited Git variables, +//! global configuration, stdin pipe deadlock, or unbounded in-memory output. +use super::{ + files::{self, Temporary}, + GitContentStore, +}; +use msg_core::{Error, Result}; +use std::{ + fs::File, + io::{Read, Write}, + process::{Child, Command, Stdio}, + thread, + time::Instant, +}; + +struct Running(Child); +impl Drop for Running { + fn drop(&mut self) { + let _ = self.0.kill(); + let _ = self.0.wait(); + } +} +impl GitContentStore { + pub(super) fn command(&self) -> Command { + let mut command = Command::new(&self.options.git_binary); + command + .env_clear() + .env("PATH", "/usr/bin:/bin") + .env("LANG", "C.UTF-8") + .env("HOME", &self.root) + .env("GIT_CONFIG_NOSYSTEM", "1") + .env("GIT_CONFIG_GLOBAL", "/dev/null") + .env("GIT_NO_REPLACE_OBJECTS", "1") + .env("GIT_TERMINAL_PROMPT", "0") + .env("GIT_AUTHOR_NAME", "msg") + .env("GIT_COMMITTER_NAME", "msg") + .env("GIT_AUTHOR_EMAIL", "msg@localhost") + .env("GIT_COMMITTER_EMAIL", "msg@localhost") + .arg("--git-dir") + .arg(&self.repo) + .args([ + "-c", + "core.fsync=all", + "-c", + "core.logAllRefUpdates=false", + "-c", + "core.hooksPath=/dev/null", + "-c", + "core.fsmonitor=false", + ]) + .current_dir(&self.root); + command + } + pub(super) fn run( + &self, + command: &mut Command, + input: Option, + max_output: u64, + ) -> Result { + let mut output = Temporary::new(&self.staging)?; + command + .stdin(input.map(Stdio::from).unwrap_or_else(Stdio::null)) + .stdout(output.file.try_clone().map_err(files::io)?) + .stderr(Stdio::null()); + let mut child = Running(command.spawn().map_err(files::io)?); + let started = Instant::now(); + loop { + if output.file.metadata().map_err(files::io)?.len() > max_output { + return Err(Error("content_output_too_large")); + } + if let Some(status) = child.0.try_wait().map_err(files::io)? { + if !status.success() { + return Err(Error("content_store_error")); + } + // The child can finish between the first size check and try_wait. + if output.file.metadata().map_err(files::io)?.len() > max_output { + return Err(Error("content_output_too_large")); + } + output.rewind()?; + return Ok(output); + } + if started.elapsed() >= self.options.git_timeout { + return Err(Error("content_store_timeout")); + } + thread::sleep(std::time::Duration::from_millis(5)); + } + } + pub(super) fn git(&self, args: &[&str], bytes: Option<&[u8]>) -> Result { + let mut input = Temporary::new(&self.staging)?; + if let Some(bytes) = bytes { + input.file.write_all(bytes).map_err(files::io)?; + } + input.rewind()?; + let mut result = self.run( + self.command().args(args), + Some(input.file.try_clone().map_err(files::io)?), + 1_048_576, + )?; + let mut text = String::new(); + result.file.read_to_string(&mut text).map_err(files::io)?; + Ok(text.trim_end_matches('\n').to_owned()) + } + pub(super) fn ref_set(&self, reference: &str, oid: &str) -> Result<()> { + files::oid(oid)?; + self.git(&["update-ref", "--no-deref", reference, oid], None) + .map(|_| ()) + } +} diff --git a/rust/crates/msg-storage/src/content/mod.rs b/rust/crates/msg-storage/src/content/mod.rs new file mode 100644 index 00000000..44ae385a --- /dev/null +++ b/rust/crates/msg-storage/src/content/mod.rs @@ -0,0 +1,521 @@ +//! Native Git/blob content storage, compatible with Python's private.git, +//! index, binary, pins and revisions layout. Synchronous and Unix-only. +//! +//! This is a trusted storage port, not an authorization boundary or an executor. +//! The caller supplies quotas and holds the metadata writer fence while coupling +//! mutations to metadata. Wait for an in-flight call to finish before releasing +//! that fence; abandoning an async blocking task is not cancellation. +//! +//! Files and Git refs are durable before successful return. This is NOT an +//! atomic metadata/content commit or a durable cross-store recovery journal. +//! Errors may leave unreferenced content, which must not be blindly deleted. +mod files; +mod git; +#[cfg(test)] +mod tests; +use crate::records::{encode, BlobRef, Revision}; +use files::{io, Temporary}; +use msg_core::{Error, Json, Result}; +use serde::{Deserialize, Serialize}; +use std::{ + fs, + io::{Read, Seek, SeekFrom, Write}, + path::{Path, PathBuf}, + time::Duration, +}; + +/// Administrator-supplied configuration, never deserialized from a request. +#[derive(Clone)] +pub struct ContentOptions { + pub binary_dir: Option, + pub staging_dir: Option, + pub group_read: bool, + pub git_binary: PathBuf, + pub git_timeout: Duration, +} +impl Default for ContentOptions { + fn default() -> Self { + Self { + binary_dir: None, + staging_dir: None, + group_read: false, + git_binary: "/usr/bin/git".into(), + git_timeout: Duration::from_secs(120), + } + } +} +pub struct GitContentStore { + root: PathBuf, + repo: PathBuf, + index: PathBuf, + binary: PathBuf, + staging: PathBuf, + options: ContentOptions, +} +#[derive(Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "lowercase", deny_unknown_fields)] +enum Entry { + Git { oid: String, size: u64 }, + Binary { size: u64 }, +} +impl Entry { + fn size(&self) -> u64 { + match self { + Self::Git { size, .. } | Self::Binary { size } => *size, + } + } +} +impl GitContentStore { + /// Initialize or open a content store in protected, service-owned paths. + /// Existing directory permissions are never repaired implicitly. Parent + /// directories must already exist; shared layouts require explicit migration. + pub fn open(root: &Path, options: ContentOptions) -> Result { + if !options.git_binary.is_absolute() || options.git_timeout.is_zero() { + return Err(Error("invalid_content_configuration")); + } + let root = files::absolute(root)?; + let store = Self { + repo: root.join("private.git"), + index: root.join("index"), + binary: files::absolute( + options + .binary_dir + .as_deref() + .unwrap_or(&root.join("binary")), + )?, + staging: files::absolute( + options + .staging_dir + .as_deref() + .unwrap_or(&root.join("staging")), + )?, + root, + options, + }; + // Do not chmod/create anything before rejecting an existing bad layout. + for path in [&store.root, &store.index, &store.binary] { + if fs::symlink_metadata(path).is_ok() { + files::check_directory(path, store.options.group_read)?; + } + } + for path in [&store.repo, &store.staging] { + if fs::symlink_metadata(path).is_ok() { + files::check_directory(path, false)?; + } + } + files::directory(&store.root, store.options.group_read)?; + files::directory(&store.staging, false)?; + if store.repo.exists() + && store.options.group_read + && store.git(&["config", "--local", "core.sharedRepository"], None)? != "0640" + { + return Err(Error("content_permission_migration_required")); + } + files::directory(&store.index, store.options.group_read)?; + files::directory(&store.binary, store.options.group_read)?; + if !store.repo.exists() { + let mut command = store.command(); + command.args([ + "-c", + "init.templateDir=", + "init", + "--bare", + "--object-format=sha1", + ]); + if store.options.group_read { + command.arg("--shared=0640"); + } + command.arg(&store.repo); + store.run(&mut command, None, 65_536)?; + files::sync_dir(&store.root)?; + } + store.layout()?; + if store.git(&["rev-parse", "--is-bare-repository"], None)? != "true" { + return Err(Error("invalid_content_repository")); + } + Ok(store) + } + fn layout(&self) -> Result<()> { + for path in [ + &self.root, + &self.index, + &self.binary, + &self.staging, + &self.repo, + ] { + files::absolute(path)?; + files::check_directory(path, false)?; + } + Ok(()) + } + fn entry(&self, blob: &BlobRef) -> Result { + self.layout()?; + let key = files::key(&blob.digest)?; + if blob.size < 0 { + return Err(Error("invalid_blob_size")); + } + let bytes = files::bounded(&self.index.join(key), 4096)?; + let json = + Json::parse(std::str::from_utf8(&bytes).map_err(|_| Error("invalid_content_index"))?)?; + let entry: Entry = + serde_json::from_str(&json.canonical()?).map_err(|_| Error("invalid_content_index"))?; + if entry.size() != blob.size as u64 { + return Err(Error("content_size_mismatch")); + } + if let Entry::Git { oid, .. } = &entry { + files::oid(oid)?; + } + Ok(entry) + } + /// Stream into private staging. A wrong expected digest or source error + /// cannot publish an index/ref. `limit` is the caller's admitted byte quota. + pub fn put( + &self, + source: &mut impl Read, + media_type: &str, + expected: Option<&str>, + limit: u64, + ) -> Result { + self.layout()?; + if let Some(expected) = expected { + files::key(expected)?; + } + let mut staged = Temporary::new(&self.staging)?; + let (digest, size) = files::stream(source, &mut staged.file, limit)?; + if expected.is_some_and(|expected| expected != digest) { + return Err(Error("digest_mismatch")); + } + staged.file.sync_all().map_err(io)?; + let key = files::key(&digest)?; + let entry = if media_type.starts_with("text/") + || ["application/json", "application/msg-template"].contains(&media_type) + { + staged.rewind()?; + let mut output = self.run( + self.command().args(["hash-object", "-w", "--stdin"]), + Some(staged.file.try_clone().map_err(io)?), + 128, + )?; + let mut oid = String::new(); + output.file.read_to_string(&mut oid).map_err(io)?; + let oid = files::oid(oid.trim())?.to_owned(); + self.ref_set(&format!("refs/staging/{key}"), &oid)?; + Entry::Git { oid, size } + } else { + let destination = self.binary.join(key); + if fs::symlink_metadata(&destination).is_ok() { + files::verified_copy( + &mut files::open(&destination)?, + &mut std::io::sink(), + &digest, + size, + )?; + } else { + // Create on the destination filesystem; this also inherits the + // intended setgid directory rather than the staging directory's gid. + let mut published = Temporary::new(&self.binary)?; + staged.rewind()?; + files::verified_copy(&mut staged.file, &mut published.file, &digest, size)?; + published.permissions(self.options.group_read)?; + if !published.install(&destination)? { + files::verified_copy( + &mut files::open(&destination)?, + &mut std::io::sink(), + &digest, + size, + )?; + } + } + Entry::Binary { size } + }; + files::replace( + &self.index.join(key), + encode(&entry)?.canonical()?.as_bytes(), + self.options.group_read, + )?; + Ok(BlobRef { + digest, + size: size as i64, + media_type: media_type.into(), + }) + } + pub fn put_bytes( + &self, + bytes: &[u8], + media_type: &str, + expected: Option<&str>, + ) -> Result { + let mut reader = bytes; + self.put(&mut reader, media_type, expected, bytes.len() as u64) + } + fn snapshot(&self, blob: &BlobRef) -> Result { + let entry = self.entry(blob)?; + let mut snapshot = Temporary::new(&self.staging)?; + match entry { + Entry::Binary { .. } => { + files::verified_copy( + &mut files::open(&self.binary.join(files::key(&blob.digest)?))?, + &mut snapshot.file, + &blob.digest, + blob.size as u64, + )?; + } + Entry::Git { oid, .. } => { + snapshot = self.run( + self.command().args(["cat-file", "blob", &oid]), + None, + blob.size as u64, + )?; + files::verified_copy( + &mut snapshot.file, + &mut std::io::sink(), + &blob.digest, + blob.size as u64, + )?; + } + } + snapshot.rewind()?; + Ok(snapshot) + } + /// Deliver from a verified private snapshot using 64 KiB buffers. Integrity + /// is checked before any bytes reach the sink, even for an empty/small range. + /// Snapshot disk usage is O(blob size), memory O(1); the caller enforces + /// concurrency/disk quotas. A failed sink may have accepted a partial range. + pub fn read_to( + &self, + blob: &BlobRef, + range: Option<(u64, u64)>, + sink: &mut impl Write, + ) -> Result<()> { + if blob.size < 0 { + return Err(Error("invalid_blob_size")); + } + let (start, end) = range.unwrap_or((0, blob.size as u64)); + if start > end || end > blob.size as u64 { + return Err(Error("invalid_byte_range")); + } + let mut snapshot = self.snapshot(blob)?; + snapshot.file.seek(SeekFrom::Start(start)).map_err(io)?; + let copied = + std::io::copy(&mut (&mut snapshot.file).take(end - start), sink).map_err(io)?; + if copied != end - start { + return Err(Error("content_truncated")); + } + Ok(()) + } + pub fn read_bytes(&self, blob: &BlobRef, limit: u64) -> Result> { + if blob.size < 0 { + return Err(Error("invalid_blob_size")); + } + if blob.size as u64 > limit { + return Err(Error("use_transfer")); + } + let mut bytes = Vec::new(); + self.read_to(blob, None, &mut bytes)?; + Ok(bytes) + } + fn pin_path(&self, blob: &BlobRef, lease: &str, create: bool) -> Result { + let root = self.root.join("pins"); + let parent = root.join(files::hex(lease.as_bytes())); + if create { + files::directory(&root, false)?; + files::directory(&parent, false)?; + } + files::absolute(&parent)?; + Ok(parent.join(files::key(&blob.digest)?)) + } + pub fn pin(&self, blob: &BlobRef, lease: &str) -> Result<()> { + // Verify that a pin retains precisely the promised bytes, not just a + // well-formed index pointing to a corrupt or different object. + self.snapshot(blob)?; + match self.entry(blob)? { + Entry::Git { oid, .. } => self.ref_set( + &format!( + "refs/pins/{}/{}", + files::hex(lease.as_bytes()), + files::key(&blob.digest)? + ), + &oid, + ), + Entry::Binary { .. } => { + files::replace(&self.pin_path(blob, lease, true)?, b"1\n", false) + } + } + } + pub fn unpin(&self, blob: &BlobRef, lease: &str) -> Result<()> { + match self.entry(blob)? { + Entry::Git { .. } => self + .git( + &[ + "update-ref", + "--no-deref", + "-d", + &format!( + "refs/pins/{}/{}", + files::hex(lease.as_bytes()), + files::key(&blob.digest)? + ), + ], + None, + ) + .map(|_| ()), + Entry::Binary { .. } => { + let path = self.pin_path(blob, lease, false)?; + match fs::symlink_metadata(&path) { + Ok(meta) if meta.file_type().is_symlink() || !meta.is_file() => { + Err(Error("unsafe_content_path")) + } + Ok(_) => { + fs::remove_file(&path).map_err(io)?; + files::sync_dir(path.parent().ok_or(Error("unsafe_content_path"))?) + } + Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(()), + Err(e) => Err(io(e)), + } + } + } + } + pub fn pinned(&self, blob: &BlobRef, lease: &str) -> Result { + match self.entry(blob)? { + Entry::Git { oid, .. } => { + // show-ref's missing-ref status is distinct from a corrupt index; + // check existence via for-each-ref, which succeeds for no matches. + let reference = format!( + "refs/pins/{}/{}", + files::hex(lease.as_bytes()), + files::key(&blob.digest)? + ); + let text = self.git( + &[ + "for-each-ref", + "--format=%(refname) %(objectname)", + &reference, + ], + None, + )?; + Ok(text + .lines() + .any(|line| line == format!("{reference} {oid}"))) + } + Entry::Binary { .. } => match files::bounded(&self.pin_path(blob, lease, false)?, 2) { + Ok(bytes) if bytes == b"1\n" => Ok(true), + Ok(_) => Err(Error("invalid_content_pin")), + Err(Error("content_missing")) => Ok(false), + Err(e) => Err(e), + }, + } + } + /// Commit canonical revision bytes and content to the existing topic refs. + /// The domain layer still validates signatures, parent authority and quotas. + /// Existing IDs are immutable: a same-manifest retry reuses the stored commit; + /// conflicting reuse fails instead of overwriting history. + pub fn commit_revision(&self, topic: &str, revision: &Revision) -> Result { + files::segment(&revision.id)?; + if revision.parents.len() > 64 { + return Err(Error("too_many_revision_parents")); + } + for parent in &revision.parents { + files::segment(parent)?; + } + self.snapshot(&revision.content)?; + let manifest = encode(revision)?.canonical()?; + let manifest_oid = + self.git(&["hash-object", "-w", "--stdin"], Some(manifest.as_bytes()))?; + files::oid(&manifest_oid)?; + let mut tree_input = format!("100644 blob {manifest_oid}\tmanifest.json\n"); + if let Entry::Git { oid, .. } = self.entry(&revision.content)? { + tree_input.push_str(&format!("100644 blob {oid}\tcontent\n")); + } + let tree = self.git(&["mktree"], Some(tree_input.as_bytes()))?; + files::oid(&tree)?; + let directory = self.root.join("revisions"); + files::directory(&directory, false)?; + let parents: Vec = revision + .parents + .iter() + .map(|parent| { + let bytes = files::bounded(&directory.join(parent), 128).map_err(|e| { + if e == Error("content_missing") { + Error("revision_content_missing") + } else { + e + } + })?; + Ok(files::oid( + std::str::from_utf8(&bytes) + .map_err(|_| Error("invalid_content_index"))? + .trim(), + )? + .to_owned()) + }) + .collect::>()?; + let path = directory.join(&revision.id); + let mut commit = match files::bounded(&path, 128) { + Ok(bytes) => files::oid( + std::str::from_utf8(&bytes) + .map_err(|_| Error("invalid_content_index"))? + .trim(), + )? + .to_owned(), + Err(Error("content_missing")) => { + let mut command = self.command(); + command.args(["commit-tree", &tree]); + for parent in &parents { + command.args(["-p", parent]); + } + // Deterministic native commits; Python-created existing commits + // retain their original wall-clock timestamps and object IDs. + let date = revision.created_at.0.to_rfc3339(); + command + .env("GIT_AUTHOR_DATE", &date) + .env("GIT_COMMITTER_DATE", &date); + let mut input = Temporary::new(&self.staging)?; + writeln!(input.file, "{}", revision.id).map_err(io)?; + input.rewind()?; + let mut output = + self.run(&mut command, Some(input.file.try_clone().map_err(io)?), 128)?; + let mut text = String::new(); + output.file.read_to_string(&mut text).map_err(io)?; + files::oid(text.trim())?.to_owned() + } + Err(e) => return Err(e), + }; + self.check_commit(&commit, &tree, &parents)?; + let reference = format!( + "refs/topics/{}/{}/{}", + files::hex(topic.as_bytes()), + files::hex(revision.resource_id.as_bytes()), + files::hex(revision.id.as_bytes()) + ); + self.ref_set(&reference, &commit)?; + let mut mapping = Temporary::new(&directory)?; + writeln!(mapping.file, "{commit}").map_err(io)?; + mapping.permissions(false)?; + if !mapping.install(&path)? { + let existing = files::bounded(&path, 128)?; + commit = files::oid( + std::str::from_utf8(&existing) + .map_err(|_| Error("invalid_content_index"))? + .trim(), + )? + .to_owned(); + self.check_commit(&commit, &tree, &parents)?; + self.ref_set(&reference, &commit)?; + } + Ok(commit) + } + fn check_commit(&self, commit: &str, tree: &str, parents: &[String]) -> Result<()> { + let body = self.git(&["cat-file", "commit", files::oid(commit)?], None)?; + let headers: Vec<&str> = body.split("\n\n").next().unwrap_or("").lines().collect(); + let actual: Vec<&str> = headers + .iter() + .filter_map(|line| line.strip_prefix("parent ")) + .collect(); + if headers.first() != Some(&format!("tree {tree}").as_str()) + || actual != parents.iter().map(String::as_str).collect::>() + { + return Err(Error("revision_content_conflict")); + } + Ok(()) + } +} diff --git a/rust/crates/msg-storage/src/content/tests.rs b/rust/crates/msg-storage/src/content/tests.rs new file mode 100644 index 00000000..ebf0abb0 --- /dev/null +++ b/rust/crates/msg-storage/src/content/tests.rs @@ -0,0 +1,397 @@ +use super::*; +use crate::records::decode; +use std::{ + fs::{DirBuilder, Permissions}, + io, + os::unix::fs::{symlink, DirBuilderExt, MetadataExt, PermissionsExt}, + sync::atomic::{AtomicU64, Ordering}, +}; +static NEXT: AtomicU64 = AtomicU64::new(0); +struct Fixture(PathBuf); +impl Fixture { + fn new() -> Self { + let path = std::env::temp_dir().join(format!( + "msg-native-content-{}-{}", + std::process::id(), + NEXT.fetch_add(1, Ordering::Relaxed) + )); + DirBuilder::new().mode(0o700).create(&path).unwrap(); + Self(path) + } + fn store(&self) -> GitContentStore { + GitContentStore::open(&self.0.join("store"), ContentOptions::default()).unwrap() + } +} +impl Drop for Fixture { + fn drop(&mut self) { + let _ = fs::remove_dir_all(&self.0); + } +} +fn revision(blob: &BlobRef, id: &str, parents: &[&str]) -> Revision { + decode( + &encode( + &serde_json::json!({"format_version":1,"id":id,"resource_id":"r_test", + "parents":parents,"content":blob,"relations":[],"actor":"u_alice","subject":"u_alice", + "author":"u_alice","created_at":"2026-10-06T00:00:00.000000Z", + "manifest_digest":format!("sha256:{}", "a".repeat(64)),"signature":null}), + ) + .unwrap(), + ) + .unwrap() +} +#[test] +fn storage_kinds_and_empty_content_round_trip() { + let f = Fixture::new(); + let store = f.store(); + for media in [ + "text/markdown", + "text/plain", + "application/json", + "application/msg-template", + "application/octet-stream", + ] { + for data in [b"".as_slice(), "Hello\0中文\n".as_bytes()] { + let blob = store.put_bytes(data, media, None).unwrap(); + assert_eq!(store.read_bytes(&blob, 1024).unwrap(), data); + assert_eq!(blob.digest, msg_core::digest(data)); + } + } + assert_eq!(fs::read_dir(&store.staging).unwrap().count(), 0); +} +#[test] +fn ranges_cross_buffers_and_reject_invalid_bounds() { + let f = Fixture::new(); + let store = f.store(); + let data: Vec = (0..200_001).map(|n| (n % 251) as u8).collect(); + for media in ["text/plain", "application/octet-stream"] { + let blob = store.put_bytes(&data, media, None).unwrap(); + for (start, end) in [ + (0, 0), + (0, 1), + (65_532, 65_540), + (5, 180_000), + (200_001, 200_001), + ] { + let mut output = Vec::new(); + store + .read_to(&blob, Some((start, end)), &mut output) + .unwrap(); + assert_eq!(output, data[start as usize..end as usize]); + } + assert_eq!( + store.read_to(&blob, Some((2, 1)), &mut Vec::new()).err(), + Some(Error("invalid_byte_range")) + ); + assert_eq!( + store + .read_to(&blob, Some((0, 200_002)), &mut Vec::new()) + .err(), + Some(Error("invalid_byte_range")) + ); + assert_eq!( + store.read_bytes(&blob, 200_000).err(), + Some(Error("use_transfer")) + ); + } +} +#[test] +fn failed_source_quota_and_digest_do_not_publish() { + struct Broken; + impl Read for Broken { + fn read(&mut self, _: &mut [u8]) -> io::Result { + Err(io::Error::other("fixture")) + } + } + let f = Fixture::new(); + let store = f.store(); + assert_eq!( + store.put(&mut Broken, "text/plain", None, 1024).err(), + Some(Error("content_store_error")) + ); + assert_eq!( + store.put(&mut &b"abc"[..], "text/plain", None, 2).err(), + Some(Error("content_too_large")) + ); + assert_eq!( + store + .put_bytes(b"abc", "text/plain", Some(&msg_core::digest(b"different"))) + .err(), + Some(Error("digest_mismatch")) + ); + assert_eq!( + store + .put_bytes(b"abc", "text/plain", Some("sha256:../wrong")) + .err(), + Some(Error("invalid_digest")) + ); + for path in [&store.staging, &store.index, &store.binary] { + assert_eq!(fs::read_dir(path).unwrap().count(), 0); + } + assert_eq!(store.git(&["for-each-ref"], None).unwrap(), ""); +} +#[test] +fn binary_duplicate_keeps_existing_hard_link_inode() { + let f = Fixture::new(); + let store = f.store(); + let blob = store + .put_bytes(b"same binary", "application/octet-stream", None) + .unwrap(); + let path = store.binary.join(files::key(&blob.digest).unwrap()); + let link = f.0.join("lfs-link"); + fs::hard_link(&path, &link).unwrap(); + let before = fs::metadata(&path).unwrap(); + store + .put_bytes(b"same binary", "application/octet-stream", None) + .unwrap(); + assert_eq!(fs::metadata(&path).unwrap().ino(), before.ino()); + assert_eq!(fs::metadata(link).unwrap().nlink(), 2); +} +#[test] +fn corrupt_binary_never_reaches_sink_or_gets_replaced() { + let f = Fixture::new(); + let store = f.store(); + let blob = store + .put_bytes(b"correct", "application/octet-stream", None) + .unwrap(); + let path = store.binary.join(files::key(&blob.digest).unwrap()); + fs::write(&path, b"corrupt").unwrap(); + let mut sink = Vec::new(); + assert_eq!( + store.read_to(&blob, None, &mut sink).err(), + Some(Error("content_digest_mismatch")) + ); + assert!(sink.is_empty()); + assert_eq!( + store + .put_bytes(b"correct", "application/octet-stream", None) + .err(), + Some(Error("content_digest_mismatch")) + ); + assert_eq!(fs::read(path).unwrap(), b"corrupt"); +} +#[test] +fn corrupt_index_and_truncated_binary_fail_closed() { + let f = Fixture::new(); + let store = f.store(); + let blob = store + .put_bytes(b"correct", "application/octet-stream", None) + .unwrap(); + fs::write(store.binary.join(files::key(&blob.digest).unwrap()), b"x").unwrap(); + assert_eq!( + store.read_bytes(&blob, 1024).err(), + Some(Error("content_size_mismatch")) + ); + let index = store.index.join(files::key(&blob.digest).unwrap()); + for value in [ + r#"{"kind":"git","oid":"--help","size":7}"#, + r#"{"kind":"binary","size":7,"size":7}"#, + r#"{"kind":"binary","size":-1}"#, + ] { + fs::write(&index, value).unwrap(); + assert!(store.read_bytes(&blob, 1024).is_err()); + } +} +#[test] +fn bad_digest_and_negative_size_cannot_escape_root() { + let f = Fixture::new(); + let store = f.store(); + for digest in [ + "../outside", + "sha256:../../etc/passwd", + "sha256:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", + ] { + let blob = BlobRef { + digest: digest.into(), + size: 1, + media_type: "text/plain".into(), + }; + assert_eq!( + store.read_bytes(&blob, 10).err(), + Some(Error("invalid_digest")) + ); + } + let blob = BlobRef { + digest: msg_core::digest(b""), + size: -1, + media_type: "text/plain".into(), + }; + assert_eq!( + store.read_bytes(&blob, 10).err(), + Some(Error("invalid_blob_size")) + ); +} +#[test] +fn symlink_roots_indices_and_binary_objects_are_rejected() { + let f = Fixture::new(); + let store = f.store(); + symlink(&store.root, f.0.join("alias")).unwrap(); + assert!(GitContentStore::open(&f.0.join("alias"), ContentOptions::default()).is_err()); + let blob = store + .put_bytes(b"abc", "application/octet-stream", None) + .unwrap(); + let path = store.binary.join(files::key(&blob.digest).unwrap()); + let external = f.0.join("external"); + fs::write(&external, b"abc").unwrap(); + fs::remove_file(&path).unwrap(); + symlink(&external, &path).unwrap(); + assert_eq!( + store.read_bytes(&blob, 10).err(), + Some(Error("unsafe_content_path")) + ); + assert_eq!( + store + .put_bytes(b"abc", "application/octet-stream", None) + .err(), + Some(Error("unsafe_content_path")) + ); + let index = store.index.join(files::key(&blob.digest).unwrap()); + fs::remove_file(&index).unwrap(); + symlink(&external, index).unwrap(); + assert_eq!( + store.read_bytes(&blob, 10).err(), + Some(Error("unsafe_content_path")) + ); + assert_eq!(fs::read(external).unwrap(), b"abc"); +} +#[test] +fn pins_are_hashed_and_idempotent() { + let f = Fixture::new(); + let store = f.store(); + for media in ["text/plain", "application/octet-stream"] { + let blob = store.put_bytes(b"pin me", media, None).unwrap(); + let lease = "../../untrusted/lease\0中文"; + assert!(!store.pinned(&blob, lease).unwrap()); + store.pin(&blob, lease).unwrap(); + store.pin(&blob, lease).unwrap(); + assert!(store.pinned(&blob, lease).unwrap()); + store.unpin(&blob, lease).unwrap(); + store.unpin(&blob, lease).unwrap(); + assert!(!store.pinned(&blob, lease).unwrap()); + } +} +#[test] +fn revisions_are_canonical_parented_and_retryable() { + let f = Fixture::new(); + let store = f.store(); + let blob = store.put_bytes(b"revision", "text/plain", None).unwrap(); + let first = revision(&blob, "v_first", &[]); + let oid = store.commit_revision("t_中文", &first).unwrap(); + assert_eq!(store.commit_revision("t_中文", &first).unwrap(), oid); + assert_eq!( + store + .git(&["show", &format!("{oid}:manifest.json")], None) + .unwrap(), + encode(&first).unwrap().canonical().unwrap() + ); + let child = store + .commit_revision("t_中文", &revision(&blob, "v_second", &["v_first"])) + .unwrap(); + assert!(store + .git(&["cat-file", "commit", &child], None) + .unwrap() + .contains(&format!("parent {oid}\n"))); + let mut conflicting = first.clone(); + conflicting.author = "u_other".into(); + assert_eq!( + store.commit_revision("t_中文", &conflicting).err(), + Some(Error("revision_content_conflict")) + ); + assert_eq!( + fs::read_to_string(store.root.join("revisions/v_first")).unwrap(), + format!("{oid}\n") + ); +} +#[test] +fn revision_paths_and_missing_parents_are_checked() { + let f = Fixture::new(); + let store = f.store(); + let blob = store.put_bytes(b"x", "text/plain", None).unwrap(); + assert_eq!( + store + .commit_revision("t", &revision(&blob, "../../escape", &[])) + .err(), + Some(Error("invalid_revision_id")) + ); + assert_eq!( + store + .commit_revision("t", &revision(&blob, "v_second", &["missing"])) + .err(), + Some(Error("revision_content_missing")) + ); + assert!(!store.root.join("revisions/v_second").exists()); +} +#[test] +fn shared_layout_requires_explicit_permission_migration() { + let f = Fixture::new(); + let store = f.store(); + let old = fs::metadata(&store.root).unwrap().mode(); + assert!(GitContentStore::open( + &store.root, + ContentOptions { + group_read: true, + ..ContentOptions::default() + } + ) + .is_err()); + assert_eq!(fs::metadata(&store.root).unwrap().mode(), old); +} +#[test] +fn newly_created_shared_files_have_the_intended_mode() { + let f = Fixture::new(); + let store = GitContentStore::open( + &f.0.join("shared"), + ContentOptions { + group_read: true, + ..ContentOptions::default() + }, + ) + .unwrap(); + let blob = store + .put_bytes(b"group", "application/octet-stream", None) + .unwrap(); + for dir in [&store.root, &store.binary, &store.index] { + assert_eq!(fs::metadata(dir).unwrap().mode() & 0o2777, 0o2750); + } + for dir in [&store.binary, &store.index] { + assert_eq!( + fs::metadata(dir.join(files::key(&blob.digest).unwrap())) + .unwrap() + .mode() + & 0o777, + 0o640 + ); + } + GitContentStore::open( + &store.root, + ContentOptions { + group_read: true, + ..ContentOptions::default() + }, + ) + .unwrap(); +} +#[test] +fn git_timeout_kills_and_reaps_the_child() { + let f = Fixture::new(); + let store = f.store(); + let script = f.0.join("slow-git"); + // Disposable test executable, not a runtime shell adapter. + fs::write(&script, "#!/bin/sh\necho $$ > child.pid\nexec sleep 30\n").unwrap(); + fs::set_permissions(&script, Permissions::from_mode(0o700)).unwrap(); + let store = GitContentStore { + options: ContentOptions { + git_binary: script, + git_timeout: Duration::from_millis(100), + ..ContentOptions::default() + }, + ..store + }; + assert_eq!( + store.put_bytes(b"timeout", "text/plain", None).err(), + Some(Error("content_store_timeout")) + ); + let pid = fs::read_to_string(store.root.join("child.pid")).unwrap(); + assert!(!Path::new(&format!("/proc/{}", pid.trim())).exists()); + assert_eq!(fs::read_dir(&store.staging).unwrap().count(), 0); + assert_eq!(fs::read_dir(&store.index).unwrap().count(), 0); +} diff --git a/rust/crates/msg-storage/src/lib.rs b/rust/crates/msg-storage/src/lib.rs new file mode 100644 index 00000000..97732060 --- /dev/null +++ b/rust/crates/msg-storage/src/lib.rs @@ -0,0 +1,317 @@ +//! Native SQLite sessions over existing Python-created databases. +//! +//! No schema creation/migration, network entry point or Python subprocess. +//! Read snapshots and write sessions share the same transaction-bound authority reads. +mod authorization; +#[cfg(unix)] +pub mod content; +pub mod postgres; +pub mod records; +#[cfg(unix)] +pub mod writer; +use msg_core::{Error, Json, Result, MAX_BYTES}; +use msg_identity::{ + models::Timestamp, + sessions::CredentialState, + store::{AuthorityStore, Record, RecoveryDelivery}, +}; +use rusqlite::{Connection, ErrorCode, OpenFlags, OptionalExtension, Transaction}; +use std::{cell::Cell, marker::PhantomData, path::Path, rc::Rc, time::Duration}; +#[cfg(unix)] +pub use writer::{SqliteWriter, WriteFailure, WriteSession}; + +pub struct SqliteAuthority { + connection: Connection, +} +pub struct ReadOnly; +pub type Snapshot<'a> = Session<'a, ReadOnly>; + +/// A borrowed database session. Its transaction is private and cannot escape. +pub struct Session<'a, Access> { + transaction: Transaction<'a>, + access: Access, + poison: Cell>, + // A database snapshot must stay with the task that owns it. + _owner: PhantomData>, +} +fn database_error(error: rusqlite::Error) -> Error { + match error.sqlite_error_code() { + Some(ErrorCode::DatabaseBusy | ErrorCode::DatabaseLocked) => Error("server_busy"), + Some(ErrorCode::ReadOnly) => Error("read_only_transaction"), + Some(ErrorCode::ConstraintViolation) => Error("constraint_conflict"), + _ => Error("storage_error"), + } +} +impl SqliteAuthority { + pub fn open_existing(path: &Path) -> Result { + let connection = Connection::open_with_flags( + path, + OpenFlags::SQLITE_OPEN_READ_ONLY | OpenFlags::SQLITE_OPEN_NO_MUTEX, + ) + .map_err(database_error)?; + connection + .busy_timeout(Duration::from_secs(10)) + .map_err(database_error)?; + connection + .execute_batch( + "PRAGMA query_only=ON; PRAGMA trusted_schema=OFF; PRAGMA foreign_keys=ON;", + ) + .map_err(database_error)?; + Ok(Self { connection }) + } + pub fn snapshot(&mut self) -> Result> { + let transaction = self.connection.transaction().map_err(database_error)?; + let snapshot = Snapshot { + transaction, + access: ReadOnly, + poison: Cell::new(None), + _owner: PhantomData, + }; + // The first read pins this snapshot before checking any authority. + snapshot.validate_schema()?; + Ok(snapshot) + } +} +impl Session<'_, Access> { + fn active(&self) -> Result<()> { + if let Some(error) = self.poison.get() { + return Err(error); + } + if self.transaction.is_autocommit() { + return Err(Error("transaction_aborted")); + } + Ok(()) + } + fn validate_schema(&self) -> Result<()> { + self.active()?; + let expected = [ + ("schema_version", "version"), ("settings", "key,value"), + ("identities", "id,kind,body"), ("credentials", "id,subject,body"), + ("certificates", "id,subject,parent,revoked,body"), + ("resources", "id,parent,body"), ("results", "subject,request_id,digest,body"), + ("credential_states", "id,expires,body"), + ("custodial_vault", "subject,signing_key_id,status"), + ("token_deliveries", "credential_id,subject,recovery_verifier,recovery_expires_at,consumed_at,request_id"), + ]; + for (table, columns) in expected { + let exists: bool = self + .transaction + .query_row( + "SELECT EXISTS(SELECT 1 FROM sqlite_schema WHERE name=?1 AND type='table')", + [table], + |r| r.get(0), + ) + .map_err(|_| Error("unsupported_storage_schema"))?; + if !exists { + return Err(Error("unsupported_storage_schema")); + } + // Names are compile-time constants, never request strings. + self.transaction + .prepare(&format!("SELECT {columns} FROM {table} LIMIT 0")) + .map_err(|_| Error("unsupported_storage_schema"))?; + } + let mut stmt = self + .transaction + .prepare("SELECT version FROM schema_version ORDER BY version LIMIT 2") + .map_err(database_error)?; + let versions = stmt + .query_map([], |r| r.get::<_, i64>(0)) + .map_err(database_error)? + .collect::, _>>() + .map_err(database_error)?; + if versions != [1] { + return Err(Error("unsupported_storage_schema")); + } + Ok(()) + } + fn json(&self, sql: &str, id: &str) -> Result> { + self.active()?; + let mut stmt = self + .transaction + .prepare_cached(sql) + .map_err(database_error)?; + let raw: Option = stmt + .query_row([id], |r| r.get(0)) + .optional() + .map_err(database_error)?; + raw.map(|value| { + if value.len() > MAX_BYTES { + return Err(Error("storage_record_too_large")); + } + Json::parse(&value) + }) + .transpose() + } +} +impl AuthorityStore for Session<'_, Access> { + fn record(&self, kind: Record, id: &str) -> Result { + // A bounded SQL substring avoids allocating a corrupt multi-GB body. + let (table, condition, missing) = match kind { + Record::Subject => ("identities", " AND kind='subject'", "subject_not_found"), + Record::Credential => ("credentials", "", "credential_not_found"), + Record::Certificate => ("certificates", "", "certificate_not_found"), + Record::Resource => ("resources", "", "not_found"), + }; + self.json(&format!("SELECT CAST(substr(CAST(body AS BLOB),1,1048577) AS TEXT) FROM {table} WHERE id=?1{condition}"), id)? + .ok_or(Error(missing)) + } + fn setting(&self, key: &str) -> Result> { + self.json( + "SELECT CAST(substr(CAST(value AS BLOB),1,1048577) AS TEXT) FROM settings WHERE key=?1", + key, + ) + } + fn has_setting(&self, key: &str) -> Result { + self.active()?; + self.transaction + .query_row( + "SELECT EXISTS(SELECT 1 FROM settings WHERE key=?1)", + [key], + |r| r.get(0), + ) + .map_err(database_error) + } + fn certificate_revoked(&self, id: &str) -> Result { + self.active()?; + let revoked: Option = self + .transaction + .query_row("SELECT revoked FROM certificates WHERE id=?1", [id], |r| { + r.get(0) + }) + .optional() + .map_err(database_error)?; + Ok(revoked.ok_or(Error("certificate_not_found"))? != 0) + } + fn request_result( + &self, + subject: Option<&str>, + id: &str, + digest: &str, + ) -> Result> { + self.active()?; + let row: Option<(String, String)> = self.transaction.query_row( + "SELECT digest,CAST(substr(CAST(body AS BLOB),1,1048577) AS TEXT) FROM results WHERE subject=?1 AND request_id=?2", + [subject.unwrap_or(""), id], |r| Ok((r.get(0)?, r.get(1)?))).optional().map_err(database_error)?; + let Some((stored_digest, raw)) = row else { + return Ok(None); + }; + if stored_digest != digest { + return Err(Error("idempotency_conflict")); + } + if raw.len() > MAX_BYTES { + return Err(Error("storage_record_too_large")); + } + let result = Json::parse(&raw)?; + let fields = result.as_object()?; + let result_subject = fields.get("subject").and_then(|v| { + if v.is_null() { + Some(None) + } else { + v.as_str().ok().map(Some) + } + }); + if fields.get("request_id").and_then(|v| v.as_str().ok()) != Some(id) + || result_subject != Some(subject) + { + return Err(Error("storage_record_mismatch")); + } + Ok(Some(result)) + } + fn credential_state(&self, id: &str) -> Result> { + self.active()?; + let row: Option<(String, String)> = self.transaction.query_row( + "SELECT expires,CAST(substr(CAST(body AS BLOB),1,1048577) AS TEXT) FROM credential_states WHERE id=?1", + [id], |r| Ok((r.get(0)?, r.get(1)?))).optional().map_err(database_error)?; + row.map(|(expires, body)| { + if body.len() > MAX_BYTES { + return Err(Error("storage_record_too_large")); + } + Ok(CredentialState { + expires_at: Timestamp::parse(&expires)?, + body: Json::parse(&body)?, + }) + }) + .transpose() + } + fn custodial_binding(&self, subject: &str) -> Result> { + self.active()?; + self.transaction + .query_row( + "SELECT signing_key_id,status FROM custodial_vault WHERE subject=?1", + [subject], + |r| Ok((r.get(0)?, r.get(1)?)), + ) + .optional() + .map_err(database_error) + } + fn recovery_delivery( + &self, + credential: &str, + subject: &str, + ) -> Result> { + self.active()?; + let row: Option<(String, String, bool, String)> = self.transaction.query_row( + "SELECT recovery_verifier,recovery_expires_at,consumed_at IS NOT NULL,request_id FROM token_deliveries WHERE credential_id=?1 AND subject=?2", + [credential, subject], |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?, r.get(3)?))).optional().map_err(database_error)?; + row.map(|(verifier, expires, consumed, request_id)| { + Ok(RecoveryDelivery { + verifier, + expires_at: Timestamp::parse(&expires)?, + consumed, + request_id, + }) + }) + .transpose() + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::{ + fs, + time::{SystemTime, UNIX_EPOCH}, + }; + fn path() -> std::path::PathBuf { + std::env::temp_dir().join(format!( + "msg-readonly-{}-{}.db", + std::process::id(), + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos() + )) + } + #[test] + fn missing_database_is_not_created() { + let path = path(); + assert!(SqliteAuthority::open_existing(&path).is_err()); + assert!(!path.exists()); + } + #[test] + fn never_creates_or_repairs_a_schema() { + let path = path(); + Connection::open(&path).unwrap().execute_batch("CREATE TABLE schema_version(version INTEGER); INSERT INTO schema_version VALUES(99);").unwrap(); + let before = fs::read(&path).unwrap(); + let mut store = SqliteAuthority::open_existing(&path).unwrap(); + assert!(matches!( + store.snapshot(), + Err(Error("unsupported_storage_schema")) + )); + assert_eq!(fs::read(&path).unwrap(), before); + drop(store); + fs::remove_file(path).unwrap(); + } + #[test] + fn connection_enforces_read_only_at_the_database_layer() { + let path = path(); + drop(Connection::open(&path).unwrap()); + let store = SqliteAuthority::open_existing(&path).unwrap(); + assert!(store + .connection + .execute_batch("CREATE TABLE injected(secret TEXT);") + .is_err()); + drop(store); + fs::remove_file(path).unwrap(); + } +} diff --git a/rust/crates/msg-storage/src/postgres/authority.rs b/rust/crates/msg-storage/src/postgres/authority.rs new file mode 100644 index 00000000..abe2a57b --- /dev/null +++ b/rust/crates/msg-storage/src/postgres/authority.rs @@ -0,0 +1,200 @@ +//! Policy facts from the transaction's immutable PostgreSQL snapshot. +use super::PgSession; +use crate::records::{Membership, Organization}; +use msg_core::{Error, Json, Result}; +use msg_identity::{ + authorization::{AuthorizationStore, DirectConversation, LegacyShare, ShareSource}, + models::{decode, SubjectKind, Timestamp}, + sessions::CredentialState, + store::{AuthorityStore, Record, RecoveryDelivery}, +}; +use std::collections::BTreeSet; + +impl PgSession { + fn json_fact(&self, sql: &'static str, id: &str) -> Result> { + self.one(sql, &[id.into()])?.map(|r| r.json(0)).transpose() + } +} +impl AuthorityStore for PgSession { + fn record(&self, kind: Record, id: &str) -> Result { + let (sql, missing) = match kind { + Record::Subject => ( + "SELECT substring(body,1,1048577) FROM identities WHERE id=$1 AND kind='subject'", + "subject_not_found", + ), + Record::Credential => ( + "SELECT substring(body,1,1048577) FROM credentials WHERE id=$1", + "credential_not_found", + ), + Record::Certificate => ( + "SELECT substring(body,1,1048577) FROM certificates WHERE id=$1", + "certificate_not_found", + ), + Record::Resource => ( + "SELECT substring(body,1,1048577) FROM resources WHERE id=$1", + "not_found", + ), + }; + self.json_fact(sql, id)?.ok_or(Error(missing)) + } + fn setting(&self, key: &str) -> Result> { + self.json_fact( + "SELECT substring(value,1,1048577) FROM settings WHERE key=$1", + key, + ) + } + fn has_setting(&self, key: &str) -> Result { + self.one( + "SELECT EXISTS(SELECT 1 FROM settings WHERE key=$1)", + &[key.into()], + )? + .ok_or(Error("storage_error"))? + .boolean(0) + } + fn certificate_revoked(&self, id: &str) -> Result { + Ok(self + .one("SELECT revoked FROM certificates WHERE id=$1", &[id.into()])? + .ok_or(Error("certificate_not_found"))? + .integer(0)? + != 0) + } + fn request_result( + &self, + subject: Option<&str>, + id: &str, + digest: &str, + ) -> Result> { + let Some(row)=self.one("SELECT digest,substring(body,1,1048577) FROM results WHERE subject=$1 AND request_id=$2",&[subject.unwrap_or("").into(),id.into()])? else {return Ok(None)}; + if row.text(0)? != digest { + return Err(Error("idempotency_conflict")); + } + let result = row.json(1)?; + let fields = result.as_object()?; + let result_subject = fields.get("subject").and_then(|v| { + if v.is_null() { + Some(None) + } else { + v.as_str().ok().map(Some) + } + }); + if fields.get("request_id").and_then(|v| v.as_str().ok()) != Some(id) + || result_subject != Some(subject) + { + return Err(Error("storage_record_mismatch")); + } + Ok(Some(result)) + } + fn credential_state(&self, id: &str) -> Result> { + self.one( + "SELECT expires,substring(body,1,1048577) FROM credential_states WHERE id=$1", + &[id.into()], + )? + .map(|r| { + Ok(CredentialState { + expires_at: Timestamp::parse(r.text(0)?)?, + body: r.json(1)?, + }) + }) + .transpose() + } + fn custodial_binding(&self, subject: &str) -> Result> { + self.one( + "SELECT signing_key_id,status FROM custodial_vault WHERE subject=$1", + &[subject.into()], + )? + .map(|r| Ok((r.text(0)?.to_owned(), r.text(1)?.to_owned()))) + .transpose() + } + fn recovery_delivery( + &self, + credential: &str, + subject: &str, + ) -> Result> { + self.one("SELECT recovery_verifier,recovery_expires_at,consumed_at,request_id FROM token_deliveries WHERE credential_id=$1 AND subject=$2",&[credential.into(),subject.into()])? + .map(|r|Ok(RecoveryDelivery{verifier:r.text(0)?.to_owned(),expires_at:Timestamp::parse(r.text(1)?)?,consumed:r.optional_text(2)?.is_some(),request_id:r.text(3)?.to_owned()})).transpose() + } +} +impl AuthorizationStore for PgSession { + fn authority(&self) -> &dyn AuthorityStore { + self + } + fn memberships(&self, subject: &str) -> Result> { + let rows=self.rows("SELECT m.org,substring(m.body,1,1048577) FROM memberships m JOIN resources r ON r.id=m.org WHERE m.subject=$1 AND r.type='organization' AND r.state='active' ORDER BY m.org LIMIT 4097",&[subject.into()])?; + let mut result = BTreeSet::new(); + for r in rows { + let m: Membership = decode(&r.json(1)?)?; + m.validate()?; + if m.status == "active" + && m.organization_id == r.text(0)? + && m.subject_id == subject + && m.organization_id != "g_public" + { + result.insert(m.organization_id); + } + } + match self.subject(subject) { + Ok(s) + if matches!( + s.kind, + SubjectKind::Registered | SubjectKind::Custodial | SubjectKind::System + ) && !s.local_only => + { + result.insert("g_public".into()); + } + Ok(_) | Err(Error("subject_not_found")) => (), + Err(e) => return Err(e), + } + Ok(result) + } + fn organization_exists(&self, id: &str) -> Result { + let Some(value) = self.json_fact( + "SELECT substring(body,1,1048577) FROM identities WHERE id=$1 AND kind='organization'", + id, + )? + else { + return Ok(false); + }; + let row: Organization = decode(&value)?; + row.validate()?; + if row.resource_id != id { + return Err(Error("storage_record_mismatch")); + } + Ok(true) + } + fn topic_admin(&self, topic: &str, subject: &str) -> Result { + self.one("SELECT EXISTS(SELECT 1 FROM topic_memberships WHERE topic=$1 AND subject=$2 AND role='admin' AND status='active')",&[topic.into(),subject.into()])?.ok_or(Error("storage_error"))?.boolean(0) + } + fn topic_ban(&self, topic: &str, subject: &str) -> Result>> { + self.one( + "SELECT expires_at FROM topic_bans WHERE topic=$1 AND subject=$2 AND status='active'", + &[topic.into(), subject.into()], + )? + .map(|r| Ok(r.optional_text(0)?.map(str::to_owned))) + .transpose() + } + fn direct_conversation(&self, resource: &str) -> Result> { + self.one("SELECT resource_id,participant_a,participant_b,state FROM dm_conversations WHERE resource_id=$1",&[resource.into()])?.map(|r|Ok(DirectConversation{ + resource_id:r.text(0)?.to_owned(),participant_a:r.text(1)?.to_owned(),participant_b:r.text(2)?.to_owned(),state:r.text(3)?.to_owned()})).transpose() + } + fn direct_blocked(&self, a: &str, b: &str) -> Result { + self.one("SELECT EXISTS(SELECT 1 FROM dm_blocks WHERE (blocker=$1 AND blocked=$2) OR (blocker=$2 AND blocked=$1))",&[a.into(),b.into()])?.ok_or(Error("storage_error"))?.boolean(0) + } + fn legacy_share(&self, resource: &str, grantee: &str) -> Result> { + self.one("SELECT grantor,expires_at,created_at FROM share_grants WHERE resource_id=$1 AND grantee=$2 AND revoked_at IS NULL",&[resource.into(),grantee.into()])?.map(|r|Ok(LegacyShare{ + grantor:r.text(0)?.to_owned(),expires_at:r.text(1)?.to_owned(),created_at:r.text(2)?.to_owned()})).transpose() + } + fn share_candidates(&self, resource: &str, now: &str) -> Result> { + self.rows("SELECT id FROM share_grants_v2 WHERE resource_id=$1 AND revoked_at IS NULL AND expires_at>$2 ORDER BY id LIMIT 4097",&[resource.into(),now.into()])?.iter().map(|r|Ok(r.text(0)?.to_owned())).collect() + } + fn share_source(&self, id: &str) -> Result> { + self.one("SELECT resource_id,grantor,grantee,grantee_kind,parent_id,substring(operations,1,1048577),substring(constraints,1,1048577),allow_reshare,expires_at,revoked_at,created_at FROM share_grants_v2 WHERE id=$1",&[id.into()])?.map(|r|Ok(ShareSource{ + resource_id:r.text(0)?.to_owned(),grantor:r.text(1)?.to_owned(),grantee:r.text(2)?.to_owned(),grantee_kind:r.text(3)?.to_owned(),parent_id:r.optional_text(4)?.map(str::to_owned), + operations:r.text(5)?.to_owned(),constraints:r.text(6)?.to_owned(),allow_reshare:r.integer(7)?==1,expires_at:r.text(8)?.to_owned(),revoked_at:r.optional_text(9)?.map(str::to_owned),created_at:r.text(10)?.to_owned()})).transpose() + } + fn csr_issuer(&self, id: &str) -> Result { + let value = self + .json_fact("SELECT substring(body,1,1048577) FROM csrs WHERE id=$1", id)? + .ok_or(Error("csr_not_found"))?; + Ok(decode::(&value)?.requested_issuer) + } +} diff --git a/rust/crates/msg-storage/src/postgres/mod.rs b/rust/crates/msg-storage/src/postgres/mod.rs new file mode 100644 index 00000000..de878511 --- /dev/null +++ b/rust/crates/msg-storage/src/postgres/mod.rs @@ -0,0 +1,514 @@ +//! Native PostgreSQL transactions over the existing, Python-migrated schema. +//! +//! One dedicated connection owns each transaction. Writers use the same +//! session advisory lock as Python and retain it through all compensations. +//! Transaction pooling is unsupported. No schema is created or upgraded here. +mod authority; +mod query; +mod records; +use msg_core::{Error, Result}; +use postgres::{fallible_iterator::FallibleIterator, types::ToSql, Client, Config}; +use postgres_native_tls::MakeTlsConnector; +pub use query::{Row, Value}; +use std::{ + cell::{Cell, RefCell}, + fmt, + marker::PhantomData, + panic::{catch_unwind, resume_unwind, AssertUnwindSafe}, + rc::Rc, + str::FromStr, + time::Duration, +}; + +const MAX_ROWS: usize = 4096; +const MAX_QUERY_BYTES: usize = 8 * 1024 * 1024; +type Cleanup = Box Result<()>>; + +#[derive(Clone)] +pub struct PgStore { + config: Config, + tls: native_tls::TlsConnector, +} +impl fmt::Debug for PgStore { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("PgStore()") + } +} +#[derive(Debug)] +pub enum PgFailure { + Rejected { + error: Error, + cleanup_failed: bool, + }, + /// Do not retry under a new request ID, delete blobs, or issue secrets. + Uncertain { + error: Error, + }, +} +impl PgFailure { + pub fn code(&self) -> &'static str { + match self { + Self::Rejected { error, .. } | Self::Uncertain { error } => error.0, + } + } +} +impl From for PgFailure { + fn from(error: Error) -> Self { + Self::Rejected { + error, + cleanup_failed: false, + } + } +} +pub struct PgSession { + client: RefCell, + write: bool, + closed: Cell, + poisoned: Cell>, + rollback: RefCell>, + savepoints: Cell, + _owner: PhantomData>, +} +fn database_error(error: &postgres::Error) -> Error { + let code = error.code().map(|c| c.code()).unwrap_or(""); + if code.starts_with("23") { + Error("constraint_conflict") + } else if matches!( + code, + "40P01" | "40001" | "55P03" | "57014" | "53300" | "57P03" + ) { + Error("server_busy") + } else if code == "25006" { + Error("read_only_transaction") + } else { + Error("storage_error") + } +} +impl PgStore { + pub fn open_existing(dsn: &str) -> Result { + let mut config = Config::from_str(dsn).map_err(|_| Error("invalid_postgres_config"))?; + config.connect_timeout(Duration::from_secs(10)); + config.application_name("msgd-native"); + let local = config.get_hosts().iter().all(|host| match host { + postgres::config::Host::Tcp(host) => { + host == "localhost" + || host + .parse::() + .is_ok_and(|ip| ip.is_loopback()) + } + #[cfg(unix)] + postgres::config::Host::Unix(_) => true, + }) && config + .get_hostaddrs() + .iter() + .all(std::net::IpAddr::is_loopback); + if !local { + if config.get_ssl_mode() == postgres::config::SslMode::Disable { + return Err(Error("postgres_tls_required")); + } + // A remote connection must not fall back to plaintext when the peer + // rejects TLS. Certificate/name validation is still enforced below. + config.ssl_mode(postgres::config::SslMode::Require); + } + // native-tls validates server certificates and names whenever TLS is used. + // There is no dangerous_accept_invalid_certs/hostnames setting. + let tls = native_tls::TlsConnector::builder() + .build() + .map_err(|_| Error("postgres_tls_error"))?; + let store = Self { config, tls }; + store + .transaction(false, |tx| tx.validate_schema()) + .map_err(|e| Error(e.code()))?; + Ok(store) + } + fn connect(&self) -> Result { + self.config + .connect(MakeTlsConnector::new(self.tls.clone())) + .map_err(|e| database_error(&e)) + } + pub fn transaction( + &self, + write: bool, + body: impl FnOnce(&PgSession) -> Result, + ) -> std::result::Result { + let mut client = self.connect()?; + client.batch_execute("SET search_path = pg_catalog, public; SET statement_timeout='30s'; SET lock_timeout='10s'; SET idle_in_transaction_session_timeout='60s';") + .map_err(|e|PgFailure::from(database_error(&e)))?; + if write { + client + .query_one("SELECT pg_advisory_lock(725274758,1886265951)", &[]) + .map_err(|e| PgFailure::from(database_error(&e)))?; + } + // Acquire the writer fence before taking the snapshot. Read-only requests + // also see one committed generation, not a mixture of policy revisions. + client + .batch_execute(if write { + "BEGIN ISOLATION LEVEL REPEATABLE READ" + } else { + "BEGIN ISOLATION LEVEL REPEATABLE READ READ ONLY" + }) + .map_err(|e| PgFailure::from(database_error(&e)))?; + let session = PgSession { + client: RefCell::new(client), + write, + closed: Cell::new(false), + poisoned: Cell::new(None), + rollback: RefCell::new(Vec::new()), + savepoints: Cell::new(0), + _owner: PhantomData, + }; + let outcome = catch_unwind(AssertUnwindSafe(|| body(&session))); + match outcome { + Ok(Ok(value)) if session.poisoned.get().is_none() => { + let commit = session.client.borrow_mut().batch_execute("COMMIT"); + match commit { + Ok(()) => { + session.closed.set(true); + session.rollback.borrow_mut().clear(); + Ok(value) + } + Err(error) => { + let definitely_aborted = error.code().is_some_and(|c| { + c.code().starts_with("23") + || matches!(c.code(), "40001" | "40P01" | "25P02") + }); + if definitely_aborted + && session + .client + .borrow_mut() + .batch_execute("ROLLBACK") + .is_ok() + { + let failed = session.compensate(0); + session.closed.set(true); + Err(PgFailure::Rejected { + error: database_error(&error), + cleanup_failed: failed, + }) + } else { + session.closed.set(true); + session.rollback.borrow_mut().clear(); + Err(PgFailure::Uncertain { + error: Error("storage_commit_uncertain"), + }) + } + } + } + } + result => { + let confirmed = session + .client + .borrow_mut() + .batch_execute("ROLLBACK") + .is_ok(); + let cleanup_failed = if confirmed { + session.compensate(0) + } else { + session.rollback.borrow_mut().clear(); + false + }; + session.closed.set(true); + match result { + Err(panic) => resume_unwind(panic), + result => { + let error = match result { + Ok(Err(error)) => error, + _ => session + .poisoned + .get() + .unwrap_or(Error("transaction_aborted")), + }; + if confirmed { + Err(PgFailure::Rejected { + error, + cleanup_failed, + }) + } else { + Err(PgFailure::Uncertain { + error: Error("storage_rollback_uncertain"), + }) + } + } + } + } + } + // Closing this dedicated connection releases the advisory lock only after + // compensation completes. A transaction-pool proxy breaks that guarantee. + } +} +impl PgSession { + fn active(&self, write: bool) -> Result<()> { + if self.closed.get() { + return Err(Error("transaction_closed")); + } + if let Some(e) = self.poisoned.get() { + return Err(e); + } + if write && !self.write { + return Err(Error("read_only_transaction")); + } + Ok(()) + } + fn capture(&self, result: Result, write: bool) -> Result { + if let Err(e) = result { + if write { + self.poisoned.set(Some(e)); + } + return Err(e); + } + result + } + pub fn require_write(&self) -> Result<()> { + self.active(true) + } + /// SQL is a compiled, trusted string; only values may originate in a request. + pub fn rows(&self, sql: &'static str, params: &[Value]) -> Result> { + self.active(false)?; + let refs: Vec<&(dyn ToSql + Sync)> = + params.iter().map(|p| p as &(dyn ToSql + Sync)).collect(); + let result = (|| { + let mut client = self.client.borrow_mut(); + let mut iter = client + .query_raw(sql, refs) + .map_err(|e| database_error(&e))?; + let mut rows = Vec::new(); + let mut bytes = 0; + while let Some(row) = iter.next().map_err(|e| database_error(&e))? { + let row = Row::decode(row)?; + bytes += row.bytes(); + if rows.len() >= MAX_ROWS || bytes > MAX_QUERY_BYTES { + return Err(Error("storage_query_limit")); + } + rows.push(row); + } + Ok(rows) + })(); + self.capture(result, self.write) + } + pub fn one(&self, sql: &'static str, params: &[Value]) -> Result> { + let mut rows = self.rows(sql, params)?; + if rows.len() > 1 { + return self.capture(Err(Error("storage_record_ambiguous")), self.write); + } + Ok(rows.pop()) + } + pub fn execute(&self, sql: &'static str, params: &[Value]) -> Result { + self.active(true)?; + let refs: Vec<&(dyn ToSql + Sync)> = + params.iter().map(|p| p as &(dyn ToSql + Sync)).collect(); + let result = self + .client + .borrow_mut() + .execute(sql, &refs) + .map_err(|e| database_error(&e)); + self.capture(result, true) + } + pub fn on_rollback(&self, effect: impl FnOnce() -> Result<()> + 'static) -> Result<()> { + self.active(true)?; + self.rollback.borrow_mut().push(Box::new(effect)); + Ok(()) + } + fn compensate(&self, start: usize) -> bool { + let effects: Vec<_> = self.rollback.borrow_mut().drain(start..).collect(); + let mut failed = false; + for effect in effects.into_iter().rev() { + failed |= !matches!(catch_unwind(AssertUnwindSafe(effect)), Ok(Ok(()))); + } + failed + } + pub fn savepoint(&self, body: impl FnOnce(&Self) -> Result) -> Result { + self.active(false)?; + let id = self + .savepoints + .get() + .checked_add(1) + .ok_or(Error("storage_savepoint_limit"))?; + self.savepoints.set(id); + // Names contain only an internal integer. They never contain user strings. + self.client + .borrow_mut() + .batch_execute(&format!("SAVEPOINT msg_native_{id}")) + .map_err(|e| database_error(&e))?; + let start = self.rollback.borrow().len(); + let result = catch_unwind(AssertUnwindSafe(|| body(self))); + if matches!(&result, Ok(Ok(_))) && self.poisoned.get().is_none() { + let release = self + .client + .borrow_mut() + .batch_execute(&format!("RELEASE SAVEPOINT msg_native_{id}")) + .map_err(|e| database_error(&e)); + self.capture(release, self.write)?; + return match result { + Ok(value) => value, + Err(p) => resume_unwind(p), + }; + } + let rollback = self.client.borrow_mut().batch_execute(&format!( + "ROLLBACK TO SAVEPOINT msg_native_{id}; RELEASE SAVEPOINT msg_native_{id}" + )); + if let Err(error) = rollback { + let error = database_error(&error); + self.poisoned.set(Some(error)); + return Err(error); + } + let poison = self.poisoned.take(); + let cleanup_failed = self.compensate(start); + if cleanup_failed { + self.poisoned.set(Some(Error("storage_cleanup_failed"))); + } + match result { + Err(p) => resume_unwind(p), + Ok(Err(e)) => Err(e), + Ok(Ok(_)) => Err(poison.unwrap_or(Error("transaction_aborted"))), + } + } + fn validate_schema(&self) -> Result<()> { + let versions = self + .rows( + "SELECT version FROM public.schema_version ORDER BY version LIMIT 2", + &[], + ) + .map_err(|_| Error("unsupported_storage_schema"))?; + if versions.len() != 1 || versions[0].integer(0)? != 1 { + return Err(Error("unsupported_storage_schema")); + } + let required = [ + ( + "resources", + vec![ + "id", + "type", + "name", + "parent", + "owner", + "grp", + "mode", + "generation", + "revision", + "state", + "created_at", + "modified_at", + "body", + ], + ), + ("identities", vec!["id", "kind", "generation", "body"]), + ("credentials", vec!["id", "subject", "body"]), + ( + "certificates", + vec!["id", "subject", "parent", "revoked", "body"], + ), + ("settings", vec!["key", "value"]), + ("results", vec!["subject", "request_id", "digest", "body"]), + ("credential_states", vec!["id", "kind", "expires", "body"]), + ("revisions", vec!["id", "resource_id", "created_at", "body"]), + ( + "relations", + vec![ + "revision_id", + "source_id", + "type", + "target_id", + "target_revision", + "body", + ], + ), + ( + "custodial_vault", + vec!["subject", "signing_key_id", "status"], + ), + ( + "token_deliveries", + vec![ + "credential_id", + "subject", + "request_id", + "recovery_verifier", + "recovery_expires_at", + "consumed_at", + ], + ), + ("memberships", vec!["org", "subject", "generation", "body"]), + ("emails", vec!["subject", "generation", "body"]), + ( + "csrs", + vec!["id", "generation", "state", "body", "state_body"], + ), + ( + "transfers", + vec!["id", "subject", "generation", "body", "limits"], + ), + ("chunks", vec!["transfer_id", "offset", "length", "body"]), + ( + "jobs", + vec!["id", "dedupe", "kind", "state", "next_at", "body"], + ), + ("resource_tags", vec!["resource_id", "tag"]), + ( + "resource_path_aliases", + vec!["parent_id", "name", "resource_id"], + ), + ("events", vec!["seq", "id", "body"]), + ("audit", vec!["seq", "digest", "previous", "body"]), + ]; + for (table, columns) in required { + let row=self.one("SELECT count(*) FROM pg_catalog.pg_attribute a JOIN pg_catalog.pg_class c ON c.oid=a.attrelid JOIN pg_catalog.pg_namespace n ON n.oid=c.relnamespace WHERE n.nspname='public' AND c.relname=$1 AND c.relkind='r' AND a.attnum>0 AND NOT a.attisdropped AND a.attname::text=ANY($2)",&[table.into(),Value::TextArray(columns.iter().map(|s|s.to_string()).collect())])?.ok_or(Error("unsupported_storage_schema"))?; + if row.integer(0)? != columns.len() as i64 { + return Err(Error("unsupported_storage_schema")); + } + } + for (table, columns) in [ + ("resources", vec!["id"]), + ("revisions", vec!["id"]), + ("identities", vec!["id"]), + ("credentials", vec!["id"]), + ("certificates", vec!["id"]), + ("csrs", vec!["id"]), + ("results", vec!["subject", "request_id"]), + ("settings", vec!["key"]), + ("memberships", vec!["org", "subject"]), + ("emails", vec!["subject"]), + ("transfers", vec!["id"]), + ("chunks", vec!["transfer_id", "offset"]), + ("jobs", vec!["id"]), + ("events", vec!["seq"]), + ("audit", vec!["seq"]), + ("resource_path_aliases", vec!["parent_id", "name"]), + ("resource_tags", vec!["tag", "resource_id"]), + ] { + self.require_constraint(table, "p", &columns)?; + } + for (table, columns) in [ + ("resources", vec!["parent", "name"]), + ("events", vec!["id"]), + ("audit", vec!["digest"]), + ("jobs", vec!["dedupe"]), + ] { + self.require_constraint(table, "u", &columns)?; + } + for (table, source, target, column) in [ + ("resources", "parent", "resources", "id"), + ("revisions", "resource_id", "resources", "id"), + ("relations", "revision_id", "revisions", "id"), + ("resource_tags", "resource_id", "resources", "id"), + ("resource_path_aliases", "resource_id", "resources", "id"), + ("resource_path_aliases", "parent_id", "resources", "id"), + ] { + let row=self.one("SELECT EXISTS(SELECT 1 FROM pg_catalog.pg_constraint c JOIN pg_catalog.pg_class t ON t.oid=c.conrelid JOIN pg_catalog.pg_namespace n ON n.oid=t.relnamespace JOIN pg_catalog.pg_class r ON r.oid=c.confrelid JOIN pg_catalog.pg_namespace rn ON rn.oid=r.relnamespace WHERE n.nspname='public' AND rn.nspname='public' AND t.relname=$1 AND r.relname=$3 AND c.contype='f' AND c.convalidated AND ARRAY(SELECT a.attname::text FROM unnest(c.conkey) WITH ORDINALITY k(attnum,ord) JOIN pg_catalog.pg_attribute a ON a.attrelid=t.oid AND a.attnum=k.attnum ORDER BY k.ord)=ARRAY[$2]::text[] AND ARRAY(SELECT a.attname::text FROM unnest(c.confkey) WITH ORDINALITY k(attnum,ord) JOIN pg_catalog.pg_attribute a ON a.attrelid=r.oid AND a.attnum=k.attnum ORDER BY k.ord)=ARRAY[$4]::text[])",&[table.into(),source.into(),target.into(),column.into()])?.ok_or(Error("unsupported_storage_schema"))?; + if !row.boolean(0)? { + return Err(Error("unsupported_storage_schema")); + } + } + let root=self.one("SELECT EXISTS(SELECT 1 FROM pg_catalog.pg_index i JOIN pg_catalog.pg_class t ON t.oid=i.indrelid JOIN pg_catalog.pg_namespace n ON n.oid=t.relnamespace WHERE n.nspname='public' AND t.relname='resources' AND i.indisunique AND i.indisvalid AND i.indisready AND i.indnkeyatts=1 AND pg_catalog.pg_get_expr(i.indexprs,i.indrelid)='1' AND pg_catalog.pg_get_expr(i.indpred,i.indrelid)='(parent IS NULL)')",&[])?.ok_or(Error("unsupported_storage_schema"))?; + if !root.boolean(0)? { + return Err(Error("unsupported_storage_schema")); + } + Ok(()) + } + fn require_constraint(&self, table: &str, kind: &str, columns: &[&str]) -> Result<()> { + let row=self.one("SELECT EXISTS(SELECT 1 FROM pg_catalog.pg_constraint c JOIN pg_catalog.pg_class t ON t.oid=c.conrelid JOIN pg_catalog.pg_namespace n ON n.oid=t.relnamespace WHERE n.nspname='public' AND t.relname=$1 AND c.contype::text=$2 AND c.convalidated AND ARRAY(SELECT a.attname::text FROM unnest(c.conkey) WITH ORDINALITY k(attnum,ord) JOIN pg_catalog.pg_attribute a ON a.attrelid=t.oid AND a.attnum=k.attnum ORDER BY k.ord)=$3)",&[table.into(),kind.into(),Value::TextArray(columns.iter().map(|s|s.to_string()).collect())])?.ok_or(Error("unsupported_storage_schema"))?; + if !row.boolean(0)? { + return Err(Error("unsupported_storage_schema")); + } + Ok(()) + } +} diff --git a/rust/crates/msg-storage/src/postgres/query.rs b/rust/crates/msg-storage/src/postgres/query.rs new file mode 100644 index 00000000..f4f6bdd8 --- /dev/null +++ b/rust/crates/msg-storage/src/postgres/query.rs @@ -0,0 +1,188 @@ +//! Bound values for trusted, source-owned SQL. Neither rows nor parameters log secrets. +use bytes::BytesMut; +use msg_core::{Error, Json, Result, MAX_BYTES}; +use postgres::types::{to_sql_checked, IsNull, ToSql, Type}; +use std::{error::Error as StdError, fmt}; + +#[derive(Clone, PartialEq)] +pub enum Value { + Null, + Bool(bool), + Int(i64), + Text(String), + Bytes(Vec), + TextArray(Vec), +} +impl fmt::Debug for Value { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(match self { + Self::Null => "NULL", + Self::Bool(_) => "", + Self::Int(_) => "", + Self::Text(_) => "", + Self::Bytes(_) => "", + Self::TextArray(_) => "", + }) + } +} +impl From<&str> for Value { + fn from(value: &str) -> Self { + Self::Text(value.into()) + } +} +impl From for Value { + fn from(value: String) -> Self { + Self::Text(value) + } +} +impl From for Value { + fn from(value: i64) -> Self { + Self::Int(value) + } +} +impl From for Value { + fn from(value: i32) -> Self { + Self::Int(value.into()) + } +} +impl From for Value { + fn from(value: u32) -> Self { + Self::Int(value.into()) + } +} +impl From for Value { + fn from(value: bool) -> Self { + Self::Bool(value) + } +} +impl From> for Value { + fn from(value: Option<&str>) -> Self { + value.map(Self::from).unwrap_or(Self::Null) + } +} +impl From> for Value { + fn from(value: Option) -> Self { + value.map(Self::from).unwrap_or(Self::Null) + } +} +impl ToSql for Value { + fn to_sql( + &self, + ty: &Type, + out: &mut BytesMut, + ) -> std::result::Result> { + match self { + Self::Null => Ok(IsNull::Yes), + Self::Bool(value) if *ty == Type::BOOL => value.to_sql(ty, out), + Self::Int(value) => match *ty { + Type::INT2 => i16::try_from(*value) + .map_err(|_| "database_integer_range")? + .to_sql(ty, out), + Type::INT4 => i32::try_from(*value) + .map_err(|_| "database_integer_range")? + .to_sql(ty, out), + Type::INT8 => value.to_sql(ty, out), + _ => Err("database_parameter_type".into()), + }, + Self::Text(value) if ::accepts(ty) => value.to_sql(ty, out), + Self::Bytes(value) if *ty == Type::BYTEA => value.to_sql(ty, out), + Self::TextArray(value) if as ToSql>::accepts(ty) => value.to_sql(ty, out), + _ => Err("database_parameter_type".into()), + } + } + fn accepts(ty: &Type) -> bool { + matches!( + *ty, + Type::BOOL | Type::INT2 | Type::INT4 | Type::INT8 | Type::BYTEA + ) || ::accepts(ty) + || as ToSql>::accepts(ty) + } + to_sql_checked!(); +} +#[derive(Clone)] +pub struct Row(pub(crate) Vec); +impl Row { + pub fn len(&self) -> usize { + self.0.len() + } + pub fn is_empty(&self) -> bool { + self.0.is_empty() + } + pub fn value(&self, index: usize) -> Result<&Value> { + self.0.get(index).ok_or(Error("storage_column_missing")) + } + pub fn text(&self, index: usize) -> Result<&str> { + match self.value(index)? { + Value::Text(v) => Ok(v), + _ => Err(Error("storage_column_type")), + } + } + pub fn optional_text(&self, index: usize) -> Result> { + match self.value(index)? { + Value::Null => Ok(None), + Value::Text(v) => Ok(Some(v)), + _ => Err(Error("storage_column_type")), + } + } + pub fn integer(&self, index: usize) -> Result { + match self.value(index)? { + Value::Int(v) => Ok(*v), + _ => Err(Error("storage_column_type")), + } + } + pub fn boolean(&self, index: usize) -> Result { + match self.value(index)? { + Value::Bool(v) => Ok(*v), + _ => Err(Error("storage_column_type")), + } + } + pub fn json(&self, index: usize) -> Result { + let s = self.text(index)?; + if s.len() > MAX_BYTES { + return Err(Error("storage_record_too_large")); + } + Json::parse(s) + } + pub(crate) fn decode(raw: postgres::Row) -> Result { + let mut values = Vec::with_capacity(raw.len()); + for (i, column) in raw.columns().iter().enumerate() { + let value = match *column.type_() { + Type::BOOL => raw + .try_get::<_, Option>(i) + .map(|v| v.map(Value::Bool).unwrap_or(Value::Null)), + Type::INT2 => raw + .try_get::<_, Option>(i) + .map(|v| v.map(|v| Value::Int(v.into())).unwrap_or(Value::Null)), + Type::INT4 => raw + .try_get::<_, Option>(i) + .map(|v| v.map(|v| Value::Int(v.into())).unwrap_or(Value::Null)), + Type::INT8 => raw + .try_get::<_, Option>(i) + .map(|v| v.map(Value::Int).unwrap_or(Value::Null)), + Type::BYTEA => raw + .try_get::<_, Option>>(i) + .map(|v| v.map(Value::Bytes).unwrap_or(Value::Null)), + Type::TEXT_ARRAY | Type::VARCHAR_ARRAY => raw + .try_get::<_, Option>>(i) + .map(|v| v.map(Value::TextArray).unwrap_or(Value::Null)), + _ => raw + .try_get::<_, Option>(i) + .map(|v| v.map(Value::Text).unwrap_or(Value::Null)), + } + .map_err(|_| Error("storage_column_type"))?; + values.push(value); + } + Ok(Self(values)) + } + pub(crate) fn bytes(&self) -> usize { + self.0 + .iter() + .map(|v| match v { + Value::Text(v) => v.len(), + Value::Bytes(v) => v.len(), + Value::TextArray(v) => v.iter().map(String::len).sum(), + _ => 8, + }) + .sum() + } +} diff --git a/rust/crates/msg-storage/src/postgres/records.rs b/rust/crates/msg-storage/src/postgres/records.rs new file mode 100644 index 00000000..ac7a0f88 --- /dev/null +++ b/rust/crates/msg-storage/src/postgres/records.rs @@ -0,0 +1,645 @@ +//! Typed durable records. These methods do not grant business authority. +use super::{PgSession, Value}; +use crate::records::*; +use msg_core::{Error, Json, Result}; +use msg_identity::{ + models::{Certificate, CertificateRequest, Credential, ResourceRef, Timestamp}, + store::{AuthorityStore, Record}, +}; +use serde::Serialize; +use std::collections::BTreeSet; +fn time(value: Timestamp) -> String { + value.0.format("%Y-%m-%dT%H:%M:%S%.6fZ").to_string() +} +fn raw(value: &impl Serialize) -> Result { + Ok(encode(value)?.canonical()?.into()) +} +fn require(yes: bool, code: &'static str) -> Result<()> { + if yes { + Ok(()) + } else { + Err(Error(code)) + } +} +fn is_hex(id: &str) -> bool { + id.len() == 32 + && id + .bytes() + .all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)) +} +impl PgSession { + fn mutate(&self, body: impl FnOnce() -> Result) -> Result { + self.require_write()?; + self.capture(body(), true) + } + pub fn resource_record(&self, id: &str) -> Result { + if is_hex(id) { + let rows=self.rows("SELECT substring(body,1,1048577) FROM resources WHERE CASE WHEN id ~ '^[0-9a-f]{32}$' THEN id WHEN id ~ '^[A-Za-z][A-Za-z0-9_]*_([0-9a-f]{32})$' THEN right(id,32) ELSE left(encode(sha256(convert_to('msg.hex-reference/v1','UTF8')||decode('00','hex')||convert_to(id,'UTF8')),'hex'),32) END=$1 LIMIT 2",&[id.into()])?; + require(rows.len() <= 1, "ambiguous_resource_id")?; + return decode(&rows.first().ok_or(Error("not_found"))?.json(0)?); + } + let resource: Resource = decode(&self.record(Record::Resource, id)?)?; + require(resource.id == id, "storage_record_mismatch")?; + Ok(resource) + } + pub fn resolve(&self, path: &str, migrated: bool) -> Result { + require( + path.starts_with('/') && !path.contains(['\\', '\0']), + "invalid_path", + )?; + let parts: Vec<_> = path.trim_end_matches('/').split('/').skip(1).collect(); + require( + parts.len() <= 256 + && parts + .iter() + .all(|p| !p.is_empty() && !matches!(*p, "." | "..")), + "invalid_path", + )?; + if !migrated { + if parts.len() == 2 && parts[0] == "_id" { + return Ok(self.resource_record(parts[1])?.id); + } + if let Some(last) = parts.last().and_then(|p| p.strip_prefix('*')) { + let resource = self.resource_record(last)?; + if parts.len() > 1 { + let parent = + self.resolve(&format!("/{}", parts[..parts.len() - 1].join("/")), false)?; + require(resource.parent.as_deref() == Some(&parent), "not_found")?; + } + return Ok(resource.id); + } + } + let mut id = self + .one("SELECT id FROM resources WHERE parent IS NULL", &[])? + .ok_or(Error("not_initialized"))? + .text(0)? + .to_owned(); + if path == "/" { + return Ok(id); + } + require(!parts.is_empty(), "invalid_path")?; + for part in parts { + let mut found = self.one( + "SELECT id FROM resources WHERE parent=$1 AND name=$2", + &[id.clone().into(), part.into()], + )?; + if found.is_none() && migrated { + found = self.one( + "SELECT resource_id FROM resource_path_aliases WHERE parent_id=$1 AND name=$2", + &[id.clone().into(), part.into()], + )?; + } + id = found.ok_or(Error("not_found"))?.text(0)?.to_owned(); + } + Ok(self.resource_record(&id)?.id) + } + pub fn resource_path(&self, id: &str) -> Result { + let mut current = self.resource_record(id)?; + let mut names = Vec::new(); + let mut seen = BTreeSet::new(); + while let Some(parent) = ¤t.parent { + require(seen.insert(current.id.clone()), "parent_cycle")?; + require(seen.len() <= 1024, "authority_work_limit")?; + names.push(current.name.clone()); + current = self.resource_record(parent)?; + } + names.reverse(); + Ok(format!("/{}", names.join("/"))) + } + pub fn children( + &self, + parent: &str, + cursor: Option<&str>, + limit: u32, + ) -> Result> { + require((1..=500).contains(&limit), "invalid_limit")?; + let rows=self.rows("SELECT substring(body,1,1048577) FROM resources WHERE parent=$1 AND id>$2 ORDER BY id LIMIT $3",&[parent.into(),cursor.unwrap_or("").into(),(limit+1).into()])?; + let items: Vec = rows + .iter() + .take(limit as usize) + .map(|r| decode(&r.json(0)?)) + .collect::>()?; + let next_cursor = if rows.len() > limit as usize { + items.last().map(|v| v.id.clone()) + } else { + None + }; + Ok(Page { items, next_cursor }) + } + pub fn revision(&self, reference: &ResourceRef) -> Result { + let resource = self.resource_record(&reference.id)?; + let id = reference + .revision + .as_ref() + .or(resource.revision.as_ref()) + .ok_or(Error("revision_not_found"))?; + let rows = if is_hex(id) { + self.rows("SELECT substring(body,1,1048577) FROM revisions WHERE resource_id=$1 AND CASE WHEN id ~ '^[0-9a-f]{32}$' THEN id WHEN id ~ '^[A-Za-z][A-Za-z0-9_]*_([0-9a-f]{32})$' THEN right(id,32) ELSE left(encode(sha256(convert_to('msg.hex-reference/v1','UTF8')||decode('00','hex')||convert_to(id,'UTF8')),'hex'),32) END=$2 LIMIT 2",&[resource.id.clone().into(),id.as_str().into()])? + } else { + self.rows( + "SELECT substring(body,1,1048577) FROM revisions WHERE resource_id=$1 AND id=$2", + &[resource.id.clone().into(), id.as_str().into()], + )? + }; + require(rows.len() <= 1, "ambiguous_revision_id")?; + let revision: Revision = + decode(&rows.first().ok_or(Error("revision_not_found"))?.json(0)?)?; + require( + revision.resource_id == resource.id && (is_hex(id) || revision.id == *id), + "storage_record_mismatch", + )?; + Ok(revision) + } + pub fn history(&self, id: &str, cursor: Option<&str>, limit: u32) -> Result> { + require((1..=500).contains(&limit), "invalid_limit")?; + let rows=self.rows("SELECT substring(body,1,1048577) FROM revisions WHERE resource_id=$1 AND id>$2 ORDER BY id LIMIT $3",&[id.into(),cursor.unwrap_or("").into(),(limit+1).into()])?; + let items: Vec = rows + .iter() + .take(limit as usize) + .map(|r| decode(&r.json(0)?)) + .collect::>()?; + let next_cursor = if rows.len() > limit as usize { + items.last().map(|v| v.id.clone()) + } else { + None + }; + Ok(Page { items, next_cursor }) + } + pub fn set_setting(&self, key: &str, value: &Json) -> Result<()> { + self.mutate(||{ + self.execute("INSERT INTO settings(key,value) VALUES($1,$2) ON CONFLICT(key) DO UPDATE SET value=excluded.value",&[key.into(),value.canonical()?.into()])?;Ok(()) + }) + } + pub fn bump_authorization_epoch(&self) -> Result<()> { + self.mutate(|| { + let old = self + .setting("authorization_epoch")? + .map(|v| { + v.as_integer()? + .parse::() + .map_err(|_| Error("invalid_authorization_epoch")) + }) + .transpose()? + .unwrap_or(0); + let next = old + .checked_add(1) + .filter(|_| old >= 0) + .ok_or(Error("invalid_authorization_epoch"))?; + self.set_setting("authorization_epoch", &Json::parse(&next.to_string())?) + }) + } + pub fn insert_resource(&self, r: &Resource) -> Result<()> { + self.mutate(||{ + r.validate()?;if let Some(parent)=&r.parent{self.resource_record(parent)?;} + self.execute("INSERT INTO resources(id,type,name,parent,owner,grp,mode,generation,revision,state,created_at,modified_at,body) VALUES($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13)",&[ + r.id.as_str().into(),r.resource_type.as_str().into(),r.name.as_str().into(),r.parent.as_deref().into(),r.owner.as_str().into(),r.group.as_str().into(),(r.mode as i64).into(),r.generation.into(),r.revision.as_deref().into(),r.state.sql().into(),time(r.created_at).into(),time(r.modified_at).into(),raw(r)?])?; + for tag in &r.tags{self.execute("INSERT INTO resource_tags(resource_id,tag) VALUES($1,$2)",&[r.id.as_str().into(),tag.as_str().into()])?;}Ok(()) + }) + } + pub fn replace_resource(&self, r: &Resource, expected: i64) -> Result<()> { + self.mutate(||{ + r.validate()?;let old=self.resource_record(&r.id)?; + require(old.generation==expected,"generation_conflict")?;require(expected.checked_add(1)==Some(r.generation),"invalid_generation")?; + require(old.resource_type==r.resource_type&&old.created_at==r.created_at&&old.created_by==r.created_by,"immutable_creation_fact")?; + if let Some(parent)=&r.parent{require(parent!=&r.id&&!self.ancestors(parent)?.iter().any(|a|a.id==r.id),"parent_cycle")?;} + if old.parent!=r.parent||old.owner!=r.owner||old.group!=r.group||old.mode!=r.mode||old.state!=r.state{self.bump_authorization_epoch()?;} + if let Some(parent)=&old.parent{if old.parent!=r.parent||old.name!=r.name{self.execute("INSERT INTO resource_path_aliases(parent_id,name,resource_id) VALUES($1,$2,$3) ON CONFLICT(parent_id,name) DO UPDATE SET resource_id=excluded.resource_id",&[parent.as_str().into(),old.name.as_str().into(),old.id.as_str().into()])?;}} + let changed=self.execute("UPDATE resources SET name=$1,parent=$2,owner=$3,grp=$4,mode=$5,generation=$6,revision=$7,state=$8,modified_at=$9,body=$10 WHERE id=$11 AND generation=$12",&[ + r.name.as_str().into(),r.parent.as_deref().into(),r.owner.as_str().into(),r.group.as_str().into(),(r.mode as i64).into(),r.generation.into(),r.revision.as_deref().into(),r.state.sql().into(),time(r.modified_at).into(),raw(r)?,r.id.as_str().into(),expected.into()])?; + require(changed==1,"generation_conflict")?; + if old.tags!=r.tags{self.execute("DELETE FROM resource_tags WHERE resource_id=$1",&[r.id.as_str().into()])?;for tag in &r.tags{self.execute("INSERT INTO resource_tags(resource_id,tag) VALUES($1,$2)",&[r.id.as_str().into(),tag.as_str().into()])?;}}Ok(()) + }) + } + pub fn append_revision(&self, r: &Revision) -> Result<()> { + self.mutate(||{ + require(r.format_version>=1,"invalid_version")?;require(r.content.size>=0,"invalid_nonnegative_integer")?; + for parent in &r.parents{self.revision(&ResourceRef{id:r.resource_id.clone(),revision:Some(parent.clone())})?;} + self.execute("INSERT INTO revisions(id,resource_id,created_at,body) VALUES($1,$2,$3,$4)",&[r.id.as_str().into(),r.resource_id.as_str().into(),time(r.created_at).into(),raw(r)?])?; + for rel in &r.relations{ + require(!rel.excerpt.is_some_and(|[a,b]|a<0||a>b),"invalid_byte_range")?; + self.execute("INSERT INTO relations(revision_id,source_id,type,target_id,target_revision,body) VALUES($1,$2,$3,$4,$5,$6)",&[r.id.as_str().into(),r.resource_id.as_str().into(),rel.relation_type.as_str().into(),rel.target.id.as_str().into(),rel.target.revision.as_deref().into(),raw(rel)?])?; + }Ok(()) + }) + } + pub fn save_result( + &self, + subject: Option<&str>, + digest: &str, + result: &OperationResult, + ) -> Result<()> { + self.mutate(|| { + require( + subject == result.subject.as_deref() && subject != Some(""), + "storage_record_mismatch", + )?; + require( + digest.strip_prefix("sha256:").is_some_and(|s| { + s.len() == 64 + && s.bytes() + .all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)) + }), + "invalid_digest", + )?; + require( + self.one("SELECT count(*) FROM results", &[])? + .ok_or(Error("storage_error"))? + .integer(0)? + < 100_000, + "storage_capacity_exceeded", + )?; + self.execute( + "INSERT INTO results(subject,request_id,digest,body) VALUES($1,$2,$3,$4)", + &[ + subject.unwrap_or("").into(), + result.request_id.as_str().into(), + digest.into(), + raw(result)?, + ], + )?; + Ok(()) + }) + } + pub fn append_event(&self, event: &Event) -> Result<()> { + self.mutate(|| { + self.execute( + "INSERT INTO events(id,body) VALUES($1,$2)", + &[event.id.as_str().into(), raw(event)?], + )?; + Ok(()) + }) + } + pub fn append_audit(&self, event: &AuditEvent) -> Result<()> { + self.mutate(|| { + let previous = self + .one("SELECT digest FROM audit ORDER BY seq DESC LIMIT 1", &[])? + .map(|r| Ok(r.text(0)?.to_owned())) + .transpose()?; + let mut event = event.clone(); + event.previous_digest = previous; + event.entry_digest.clear(); + let mut fields = encode(&event)?.into_object()?; + fields.remove("entry_digest"); + event.entry_digest = + msg_core::digest(Json::from_object(fields).canonical()?.as_bytes()); + self.execute( + "INSERT INTO audit(digest,previous,body) VALUES($1,$2,$3)", + &[ + event.entry_digest.as_str().into(), + event.previous_digest.as_deref().into(), + raw(&event)?, + ], + )?; + Ok(()) + }) + } + pub fn update_identity(&self, record: IdentityRecord<'_>, expected: i64) -> Result<()> { + self.mutate(|| { + let (select, insert, update, mut keys, generation, body, kind) = match record { + IdentityRecord::Subject(s) => ( + "SELECT generation FROM identities WHERE id=$1", + "INSERT INTO identities(id,kind,generation,body) VALUES($1,$2,$3,$4)", + "UPDATE identities SET generation=$1,body=$2 WHERE id=$3", + vec![s.resource_id.as_str().into()], + i64::try_from(s.auth_version).map_err(|_| Error("invalid_generation"))?, + raw(s)?, + Some("subject"), + ), + IdentityRecord::Organization(g) => { + require( + g.membership_version >= 0 + && matches!( + g.membership_policy.as_str(), + "open" | "approval" | "invite" | "managed" + ) + && g.builtin + .as_ref() + .is_none_or(|s| matches!(s.as_str(), "public" | "admins")), + "invalid_identity_record", + )?; + ( + "SELECT generation FROM identities WHERE id=$1", + "INSERT INTO identities(id,kind,generation,body) VALUES($1,$2,$3,$4)", + "UPDATE identities SET generation=$1,body=$2 WHERE id=$3", + vec![g.resource_id.as_str().into()], + g.membership_version, + raw(g)?, + Some("organization"), + ) + } + IdentityRecord::Membership(m) => { + require( + m.version >= 0 + && matches!( + m.role.as_str(), + "owner" | "maintainer" | "member" | "admin" + ) + && matches!( + m.status.as_str(), + "active" | "pending" | "invited" | "rejected" + ), + "invalid_identity_record", + )?; + self.bump_authorization_epoch()?; + ( + "SELECT generation FROM memberships WHERE org=$1 AND subject=$2", + "INSERT INTO memberships(org,subject,generation,body) VALUES($1,$2,$3,$4)", + "UPDATE memberships SET generation=$1,body=$2 WHERE org=$3 AND subject=$4", + vec![ + m.organization_id.as_str().into(), + m.subject_id.as_str().into(), + ], + m.version, + raw(m)?, + None, + ) + } + IdentityRecord::Email(e) => ( + "SELECT generation FROM emails WHERE subject=$1", + "INSERT INTO emails(subject,generation,body) VALUES($1,$2,$3)", + "UPDATE emails SET generation=$1,body=$2 WHERE subject=$3", + vec![e.subject_id.as_str().into()], + expected.checked_add(1).ok_or(Error("invalid_generation"))?, + raw(e)?, + None, + ), + }; + let old = self.one(select, &keys)?; + require( + match &old { + None => expected == -1, + Some(row) => row.integer(0)? == expected, + }, + "generation_conflict", + )?; + if old.is_none() { + if let Some(kind) = kind { + keys.push(kind.into()); + } + keys.push(generation.into()); + keys.push(body); + self.execute(insert, &keys)?; + } else { + let mut params = vec![generation.into(), body]; + params.extend(keys); + self.execute(update, ¶ms)?; + } + Ok(()) + }) + } + pub fn save_credential(&self, c: &Credential, expected_auth_version: u64) -> Result<()> { + self.mutate(||{ + require(self.subject(&c.subject_id)?.auth_version==expected_auth_version,"auth_version_conflict")?; + if let Some(row)=self.one("SELECT substring(body,1,1048577) FROM credentials WHERE id=$1",&[c.id.as_str().into()])?{ + let old:Credential=msg_identity::models::decode(&row.json(0)?)?; + require(old.subject_id==c.subject_id&&old.kind==c.kind&&old.verifier.0==c.verifier.0&&old.source_credential_id==c.source_credential_id,"credential_identity_immutable")?; + } + self.execute("INSERT INTO credentials(id,subject,body) VALUES($1,$2,$3) ON CONFLICT(id) DO UPDATE SET body=excluded.body",&[c.id.as_str().into(),c.subject_id.as_str().into(),raw(c)?])?;Ok(()) + }) + } + pub fn csr(&self, id: &str) -> Result { + msg_identity::models::decode( + &self + .one( + "SELECT substring(body,1,1048577) FROM csrs WHERE id=$1", + &[id.into()], + )? + .ok_or(Error("csr_not_found"))? + .json(0)?, + ) + } + pub fn csr_state(&self, id: &str) -> Result { + decode( + &self + .one( + "SELECT substring(state_body,1,1048577) FROM csrs WHERE id=$1", + &[id.into()], + )? + .ok_or(Error("csr_not_found"))? + .json(0)?, + ) + } + pub fn save_csr(&self, request: &CertificateRequest) -> Result<()> { + self.mutate(||{ + let state=CertificateRequestState{request_id:request.resource_id.clone(),status:"pending".into(),generation:0,certificate_id:None}; + self.execute("INSERT INTO csrs(id,generation,state,body,state_body) VALUES($1,0,'pending',$2,$3)",&[request.resource_id.as_str().into(),raw(request)?,raw(&state)?])?;Ok(()) + }) + } + pub fn transition_csr(&self, state: &CertificateRequestState, expected: i64) -> Result<()> { + self.mutate(||{ + require(matches!(state.status.as_str(),"pending"|"issued"|"rejected"|"cancelled"|"expired"),"invalid_csr_state")?; + require(expected.checked_add(1)==Some(state.generation),"invalid_generation")?; + let old=self.csr_state(&state.request_id)?;require(old.status=="pending","csr_not_pending")?;require(old.generation==expected,"generation_conflict")?; + require(self.execute("UPDATE csrs SET generation=$1,state=$2,state_body=$3 WHERE id=$4 AND generation=$5",&[state.generation.into(),state.status.as_str().into(),raw(state)?,state.request_id.as_str().into(),expected.into()])?==1,"generation_conflict") + }) + } + pub fn register_certificate(&self, cert: &Certificate, csr: Option<(&str, i64)>) -> Result<()> { + self.mutate(|| { + self.execute( + "INSERT INTO certificates(id,subject,parent,revoked,body) VALUES($1,$2,$3,0,$4)", + &[ + cert.resource_id.as_str().into(), + cert.subject_id.as_str().into(), + cert.parent_certificate_id.as_deref().into(), + raw(cert)?, + ], + )?; + if let Some((id, generation)) = csr { + self.set_setting( + &format!("certificate_request:{}", cert.resource_id), + &Json::string(id), + )?; + self.transition_csr( + &CertificateRequestState { + request_id: id.into(), + status: "issued".into(), + generation: generation + .checked_add(1) + .ok_or(Error("invalid_generation"))?, + certificate_id: Some(cert.resource_id.clone()), + }, + generation, + )?; + } + Ok(()) + }) + } + pub fn revoke_certificate(&self, id: &str, event: &AuditEvent) -> Result<()> { + self.mutate(|| { + self.certificate(id)?; + self.execute( + "UPDATE certificates SET revoked=1 WHERE id=$1", + &[id.into()], + )?; + self.bump_authorization_epoch()?; + self.append_audit(event) + }) + } + pub fn transfer(&self, id: &str) -> Result { + decode( + &self + .one( + "SELECT substring(body,1,1048577) FROM transfers WHERE id=$1", + &[id.into()], + )? + .ok_or(Error("transfer_not_found"))? + .json(0)?, + ) + } + pub fn save_transfer(&self, t: &TransferSession, expected: Option) -> Result<()> { + self.mutate(|| { + require( + matches!(t.direction.as_str(), "upload" | "download") + && matches!( + t.state.as_str(), + "open" | "sealed" | "cancelled" | "expired" + ) + && t.generation >= 0 + && t.expected_size.is_none_or(|n| n >= 0), + "invalid_transfer", + )?; + if let Some(expected) = expected { + require( + expected.checked_add(1) == Some(t.generation), + "invalid_generation", + )?; + require( + self.execute( + "UPDATE transfers SET generation=$1,body=$2 WHERE id=$3 AND generation=$4", + &[ + t.generation.into(), + raw(t)?, + t.id.as_str().into(), + expected.into(), + ], + )? == 1, + "generation_conflict", + )?; + } else { + self.execute( + "INSERT INTO transfers(id,subject,generation,body) VALUES($1,$2,$3,$4)", + &[ + t.id.as_str().into(), + t.subject_id.as_str().into(), + t.generation.into(), + raw(t)?, + ], + )?; + } + Ok(()) + }) + } + pub fn put_chunk(&self, c: &TransferChunk) -> Result<()> { + self.mutate(||{ + require(c.offset>=0&&c.content.size>=0,"invalid_nonnegative_integer")?; + let end=c.offset.checked_add(c.content.size).ok_or(Error("invalid_byte_range"))?; + let rows=self.rows("SELECT \"offset\",length,substring(body,1,1048577) FROM chunks WHERE transfer_id=$1 AND \"offset\"<$2 AND \"offset\"+length>$3",&[c.transfer_id.as_str().into(),end.into(),c.offset.into()])?; + if !rows.is_empty(){require(rows.len()==1&&rows[0].integer(0)?==c.offset&&rows[0].integer(1)?==c.content.size,"chunk_conflict")?; + let old:TransferChunk=decode(&rows[0].json(2)?)?;require(encode(&old.content)?.canonical()?==encode(&c.content)?.canonical()?,"chunk_conflict")?;return Ok(());} + self.execute("INSERT INTO chunks(transfer_id,\"offset\",length,body) VALUES($1,$2,$3,$4)",&[c.transfer_id.as_str().into(),c.offset.into(),c.content.size.into(),raw(c)?])?;Ok(()) + }) + } + pub fn missing_ranges( + &self, + id: &str, + cursor: Option<&str>, + limit: u32, + ) -> Result> { + let size = self + .transfer(id)? + .expected_size + .ok_or(Error("size_required"))?; + require((1..=500).contains(&limit), "invalid_limit")?; + let mut start = cursor + .unwrap_or("0") + .parse::() + .map_err(|_| Error("invalid_cursor"))?; + require(start >= 0 && start <= size, "invalid_cursor")?; + let rows=self.rows("SELECT \"offset\",length FROM chunks WHERE transfer_id=$1 AND \"offset\"+length>$2 ORDER BY \"offset\" LIMIT 4097",&[id.into(),start.into()])?; + let mut gaps = Vec::new(); + for row in rows { + let offset = row.integer(0)?; + let length = row.integer(1)?; + require(offset >= 0 && length >= 0, "invalid_byte_range")?; + if offset > start { + gaps.push([start, offset]); + if gaps.len() > limit as usize { + break; + } + } + start = start.max( + offset + .checked_add(length) + .ok_or(Error("invalid_byte_range"))?, + ); + } + if gaps.len() <= limit as usize && start < size { + gaps.push([start, size]); + } + let next_cursor = gaps.get(limit as usize).map(|v| v[0].to_string()); + gaps.truncate(limit as usize); + Ok(Page { + items: gaps, + next_cursor, + }) + } + pub fn job(&self, id: &str) -> Result { + let row = self + .one( + "SELECT state,next_at,substring(body,1,1048577) FROM jobs WHERE id=$1", + &[id.into()], + )? + .ok_or(Error("job_not_found"))?; + let job: EffectJob = decode(&row.json(2)?)?; + require( + job.id == id + && job.state == row.text(0)? + && time(job.next_attempt_at) == row.text(1)?, + "storage_record_mismatch", + )?; + Ok(job) + } + pub fn enqueue(&self, job: &EffectJob) -> Result<()> { + self.mutate(|| { + require( + job.attempts == 0 && job.state == "pending" && job.lease_until.is_none(), + "invalid_job_state", + )?; + self.execute( + "INSERT INTO jobs(id,dedupe,kind,state,next_at,body) VALUES($1,$2,$3,$4,$5,$6)", + &[ + job.id.as_str().into(), + job.dedupe_key.as_str().into(), + job.kind.as_str().into(), + job.state.as_str().into(), + time(job.next_attempt_at).into(), + raw(job)?, + ], + )?; + Ok(()) + }) + } + pub fn save_job(&self, job: &EffectJob) -> Result<()> { + self.mutate(|| { + require( + job.attempts >= 0 + && matches!( + job.state.as_str(), + "pending" | "running" | "done" | "failed" | "uncertain" + ), + "invalid_job_state", + )?; + require( + self.execute( + "UPDATE jobs SET state=$1,next_at=$2,body=$3 WHERE id=$4", + &[ + job.state.as_str().into(), + time(job.next_attempt_at).into(), + raw(job)?, + job.id.as_str().into(), + ], + )? == 1, + "job_not_found", + ) + }) + } +} diff --git a/rust/crates/msg-storage/src/records.rs b/rust/crates/msg-storage/src/records.rs new file mode 100644 index 00000000..cb38d897 --- /dev/null +++ b/rust/crates/msg-storage/src/records.rs @@ -0,0 +1,421 @@ +//! Typed storage rows, not an operation schema or authorization decision. +//! The domain layer still owns tags, quotas, ACLs, receipt signing and policy. +use msg_core::{Error, Json, Result}; +use msg_crypto::Signature; +use msg_identity::models::{ResourceRef, Timestamp}; +use serde::{de::DeserializeOwned, Deserialize, Deserializer, Serialize, Serializer}; +use std::collections::BTreeMap; + +pub fn decode(value: &Json) -> Result { + serde_json::from_str(&value.canonical()?).map_err(|_| Error("invalid_storage_record")) +} +pub fn encode(value: &T) -> Result { + Json::parse(&serde_json::to_string(value).map_err(|_| Error("invalid_storage_record"))?) +} +fn required_option<'de, T: Deserialize<'de>, D: Deserializer<'de>>( + d: D, +) -> std::result::Result, D::Error> { + Option::::deserialize(d) +} +mod mode { + use super::*; + pub fn serialize(n: &u16, s: S) -> std::result::Result { + if *n > 0o7777 { + return Err(serde::ser::Error::custom("invalid_mode")); + } + s.serialize_str(&format!("{n:04o}")) + } + pub fn deserialize<'de, D: Deserializer<'de>>(d: D) -> std::result::Result { + let value = String::deserialize(d)?; + if value.len() != 4 || !value.bytes().all(|b| (b'0'..=b'7').contains(&b)) { + return Err(serde::de::Error::custom("invalid_mode")); + } + u16::from_str_radix(&value, 8).map_err(serde::de::Error::custom) + } +} +#[derive(Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ResourceState { + Active, + Archived, + Purged, +} +impl ResourceState { + pub(crate) fn sql(self) -> &'static str { + match self { + Self::Active => "active", + Self::Archived => "archived", + Self::Purged => "purged", + } + } +} +#[derive(Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Resource { + pub id: String, + #[serde(rename = "type")] + pub resource_type: String, + pub type_version: i64, + pub name: String, + #[serde(deserialize_with = "required_option")] + pub parent: Option, + pub owner: String, + pub group: String, + #[serde(with = "mode")] + pub mode: u16, + pub generation: i64, + #[serde(deserialize_with = "required_option")] + pub revision: Option, + pub state: ResourceState, + pub created_at: Timestamp, + pub created_by: String, + pub modified_at: Timestamp, + pub modified_by: String, + #[serde(default)] + pub tags: Vec, +} +impl Resource { + pub(crate) fn validate(&self) -> Result<()> { + if self.generation < 0 { + return Err(Error("invalid_nonnegative_integer")); + } + if self.type_version < 1 { + return Err(Error("invalid_version")); + } + if self.mode > 0o7777 { + return Err(Error("invalid_mode")); + } + if self.parent.as_deref() == Some(&self.id) { + return Err(Error("parent_cycle")); + } + // Unicode tag normalization belongs to the still-unported domain layer. + // Preserve existing normalized Unicode; never re-normalize stored bytes. + if self.tags.len() > 16 || self.tags.windows(2).any(|w| w[0] >= w[1]) { + return Err(Error("invalid_tags")); + } + Ok(()) + } +} +#[derive(Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct BlobRef { + pub digest: String, + pub size: i64, + pub media_type: String, +} +#[derive(Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Relation { + #[serde(rename = "type")] + pub relation_type: String, + pub target: ResourceRef, + #[serde(default)] + pub excerpt: Option<[i64; 2]>, +} +#[derive(Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Revision { + pub format_version: i64, + pub id: String, + pub resource_id: String, + pub parents: Vec, + pub content: BlobRef, + pub relations: Vec, + pub actor: String, + pub subject: String, + pub author: String, + pub created_at: Timestamp, + pub manifest_digest: String, + #[serde(default)] + pub signature: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub summary: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub change_note: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub source_kind: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub source_version: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub source_digest: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub signature_source: Option, +} +#[derive(Clone, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum SourceKind { + Release, + User, + Operation, +} +#[derive(Clone, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum SignatureSource { + Custodial, +} +#[derive(Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct OperationError { + pub code: String, + pub retryable: bool, + #[serde(default)] + pub field_path: Option, + #[serde(default)] + pub retry_after_seconds: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub message: Option, +} +#[derive(Clone, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ResultStatus { + Ok, + Accepted, + Error, + Uncertain, +} +#[derive(Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct OperationResult { + pub request_id: String, + pub operation: String, + pub status: ResultStatus, + #[serde(deserialize_with = "required_option")] + pub actor: Option, + #[serde(deserialize_with = "required_option")] + pub subject: Option, + #[serde(default)] + pub resources: Vec, + #[serde(default)] + pub committed_at: Option, + #[serde(default)] + pub replayed: bool, + #[serde(default)] + pub receipt: Option, + #[serde(default)] + pub error: Option, + #[serde(default)] + pub data: Option>, + #[serde(default)] + pub output: Option, + #[serde(default)] + pub prefer_cli: bool, + #[serde(default)] + pub cli_url: Option, +} +#[derive(Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Event { + pub id: String, + #[serde(rename = "type")] + pub event_type: String, + pub time: Timestamp, + pub request_id: String, + #[serde(deserialize_with = "required_option")] + pub actor: Option, + #[serde(deserialize_with = "required_option")] + pub subject: Option, + pub resources: Vec, + pub data: BTreeMap, +} +#[derive(Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct AuditEvent { + pub event: Event, + pub authority: Vec, + #[serde(deserialize_with = "required_option")] + pub before_digest: Option, + #[serde(deserialize_with = "required_option")] + pub after_digest: Option, + #[serde(deserialize_with = "required_option")] + pub previous_digest: Option, + pub entry_digest: String, + pub result: String, +} + +#[derive(Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Organization { + pub resource_id: String, + pub membership_version: i64, + #[serde(default)] + pub builtin: Option, + #[serde(default = "invite")] + pub membership_policy: String, +} +fn invite() -> String { + "invite".into() +} +fn active() -> String { + "active".into() +} +#[derive(Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Membership { + pub organization_id: String, + pub subject_id: String, + pub role: String, + pub version: i64, + #[serde(default = "active")] + pub status: String, + #[serde(default)] + pub joined_at: Option, + #[serde(default)] + pub invited_by: Option, +} +impl Organization { + pub fn validate(&self) -> Result<()> { + if self.membership_version < 0 + || !matches!( + self.membership_policy.as_str(), + "open" | "approval" | "invite" | "managed" + ) + || self + .builtin + .as_ref() + .is_some_and(|s| !matches!(s.as_str(), "public" | "admins")) + { + return Err(Error("invalid_identity_record")); + } + Ok(()) + } +} +impl Membership { + pub fn validate(&self) -> Result<()> { + if self.version < 0 + || !matches!( + self.role.as_str(), + "owner" | "maintainer" | "member" | "admin" + ) + || !matches!( + self.status.as_str(), + "active" | "pending" | "invited" | "rejected" + ) + { + return Err(Error("invalid_identity_record")); + } + Ok(()) + } +} + +#[derive(Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct EmailSettings { + pub subject_id: String, + #[serde(deserialize_with = "required_option")] + pub address: Option, + #[serde(deserialize_with = "required_option")] + pub verified_at: Option, + #[serde(default)] + pub enabled_events: std::collections::BTreeSet, +} +pub enum IdentityRecord<'a> { + Subject(&'a msg_identity::models::Subject), + Organization(&'a Organization), + Membership(&'a Membership), + Email(&'a EmailSettings), +} +#[derive(Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct CertificateRequestState { + pub request_id: String, + pub status: String, + pub generation: i64, + #[serde(default)] + pub certificate_id: Option, +} +#[derive(Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct TransferSession { + pub id: String, + pub subject_id: String, + pub direction: String, + pub state: String, + #[serde(deserialize_with = "required_option")] + pub target: Option, + #[serde(deserialize_with = "required_option")] + pub expected_size: Option, + #[serde(deserialize_with = "required_option")] + pub expected_digest: Option, + pub expires_at: Timestamp, + pub generation: i64, + #[serde(default)] + pub output: Option, +} +#[derive(Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct TransferChunk { + pub transfer_id: String, + pub offset: i64, + pub content: BlobRef, +} +#[derive(Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct EffectJob { + pub id: String, + pub event_id: String, + pub kind: String, + pub dedupe_key: String, + // A stored snapshot is not an authentication result. Workers must obtain a + // freshly validated Principal through the stored-job authentication port. + principal: Json, + pub operation: String, + pub arguments: BTreeMap, + pub state: String, + pub attempts: i64, + pub next_attempt_at: Timestamp, + #[serde(deserialize_with = "required_option")] + pub lease_until: Option, + #[serde(default)] + pub result: Option, +} +/// Trusted effect description. The identity is supplied separately after authentication. +pub struct JobDraft { + pub id: String, + pub event_id: String, + pub kind: String, + pub dedupe_key: String, + pub operation: String, + pub arguments: BTreeMap, +} +impl EffectJob { + pub fn new( + principal: &msg_identity::authentication::Principal, + draft: JobDraft, + now: Timestamp, + ) -> Result { + let JobDraft { + id, + event_id, + kind, + dedupe_key, + operation, + arguments, + } = draft; + Ok(Self { + id, + event_id, + kind, + dedupe_key, + principal: encode(principal)?, + operation, + arguments, + state: "pending".into(), + attempts: 0, + next_attempt_at: now, + lease_until: None, + result: None, + }) + } + pub fn subject(&self) -> Result<&str> { + self.principal + .as_object()? + .get("subject") + .ok_or(Error("worker_identity_required"))? + .as_str() + } +} +#[derive(Clone, Serialize)] +pub struct Page { + pub items: Vec, + pub next_cursor: Option, +} diff --git a/rust/crates/msg-storage/src/writer.rs b/rust/crates/msg-storage/src/writer.rs new file mode 100644 index 00000000..7bc3e42f --- /dev/null +++ b/rust/crates/msg-storage/src/writer.rs @@ -0,0 +1,632 @@ +//! Unix writer fence and atomic metadata sessions. No service entry point. +//! +//! Run this synchronous API on the database-owning blocking thread. Do not drop +//! an async wrapper and assume that its in-flight transaction was cancelled. +//! One transaction owns the fence until rollback AND compensation have finished. +use crate::{database_error, records, ReadOnly, Session}; +use fs2::FileExt; +use msg_core::{Error, Json, Result}; +use msg_identity::store::{AuthorityStore, Record}; +use records::{AuditEvent, Event, OperationResult, Resource, Revision}; +use rusqlite::{params, Connection, OpenFlags, OptionalExtension, TransactionBehavior}; +use std::{ + cell::Cell, + fs::{self, File, OpenOptions}, + io::ErrorKind, + marker::PhantomData, + os::unix::fs::{MetadataExt, OpenOptionsExt}, + panic::{catch_unwind, resume_unwind, AssertUnwindSafe}, + path::{Path, PathBuf}, + thread, + time::{Duration, Instant}, +}; + +#[derive(Debug)] +pub struct WriteFailure { + pub cause: Error, + /// Stable diagnostic codes only: never SQL text, credentials or row bodies. + pub cleanup_errors: Vec, +} +impl From for WriteFailure { + fn from(cause: Error) -> Self { + Self { + cause, + cleanup_errors: Vec::new(), + } + } +} +/// Only the store can construct this access state. +#[derive(Default)] +pub struct WriteAccess { + effects: Vec Result<()>>>, + cleanup_errors: Vec, + savepoint: u64, + commit_uncertain: bool, +} +pub type WriteSession<'a> = Session<'a, WriteAccess>; + +pub struct SqliteWriter { + connection: Connection, + path: PathBuf, + timeout: Duration, + needs_recovery: bool, + identity: (u64, u64), +} +impl SqliteWriter { + /// Open a pre-migrated WAL database. Never create it or change its schema. + /// The configured database path must identify one file, not a symlink or + /// hard-link alias that another writer could fence under a different name. + pub fn open_existing(path: &Path, timeout: Duration) -> Result { + if timeout.as_millis() > i32::MAX as u128 { + return Err(Error("invalid_timeout")); + } + let metadata = fs::symlink_metadata(path).map_err(|_| Error("storage_error"))?; + if !metadata.is_file() || metadata.nlink() != 1 { + return Err(Error("unsafe_storage_path")); + } + let path = fs::canonicalize(path).map_err(|_| Error("storage_error"))?; + let mut connection = Connection::open_with_flags( + &path, + OpenFlags::SQLITE_OPEN_READ_WRITE + | OpenFlags::SQLITE_OPEN_NO_MUTEX + | OpenFlags::SQLITE_OPEN_NOFOLLOW, + ) + .map_err(database_error)?; + connection.busy_timeout(timeout).map_err(database_error)?; + connection + .execute_batch( + "PRAGMA foreign_keys=ON; PRAGMA trusted_schema=OFF; PRAGMA synchronous=FULL;", + ) + .map_err(database_error)?; + let mode: String = connection + .query_row("PRAGMA journal_mode", [], |r| r.get(0)) + .map_err(database_error)?; + if mode != "wal" { + return Err(Error("unsupported_storage_journal")); + } + { + let session = Session { + transaction: connection.transaction().map_err(database_error)?, + access: ReadOnly, + poison: Cell::new(None), + _owner: PhantomData, + }; + session.validate_schema()?; + validate_write_schema(&session.transaction)?; + } + Ok(Self { + connection, + path, + timeout, + needs_recovery: false, + identity: (metadata.dev(), metadata.ino()), + }) + } + + /// Return a value only after COMMIT succeeds. All fallible writes are sticky: + /// ignoring their error cannot commit an earlier partial mutation. + /// + /// Sessions cannot escape, commit themselves, or move to another thread. + /// ```compile_fail + /// use msg_storage::WriteSession; + /// fn handler(tx: &mut WriteSession<'_>) { tx.commit().unwrap(); } + /// ``` + /// ```compile_fail + /// use msg_storage::WriteSession; + /// fn assert_send() {} + /// fn check() { assert_send::>(); } + /// ``` + /// ```compile_fail + /// use msg_storage::SqliteWriter; + /// fn escape(store: &mut SqliteWriter) { let _tx = store.transaction(|tx| Ok(tx)); } + /// ``` + /// ```compile_fail + /// use msg_storage::Snapshot; + /// use msg_core::Json; + /// fn mutate_read(tx: &mut Snapshot<'_>) { tx.set_setting("x", &Json::parse("1").unwrap()); } + /// ``` + pub fn transaction( + &mut self, + f: impl FnOnce(&mut WriteSession<'_>) -> Result, + ) -> std::result::Result { + if self.needs_recovery { + return Err(Error("storage_recovery_required").into()); + } + let deadline = Instant::now() + .checked_add(self.timeout) + .ok_or(Error("invalid_timeout"))?; + // This guard is intentionally outside the SQL transaction. Do not unlink + // its file on release: a second inode would break the process fence. + let _fence = acquire_fence(&self.path, deadline)?; + let metadata = + fs::symlink_metadata(&self.path).map_err(|_| Error("storage_recovery_required"))?; + if !metadata.is_file() + || metadata.nlink() != 1 + || (metadata.dev(), metadata.ino()) != self.identity + { + self.needs_recovery = true; + return Err(Error("storage_recovery_required").into()); + } + self.connection + .busy_timeout(deadline.saturating_duration_since(Instant::now())) + .map_err(database_error)?; + let transaction = self + .connection + .transaction_with_behavior(TransactionBehavior::Immediate) + .map_err(database_error)?; + let mut session = Session { + transaction, + access: WriteAccess::default(), + poison: Cell::new(None), + _owner: PhantomData, + }; + // Revalidate after the writer lock: an old schema read is not authority. + let outcome = catch_unwind(AssertUnwindSafe(|| { + session.validate_schema()?; + validate_write_schema(&session.transaction)?; + let result = f(&mut session)?; + session.active()?; + if let Err(error) = session.transaction.execute_batch("COMMIT") { + return Err(session.commit_failed(error)); + } + Ok(result) + })); + match outcome { + Ok(Ok(value)) => { + session.access.effects.clear(); + Ok(value) + } + other => { + if !session.transaction.is_autocommit() { + if let Err(error) = session.transaction.execute_batch("ROLLBACK") { + session.access.cleanup_errors.push(database_error(error)); + } + } + // An I/O error can leave COMMIT's outcome unknown. Do not undo + // external data that might now be referenced by committed rows. + if !session.access.commit_uncertain { + session.compensate(0); + } + self.needs_recovery = session.access.commit_uncertain + || !session.access.cleanup_errors.is_empty() + || !session.transaction.is_autocommit(); + match other { + Ok(Err(cause)) => Err(WriteFailure { + cause, + cleanup_errors: std::mem::take(&mut session.access.cleanup_errors), + }), + Err(panic) => resume_unwind(panic), + Ok(Ok(_)) => unreachable!(), + } + } + } + } +} +fn acquire_fence(path: &Path, deadline: Instant) -> Result { + let mut name = path + .file_name() + .ok_or(Error("unsafe_storage_path"))? + .to_os_string(); + name.push(".writer.lock"); + let file = OpenOptions::new() + .read(true) + .write(true) + .create(true) + .truncate(false) + .mode(0o600) + .custom_flags(libc::O_NOFOLLOW | libc::O_CLOEXEC | libc::O_NONBLOCK) + .open(path.with_file_name(name)) + .map_err(|_| Error("unsafe_writer_lock"))?; + let metadata = file.metadata().map_err(|_| Error("unsafe_writer_lock"))?; + if !metadata.is_file() || metadata.nlink() != 1 { + return Err(Error("unsafe_writer_lock")); + } + loop { + match file.try_lock_exclusive() { + Ok(()) => return Ok(file), + Err(e) if e.kind() == ErrorKind::Interrupted => {} + Err(e) if e.kind() == ErrorKind::WouldBlock => {} + Err(_) => return Err(Error("writer_lock_error")), + } + let remaining = deadline.saturating_duration_since(Instant::now()); + if remaining.is_zero() { + return Err(Error("server_busy")); + } + thread::sleep(remaining.min(Duration::from_millis(10))); + } +} +fn validate_write_schema(db: &Connection) -> Result<()> { + // Only known tables/columns may participate. This is not a schema migrator. + for (table, columns) in [ + ("resources", "id,type,name,parent,owner,grp,mode,generation,revision,state,created_at,modified_at,body"), + ("resource_tags", "resource_id,tag"), ("resource_path_aliases", "parent_id,name,resource_id"), + ("revisions", "id,resource_id,created_at,body"), + ("relations", "revision_id,source_id,type,target_id,target_revision,body"), + ("events", "seq,id,body"), ("audit", "seq,digest,previous,body"), + ] { + let exists: bool = db.query_row("SELECT EXISTS(SELECT 1 FROM sqlite_schema WHERE name=? AND type='table')", [table], |r| r.get(0)).map_err(database_error)?; + if !exists { return Err(Error("unsupported_storage_schema")); } + db.prepare(&format!("SELECT {columns} FROM {table} LIMIT 0")).map_err(|_| Error("unsupported_storage_schema"))?; + } + // Result uniqueness is a durability requirement, not an optional index. + let mut statement = db + .prepare("SELECT name,pk FROM pragma_table_info('results') WHERE pk>0 ORDER BY pk") + .map_err(database_error)?; + let keys = statement + .query_map([], |r| Ok((r.get::<_, String>(0)?, r.get::<_, i64>(1)?))) + .map_err(database_error)? + .collect::, _>>() + .map_err(database_error)?; + if keys != [("subject".into(), 1), ("request_id".into(), 2)] { + return Err(Error("unsupported_storage_schema")); + } + validate_constraints(db)?; + Ok(()) +} +fn validate_constraints(db: &Connection) -> Result<()> { + for (table, expected) in [ + ("resources", &["id"][..]), + ("revisions", &["id"]), + ("settings", &["key"]), + ("resource_tags", &["tag", "resource_id"]), + ("resource_path_aliases", &["parent_id", "name"]), + ("events", &["seq"]), + ("audit", &["seq"]), + ] { + let mut stmt = db + .prepare("SELECT name FROM pragma_table_info(?1) WHERE pk>0 ORDER BY pk") + .map_err(database_error)?; + let actual = stmt + .query_map([table], |r| r.get::<_, String>(0)) + .map_err(database_error)? + .collect::, _>>() + .map_err(database_error)?; + if actual != expected { + return Err(Error("unsupported_storage_schema")); + } + } + for (table, expected) in [ + ("resources", &["parent", "name"][..]), + ("events", &["id"]), + ("audit", &["digest"]), + ] { + let mut stmt = db + .prepare("SELECT name FROM pragma_index_list(?1) WHERE \"unique\"=1 AND partial=0") + .map_err(database_error)?; + let indexes = stmt + .query_map([table], |r| r.get::<_, String>(0)) + .map_err(database_error)? + .collect::, _>>() + .map_err(database_error)?; + let mut found = false; + for name in indexes { + let mut stmt = db + .prepare("SELECT name FROM pragma_index_info(?1) ORDER BY seqno") + .map_err(database_error)?; + let columns = stmt + .query_map([name], |r| r.get::<_, Option>(0)) + .map_err(database_error)? + .collect::, _>>() + .map_err(database_error)?; + found |= columns + .iter() + .map(|v| v.as_deref()) + .eq(expected.iter().map(|v| Some(*v))); + } + if !found { + return Err(Error("unsupported_storage_schema")); + } + } + for (name, expected) in [ + ("one_root", "createuniqueindexone_rootonresources((1))whereparentisnull"), + ("audit_no_update", "createtriggeraudit_no_updatebeforeupdateonauditbeginselectraise(abort,'append_only_audit');end"), + ("audit_no_delete", "createtriggeraudit_no_deletebeforedeleteonauditbeginselectraise(abort,'append_only_audit');end"), + ] { + let sql: Option = db.query_row("SELECT sql FROM sqlite_schema WHERE name=?1", [name], |r| r.get(0)).optional().map_err(database_error)?; + let normalized = sql.unwrap_or_default().chars().filter(|c| !c.is_ascii_whitespace()).collect::().to_ascii_lowercase(); + if normalized.trim_end_matches(';') != expected { return Err(Error("unsupported_storage_schema")); } + } + for (table, expected) in [ + ( + "resources", + &[("parent", "resources", "id", "NO ACTION")][..], + ), + ( + "resource_tags", + &[("resource_id", "resources", "id", "CASCADE")], + ), + ( + "resource_path_aliases", + &[ + ("parent_id", "resources", "id", "CASCADE"), + ("resource_id", "resources", "id", "CASCADE"), + ], + ), + ( + "revisions", + &[("resource_id", "resources", "id", "NO ACTION")], + ), + ( + "relations", + &[("revision_id", "revisions", "id", "CASCADE")], + ), + ] { + let mut stmt = db.prepare("SELECT \"from\",\"table\",\"to\",on_delete FROM pragma_foreign_key_list(?1) ORDER BY \"from\"").map_err(database_error)?; + let actual = stmt + .query_map([table], |r| { + Ok(( + r.get::<_, String>(0)?, + r.get::<_, String>(1)?, + r.get::<_, String>(2)?, + r.get::<_, String>(3)?, + )) + }) + .map_err(database_error)? + .collect::, _>>() + .map_err(database_error)?; + if !actual + .iter() + .map(|(a, b, c, d)| (a.as_str(), b.as_str(), c.as_str(), d.as_str())) + .eq(expected.iter().copied()) + { + return Err(Error("unsupported_storage_schema")); + } + } + Ok(()) +} +impl WriteSession<'_> { + fn commit_failed(&mut self, error: rusqlite::Error) -> Error { + if self.transaction.is_autocommit() { + self.access.commit_uncertain = true; + Error("commit_outcome_uncertain") + } else { + database_error(error) + } + } + fn mutate(&mut self, f: impl FnOnce(&mut Self) -> Result) -> Result { + self.active()?; + let result = f(self); + if let Err(error) = &result { + self.poison.set(Some(*error)); + } + result + } + pub fn on_rollback(&mut self, effect: impl FnOnce() -> Result<()> + 'static) -> Result<()> { + self.active()?; + self.access.effects.push(Box::new(effect)); + Ok(()) + } + fn compensate(&mut self, start: usize) { + for effect in self.access.effects.drain(start..).rev() { + match catch_unwind(AssertUnwindSafe(effect)) { + Ok(Ok(())) => {} + Ok(Err(error)) => self.access.cleanup_errors.push(error), + Err(_) => self + .access + .cleanup_errors + .push(Error("rollback_effect_panicked")), + } + } + } + /// A rejected inner unit may be handled ONLY after its changes and external + /// rollback effects are undone. A full SQLite abort poisons the outer unit. + pub fn savepoint(&mut self, f: impl FnOnce(&mut Self) -> Result) -> Result { + self.active()?; + self.access.savepoint = self + .access + .savepoint + .checked_add(1) + .ok_or(Error("savepoint_limit"))?; + let name = format!("msg_sp_{}", self.access.savepoint); + self.transaction + .execute_batch(&format!("SAVEPOINT {name}")) + .map_err(database_error)?; + let effects_start = self.access.effects.len(); + let cleanup_start = self.access.cleanup_errors.len(); + let result = catch_unwind(AssertUnwindSafe(|| { + let value = f(self)?; + self.active()?; + self.transaction + .execute_batch(&format!("RELEASE {name}")) + .map_err(database_error)?; + Ok(value) + })); + match result { + Ok(Ok(value)) => Ok(value), + other => { + let sql_rollback = if self.transaction.is_autocommit() { + Err(Error("transaction_aborted")) + } else { + self.transaction + .execute_batch(&format!("ROLLBACK TO {name}; RELEASE {name}")) + .map_err(database_error) + }; + if let Err(error) = sql_rollback { + self.access.cleanup_errors.push(error); + } + self.compensate(effects_start); + self.poison + .set(if self.access.cleanup_errors.len() == cleanup_start { + None + } else { + Some(Error("storage_recovery_required")) + }); + match other { + Ok(Err(error)) => Err(error), + Err(panic) => resume_unwind(panic), + Ok(Ok(_)) => unreachable!(), + } + } + } + } + pub fn set_setting(&mut self, key: &str, value: &Json) -> Result<()> { + self.mutate(|s| { + let raw = value.canonical()?; + if raw.len() > msg_core::MAX_BYTES { return Err(Error("storage_record_too_large")); } + s.transaction.execute("INSERT INTO settings(key,value) VALUES(?1,?2) ON CONFLICT(key) DO UPDATE SET value=excluded.value", params![key, raw]).map_err(database_error)?; + Ok(()) + }) + } + pub fn insert_resource(&mut self, resource: &Resource) -> Result<()> { + self.mutate(|s| { + resource.validate()?; + let body = records::encode(resource)?.canonical()?; + if let Some(parent) = &resource.parent { s.resource(parent)?; } + s.transaction.execute("INSERT INTO resources(id,type,name,parent,owner,grp,mode,generation,revision,state,created_at,modified_at,body) VALUES(?1,?2,?3,?4,?5,?6,?7,?8,?9,?10,?11,?12,?13)", + params![resource.id, resource.resource_type, resource.name, resource.parent, resource.owner, resource.group, resource.mode, resource.generation, resource.revision, resource.state.sql(), timestamp(resource.created_at)?, timestamp(resource.modified_at)?, body]).map_err(database_error)?; + s.insert_tags(resource) + }) + } + fn insert_tags(&self, resource: &Resource) -> Result<()> { + for tag in &resource.tags { + self.transaction + .execute( + "INSERT INTO resource_tags(resource_id,tag) VALUES(?1,?2)", + params![resource.id, tag], + ) + .map_err(database_error)?; + } + Ok(()) + } + pub fn replace_resource( + &mut self, + resource: &Resource, + expected_generation: i64, + ) -> Result<()> { + self.mutate(|s| { + resource.validate()?; + let old: Resource = records::decode(&s.record(Record::Resource, &resource.id)?)?; + if old.id != resource.id { return Err(Error("storage_record_mismatch")); } + if old.generation != expected_generation { return Err(Error("generation_conflict")); } + if expected_generation.checked_add(1) != Some(resource.generation) { return Err(Error("invalid_generation")); } + if old.resource_type != resource.resource_type || old.created_at != resource.created_at || old.created_by != resource.created_by { return Err(Error("immutable_creation_fact")); } + if let Some(parent) = &resource.parent { + if parent == &resource.id || s.ancestors(parent)?.iter().any(|a| a.id == resource.id) { return Err(Error("parent_cycle")); } + } + let body = records::encode(resource)?.canonical()?; + if old.parent != resource.parent || old.owner != resource.owner || old.group != resource.group || old.mode != resource.mode || old.state != resource.state { + let epoch = s.setting("authorization_epoch")?.map(|v| v.as_integer()?.parse::().map_err(|_| Error("invalid_authorization_epoch"))).transpose()?.unwrap_or(0); + let next = epoch.checked_add(1).filter(|_| epoch >= 0).ok_or(Error("invalid_authorization_epoch"))?; + s.set_setting("authorization_epoch", &Json::parse(&next.to_string())?)?; + } + if let Some(parent) = &old.parent { + if old.parent != resource.parent || old.name != resource.name { + s.transaction.execute("INSERT INTO resource_path_aliases(parent_id,name,resource_id) VALUES(?1,?2,?3) ON CONFLICT(parent_id,name) DO UPDATE SET resource_id=excluded.resource_id", params![parent, old.name, old.id]).map_err(database_error)?; + } + } + let changed = s.transaction.execute("UPDATE resources SET name=?1,parent=?2,owner=?3,grp=?4,mode=?5,generation=?6,revision=?7,state=?8,modified_at=?9,body=?10 WHERE id=?11 AND generation=?12", + params![resource.name,resource.parent,resource.owner,resource.group,resource.mode,resource.generation,resource.revision,resource.state.sql(),timestamp(resource.modified_at)?,body,resource.id,expected_generation]).map_err(database_error)?; + if changed != 1 { return Err(Error("generation_conflict")); } + if old.tags != resource.tags { + s.transaction.execute("DELETE FROM resource_tags WHERE resource_id=?1", [&resource.id]).map_err(database_error)?; + s.insert_tags(resource)?; + } + Ok(()) + }) + } + pub fn append_revision(&mut self, revision: &Revision) -> Result<()> { + self.mutate(|s| { + if revision.format_version < 1 { return Err(Error("invalid_version")); } + if revision.content.size < 0 { return Err(Error("invalid_nonnegative_integer")); } + for parent in &revision.parents { + let exists: bool = s.transaction.query_row("SELECT EXISTS(SELECT 1 FROM revisions WHERE id=?1 AND resource_id=?2)", params![parent, revision.resource_id], |r| r.get(0)).map_err(database_error)?; + if !exists { return Err(Error("revision_not_found")); } + } + s.transaction.execute("INSERT INTO revisions(id,resource_id,created_at,body) VALUES(?1,?2,?3,?4)", params![revision.id,revision.resource_id,timestamp(revision.created_at)?,records::encode(revision)?.canonical()?]).map_err(database_error)?; + for relation in &revision.relations { + if relation.excerpt.is_some_and(|[a,b]| a < 0 || a > b) { return Err(Error("invalid_byte_range")); } + s.transaction.execute("INSERT INTO relations(revision_id,source_id,type,target_id,target_revision,body) VALUES(?1,?2,?3,?4,?5,?6)", params![revision.id,revision.resource_id,relation.relation_type,relation.target.id,relation.target.revision,records::encode(relation)?.canonical()?]).map_err(database_error)?; + } + Ok(()) + }) + } + /// Storage-only port: the caller must authenticate/authorize the operation, + /// redact secret delivery data and produce its receipt before saving. + /// Neither this method nor a result lookup grants execution authority. + pub fn save_result( + &mut self, + subject: Option<&str>, + digest: &str, + result: &OperationResult, + ) -> Result<()> { + self.mutate(|s| { + if subject != result.subject.as_deref() || subject == Some("") { + return Err(Error("storage_record_mismatch")); + } + if !digest.strip_prefix("sha256:").is_some_and(|v| { + v.len() == 64 + && v.bytes() + .all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)) + }) { + return Err(Error("invalid_digest")); + } + let count: i64 = s + .transaction + .query_row("SELECT COUNT(*) FROM results", [], |r| r.get(0)) + .map_err(database_error)?; + if count >= 100_000 { + return Err(Error("storage_capacity_exceeded")); + } + s.transaction + .execute( + "INSERT INTO results(subject,request_id,digest,body) VALUES(?1,?2,?3,?4)", + params![ + subject.unwrap_or(""), + result.request_id, + digest, + records::encode(result)?.canonical()? + ], + ) + .map_err(database_error)?; + Ok(()) + }) + } + pub fn append_event(&mut self, event: &Event) -> Result<()> { + self.mutate(|s| { + s.transaction + .execute( + "INSERT INTO events(id,body) VALUES(?1,?2)", + params![event.id, records::encode(event)?.canonical()?], + ) + .map_err(database_error)?; + Ok(()) + }) + } + pub fn append_audit(&mut self, event: &AuditEvent) -> Result<()> { + self.mutate(|s| { + let previous: Option = s + .transaction + .query_row( + "SELECT digest FROM audit ORDER BY seq DESC LIMIT 1", + [], + |r| r.get(0), + ) + .optional() + .map_err(database_error)?; + let mut entry = event.clone(); + entry.previous_digest = previous; + entry.entry_digest.clear(); + let mut fields = records::encode(&entry)?.into_object()?; + fields.remove("entry_digest"); + entry.entry_digest = + msg_core::digest(Json::from_object(fields).canonical()?.as_bytes()); + s.transaction + .execute( + "INSERT INTO audit(digest,previous,body) VALUES(?1,?2,?3)", + params![ + entry.entry_digest, + entry.previous_digest, + records::encode(&entry)?.canonical()? + ], + ) + .map_err(database_error)?; + Ok(()) + }) + } +} +fn timestamp(value: msg_identity::models::Timestamp) -> Result { + Ok(records::encode(&value)?.as_str()?.to_owned()) +} + +#[cfg(test)] +mod tests; diff --git a/rust/crates/msg-storage/src/writer/tests.rs b/rust/crates/msg-storage/src/writer/tests.rs new file mode 100644 index 00000000..8c36381c --- /dev/null +++ b/rust/crates/msg-storage/src/writer/tests.rs @@ -0,0 +1,423 @@ +//! Focused failure injection; Python-created full schemas are tested separately. +use super::*; +use std::{ + cell::RefCell, + rc::Rc, + sync::atomic::{AtomicU64, Ordering}, + time::{SystemTime, UNIX_EPOCH}, +}; +static NEXT: AtomicU64 = AtomicU64::new(0); +struct Fixture(PathBuf); +impl Fixture { + fn new() -> Self { + let path = std::env::temp_dir().join(format!( + "msg-writer-{}-{}-{}", + std::process::id(), + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos(), + NEXT.fetch_add(1, Ordering::Relaxed) + )); + fs::create_dir(&path).unwrap(); + let fixture = Self(path); + let db = Connection::open(fixture.db()).unwrap(); + db.execute_batch("PRAGMA journal_mode=WAL; + CREATE TABLE schema_version(version INTEGER PRIMARY KEY); INSERT INTO schema_version VALUES(1); + CREATE TABLE settings(key TEXT PRIMARY KEY,value TEXT NOT NULL); + CREATE TABLE identities(id TEXT PRIMARY KEY,kind TEXT,body TEXT); + CREATE TABLE credentials(id TEXT PRIMARY KEY,subject TEXT,body TEXT); + CREATE TABLE certificates(id TEXT PRIMARY KEY,subject TEXT,parent TEXT,revoked INTEGER,body TEXT); + CREATE TABLE resources(id TEXT PRIMARY KEY,type TEXT,name TEXT,parent TEXT REFERENCES resources(id) DEFERRABLE INITIALLY DEFERRED,owner TEXT,grp TEXT,mode INTEGER,generation INTEGER,revision TEXT,state TEXT,created_at TEXT,modified_at TEXT,body TEXT,UNIQUE(parent,name)); + CREATE UNIQUE INDEX one_root ON resources((1)) WHERE parent IS NULL; + CREATE TABLE results(subject TEXT,request_id TEXT,digest TEXT,body TEXT,PRIMARY KEY(subject,request_id)); + CREATE TABLE credential_states(id TEXT PRIMARY KEY,kind TEXT,expires TEXT,body TEXT); + CREATE TABLE custodial_vault(subject TEXT PRIMARY KEY,signing_key_id TEXT,status TEXT); + CREATE TABLE token_deliveries(credential_id TEXT,subject TEXT,recovery_verifier TEXT,recovery_expires_at TEXT,consumed_at TEXT,request_id TEXT); + CREATE TABLE resource_tags(resource_id TEXT REFERENCES resources(id) ON DELETE CASCADE,tag TEXT,PRIMARY KEY(tag,resource_id)); + CREATE TABLE resource_path_aliases(parent_id TEXT REFERENCES resources(id) ON DELETE CASCADE,name TEXT,resource_id TEXT REFERENCES resources(id) ON DELETE CASCADE,PRIMARY KEY(parent_id,name)); + CREATE TABLE revisions(id TEXT PRIMARY KEY,resource_id TEXT REFERENCES resources(id),created_at TEXT,body TEXT); + CREATE TABLE relations(revision_id TEXT REFERENCES revisions(id) ON DELETE CASCADE,source_id TEXT,type TEXT,target_id TEXT,target_revision TEXT,body TEXT); + CREATE TABLE events(seq INTEGER PRIMARY KEY AUTOINCREMENT,id TEXT UNIQUE,body TEXT); + CREATE TABLE audit(seq INTEGER PRIMARY KEY AUTOINCREMENT,digest TEXT UNIQUE,previous TEXT,body TEXT); + CREATE TRIGGER audit_no_update BEFORE UPDATE ON audit BEGIN SELECT RAISE(ABORT,'append_only_audit'); END; + CREATE TRIGGER audit_no_delete BEFORE DELETE ON audit BEGIN SELECT RAISE(ABORT,'append_only_audit'); END; + ").unwrap(); + fixture + } + fn db(&self) -> PathBuf { + self.0.join("metadata.db") + } + fn writer(&self) -> SqliteWriter { + SqliteWriter::open_existing(&self.db(), Duration::from_millis(40)).unwrap() + } + fn setting(&self, key: &str) -> Option { + Connection::open(self.db()) + .unwrap() + .query_row("SELECT value FROM settings WHERE key=?", [key], |r| { + r.get(0) + }) + .optional() + .unwrap() + } +} +impl Drop for Fixture { + fn drop(&mut self) { + fs::remove_dir_all(&self.0).unwrap(); + } +} +fn value() -> Json { + Json::parse("1").unwrap() +} +fn effect(tx: &mut WriteSession<'_>, log: &Rc>>, n: i32) { + let log = Rc::clone(log); + tx.on_rollback(move || { + log.borrow_mut().push(n); + Ok(()) + }) + .unwrap(); +} +#[test] +fn commit_and_read_your_writes() { + let fixture = Fixture::new(); + let mut writer = fixture.writer(); + let got = writer + .transaction(|tx| { + tx.set_setting("x", &value())?; + tx.setting("x")?.unwrap().canonical() + }) + .unwrap(); + assert_eq!(got, "1"); + assert_eq!(fixture.setting("x").as_deref(), Some("1")); +} +#[test] +fn handler_failure_rolls_back_in_reverse_order() { + let fixture = Fixture::new(); + let mut writer = fixture.writer(); + let log = Rc::new(RefCell::new(Vec::new())); + let result: std::result::Result<(), _> = writer.transaction(|tx| { + effect(tx, &log, 1); + tx.set_setting("x", &value())?; + effect(tx, &log, 2); + Err(Error("fixture_failure")) + }); + assert_eq!(result.unwrap_err().cause, Error("fixture_failure")); + assert_eq!(*log.borrow(), [2, 1]); + assert!(fixture.setting("x").is_none()); + writer + .transaction(|tx| tx.set_setting("x", &value())) + .unwrap(); +} +#[test] +fn swallowed_sql_failure_cannot_commit_partial_work() { + let fixture = Fixture::new(); + let mut writer = fixture.writer(); + writer.connection.execute_batch("CREATE TRIGGER reject_bad BEFORE INSERT ON settings WHEN NEW.key='bad' BEGIN SELECT RAISE(ABORT,'fixture'); END;").unwrap(); + let error = writer + .transaction(|tx| { + tx.set_setting("x", &value())?; + let _ = tx.set_setting("bad", &value()); + Ok(()) + }) + .unwrap_err(); + assert_eq!(error.cause, Error("constraint_conflict")); + assert!(fixture.setting("x").is_none()); +} +#[test] +fn sqlite_full_abort_cannot_restart_in_autocommit() { + let fixture = Fixture::new(); + let mut writer = fixture.writer(); + writer.connection.execute_batch("CREATE TRIGGER reject_bad BEFORE INSERT ON settings WHEN NEW.key='bad' BEGIN SELECT RAISE(ROLLBACK,'fixture'); END;").unwrap(); + let error = writer + .transaction(|tx| { + tx.set_setting("x", &value())?; + assert!(tx.set_setting("bad", &value()).is_err()); + assert!(tx.set_setting("autocommit_escape", &value()).is_err()); + Ok(()) + }) + .unwrap_err(); + assert_eq!(error.cause, Error("constraint_conflict")); + assert!(fixture.setting("x").is_none()); + assert!(fixture.setting("autocommit_escape").is_none()); +} +#[test] +fn savepoint_failure_isolated_from_outer_work() { + let fixture = Fixture::new(); + let mut writer = fixture.writer(); + let log = Rc::new(RefCell::new(Vec::new())); + writer + .transaction(|tx| { + effect(tx, &log, 1); + tx.set_setting("outer", &value())?; + let result: Result<()> = tx.savepoint(|tx| { + tx.set_setting("inner", &value())?; + effect(tx, &log, 2); + Err(Error("fixture_failure")) + }); + assert!(result.is_err()); + assert_eq!(*log.borrow(), [2]); + assert!(tx.setting("inner")?.is_none()); + Ok(()) + }) + .unwrap(); + assert_eq!(*log.borrow(), [2]); + assert!(fixture.setting("inner").is_none()); + assert!(fixture.setting("outer").is_some()); +} +#[test] +fn released_inner_effects_remain_until_outer_commit() { + let fixture = Fixture::new(); + let mut writer = fixture.writer(); + let log = Rc::new(RefCell::new(Vec::new())); + let _: std::result::Result<(), _> = writer.transaction(|tx| { + effect(tx, &log, 1); + tx.savepoint(|tx| { + effect(tx, &log, 2); + tx.savepoint(|tx| { + effect(tx, &log, 3); + Ok(()) + }) + })?; + Err(Error("fixture_failure")) + }); + assert_eq!(*log.borrow(), [3, 2, 1]); +} +#[test] +fn ignored_inner_mutation_failure_is_rolled_back() { + let fixture = Fixture::new(); + let mut writer = fixture.writer(); + writer.connection.execute_batch("CREATE TRIGGER reject_bad BEFORE INSERT ON settings WHEN NEW.key='bad' BEGIN SELECT RAISE(ABORT,'fixture'); END;").unwrap(); + writer + .transaction(|tx| { + tx.set_setting("outer", &value())?; + assert!(tx + .savepoint(|tx| { + tx.set_setting("inner", &value())?; + let _ = tx.set_setting("bad", &value()); + Ok(()) + }) + .is_err()); + tx.set_setting("after", &value()) + }) + .unwrap(); + assert!(fixture.setting("inner").is_none()); + assert!(fixture.setting("after").is_some()); +} +#[test] +fn deferred_commit_failure_returns_no_success_and_compensates() { + let fixture = Fixture::new(); + let mut writer = fixture.writer(); + let log = Rc::new(RefCell::new(Vec::new())); + let result = writer.transaction(|tx| { + tx.set_setting("x", &value())?; + effect(tx, &log, 1); + // Test-only injection: callers have no public SQL/transaction handle. + tx.transaction + .execute( + "INSERT INTO resources(id,parent) VALUES('orphan','missing')", + [], + ) + .map_err(database_error)?; + Ok("must_not_escape") + }); + assert_eq!(result.unwrap_err().cause, Error("constraint_conflict")); + assert!(fixture.setting("x").is_none()); + assert_eq!(*log.borrow(), [1]); +} +#[test] +fn outer_panic_rolls_back_and_runs_all_effects() { + let fixture = Fixture::new(); + let mut writer = fixture.writer(); + let log = Rc::new(RefCell::new(Vec::new())); + assert!(catch_unwind(AssertUnwindSafe(|| { + let _: std::result::Result<(), _> = writer.transaction(|tx| { + tx.set_setting("x", &value())?; + effect(tx, &log, 1); + panic!("fixture panic") + }); + })) + .is_err()); + assert!(fixture.setting("x").is_none()); + assert_eq!(*log.borrow(), [1]); + writer + .transaction(|tx| tx.set_setting("after", &value())) + .unwrap(); +} +#[test] +fn inner_panic_can_be_caught_only_after_rollback() { + let fixture = Fixture::new(); + let mut writer = fixture.writer(); + writer + .transaction(|tx| { + assert!(catch_unwind(AssertUnwindSafe(|| { + let _: Result<()> = tx.savepoint(|tx| { + tx.set_setting("inner", &value())?; + panic!("fixture panic") + }); + })) + .is_err()); + tx.set_setting("after", &value()) + }) + .unwrap(); + assert!(fixture.setting("inner").is_none()); + assert!(fixture.setting("after").is_some()); +} +#[test] +fn cleanup_failure_preserves_cause_continues_and_disables_writer() { + let fixture = Fixture::new(); + let mut writer = fixture.writer(); + let log = Rc::new(RefCell::new(Vec::new())); + let result: std::result::Result<(), _> = writer.transaction(|tx| { + effect(tx, &log, 1); + tx.on_rollback(|| Err(Error("cleanup_failed")))?; + effect(tx, &log, 2); + tx.on_rollback(|| panic!("cleanup panic"))?; + Err(Error("primary")) + }); + let error = result.unwrap_err(); + assert_eq!(error.cause, Error("primary")); + assert_eq!( + error.cleanup_errors, + [Error("rollback_effect_panicked"), Error("cleanup_failed")] + ); + assert_eq!(*log.borrow(), [2, 1]); + assert_eq!( + writer.transaction(|_| Ok(())).unwrap_err().cause, + Error("storage_recovery_required") + ); +} +#[test] +fn writer_fence_covers_compensation_after_sqlite_abort() { + let fixture = Fixture::new(); + let mut writer = fixture.writer(); + writer.connection.execute_batch("CREATE TRIGGER reject_bad BEFORE INSERT ON settings WHEN NEW.key='bad' BEGIN SELECT RAISE(ROLLBACK,'fixture'); END;").unwrap(); + let other_path = fixture.db(); + let result = writer.transaction(|tx| { + tx.on_rollback(move || { + let mut other = SqliteWriter::open_existing(&other_path, Duration::ZERO)?; + assert_eq!( + other + .transaction(|tx| tx.set_setting("interleaved", &value())) + .unwrap_err() + .cause, + Error("server_busy") + ); + Ok(()) + })?; + tx.set_setting("bad", &value()) + }); + assert!(result.is_err()); + assert!(fixture.setting("interleaved").is_none()); + fixture + .writer() + .transaction(|tx| tx.set_setting("after", &value())) + .unwrap(); +} +#[test] +fn nested_full_abort_never_commits_outer_work() { + let fixture = Fixture::new(); + let mut writer = fixture.writer(); + writer.connection.execute_batch("CREATE TRIGGER reject_bad BEFORE INSERT ON settings WHEN NEW.key='bad' BEGIN SELECT RAISE(ROLLBACK,'fixture'); END;").unwrap(); + let result = writer.transaction(|tx| { + tx.set_setting("outer", &value())?; + let _ = tx.savepoint(|tx| tx.set_setting("bad", &value())); + assert!(tx.set_setting("escape", &value()).is_err()); + Ok(()) + }); + assert!(result.is_err()); + assert!(fixture.setting("outer").is_none()); + assert!(fixture.setting("escape").is_none()); +} +#[test] +fn missing_bad_journal_or_schema_is_not_repaired() { + let fixture = Fixture::new(); + assert!(SqliteWriter::open_existing(&fixture.0.join("missing"), Duration::ZERO).is_err()); + assert!(!fixture.0.join("missing").exists()); + let db = Connection::open(fixture.db()).unwrap(); + db.execute_batch("PRAGMA journal_mode=DELETE").unwrap(); + drop(db); + assert!(matches!( + SqliteWriter::open_existing(&fixture.db(), Duration::ZERO), + Err(Error("unsupported_storage_journal")) + )); + let db = Connection::open(fixture.db()).unwrap(); + db.execute_batch("PRAGMA journal_mode=WAL; UPDATE schema_version SET version=99;") + .unwrap(); + drop(db); + assert!(matches!( + SqliteWriter::open_existing(&fixture.db(), Duration::ZERO), + Err(Error("unsupported_storage_schema")) + )); +} +#[test] +fn lock_symlink_and_file_aliases_are_rejected() { + let fixture = Fixture::new(); + let mut writer = fixture.writer(); + let target = fixture.0.join("target"); + fs::write(&target, "keep").unwrap(); + std::os::unix::fs::symlink(&target, fixture.0.join("metadata.db.writer.lock")).unwrap(); + assert_eq!( + writer.transaction(|_| Ok(())).unwrap_err().cause, + Error("unsafe_writer_lock") + ); + assert_eq!(fs::read_to_string(target).unwrap(), "keep"); + let link = fixture.0.join("alias"); + std::os::unix::fs::symlink(fixture.db(), &link).unwrap(); + assert!(matches!( + SqliteWriter::open_existing(&link, Duration::ZERO), + Err(Error("unsafe_storage_path")) + )); + fs::hard_link(fixture.db(), fixture.0.join("hardlink")).unwrap(); + assert!(matches!( + SqliteWriter::open_existing(&fixture.db(), Duration::ZERO), + Err(Error("unsafe_storage_path")) + )); +} +#[test] +fn replaced_database_and_excessive_timeout_fail_closed() { + let fixture = Fixture::new(); + let mut writer = fixture.writer(); + assert!(matches!( + SqliteWriter::open_existing(&fixture.db(), Duration::MAX), + Err(Error("invalid_timeout")) + )); + fs::rename(fixture.db(), fixture.0.join("old.db")).unwrap(); + let replacement = Connection::open(fixture.db()).unwrap(); + drop(replacement); + assert_eq!( + writer.transaction(|_| Ok(())).unwrap_err().cause, + Error("storage_recovery_required") + ); +} + +#[test] +fn uncertain_commit_does_not_destroy_possibly_committed_external_data() { + let fixture = Fixture::new(); + let mut writer = fixture.writer(); + let log = Rc::new(RefCell::new(Vec::new())); + let result: std::result::Result<(), _> = writer.transaction(|tx| { + tx.set_setting("committed", &value())?; + effect(tx, &log, 1); + // Deterministic commit-result injection, not a simulated physical disk: + // commit really persisted, but the driver reports an I/O failure. + tx.transaction.execute_batch("COMMIT").unwrap(); + Err(tx.commit_failed(rusqlite::Error::SqliteFailure( + rusqlite::ffi::Error::new(rusqlite::ffi::SQLITE_IOERR), + None, + ))) + }); + assert_eq!(result.unwrap_err().cause, Error("commit_outcome_uncertain")); + assert!(log.borrow().is_empty()); + assert!(fixture.setting("committed").is_some()); + assert_eq!( + writer.transaction(|_| Ok(())).unwrap_err().cause, + Error("storage_recovery_required") + ); +} +#[test] +fn missing_uniqueness_or_audit_guard_is_not_repaired() { + for sql in ["DROP INDEX one_root", "DROP TRIGGER audit_no_delete", "DROP TABLE results; CREATE TABLE results(subject TEXT,request_id TEXT,digest TEXT,body TEXT)"] { + let fixture = Fixture::new(); let db = Connection::open(fixture.db()).unwrap(); db.execute_batch(sql).unwrap(); drop(db); + assert!(matches!(SqliteWriter::open_existing(&fixture.db(), Duration::ZERO), Err(Error("unsupported_storage_schema")))); + } +} diff --git a/rust/tests/postgres_fixture.py b/rust/tests/postgres_fixture.py new file mode 100644 index 00000000..f60bd8f7 --- /dev/null +++ b/rust/tests/postgres_fixture.py @@ -0,0 +1,86 @@ +"""A disposable, Python-created PostgreSQL database for native conformance tests. + +Requires explicit MSG_PG_TEST_ADMIN_DSN. Only a freshly created, unpredictable +msg_rust_test_* database is populated and removed. Never truncates an existing DB. +""" + +from __future__ import annotations + +import os +import uuid +from contextlib import contextmanager + +import psycopg +from psycopg import sql +from psycopg.conninfo import conninfo_to_dict, make_conninfo + +from msg.storage.postgres import PostgresMetadataStore, _postgres_sql + + +class Connection: + def __init__(self, dsn): + self.connection = psycopg.connect(dsn, autocommit=True) + + def execute(self, statement, parameters=()): + return self.connection.execute( + _postgres_sql(statement, has_parameters=bool(parameters)), parameters or None + ) + + def close(self): + self.connection.close() + + +@contextmanager +def database(): + admin = os.environ.get('MSG_PG_TEST_ADMIN_DSN') + if not admin: + raise RuntimeError('MSG_PG_TEST_ADMIN_DSN is required for isolated PostgreSQL tests') + config = conninfo_to_dict(admin) + host = config.get('host', '') + if host and not (host.startswith('/') or host in {'127.0.0.1', 'localhost', '::1'}): + raise RuntimeError('PostgreSQL conformance only connects to a local test instance') + name = 'msg_rust_test_' + uuid.uuid4().hex + with psycopg.connect(admin, autocommit=True) as connection: + connection.execute(sql.SQL('CREATE DATABASE {}').format(sql.Identifier(name))) + dsn = make_conninfo(admin, dbname=name) + old = os.environ.get('MSG_PARITY_POSTGRES_DSN') + try: + os.environ['MSG_PARITY_POSTGRES_DSN'] = dsn + yield dsn + finally: + if old is None: + os.environ.pop('MSG_PARITY_POSTGRES_DSN', None) + else: + os.environ['MSG_PARITY_POSTGRES_DSN'] = old + with psycopg.connect(admin, autocommit=True) as connection: + connection.execute(sql.SQL('DROP DATABASE {}').format(sql.Identifier(name))) + + +async def replace_fixture_storage(fixture, dsn): + """Copy only deterministic public test records, then use real PG for both sides.""" + metadata = PostgresMetadataStore(dsn) + with psycopg.connect(dsn) as connection: + for table in ( + 'resources', + 'identities', + 'credentials', + 'certificates', + 'settings', + 'memberships', + 'dm_conversations', + ): + records = fixture.conn.execute('SELECT * FROM ' + table) + columns = [entry[0] for entry in records.description] + query = sql.SQL('INSERT INTO {} ({}) VALUES ({})').format( + sql.Identifier(table), + sql.SQL(',').join(map(sql.Identifier, columns)), + sql.SQL(',').join(sql.Placeholder() for _ in columns), + ) + if table == 'settings': + query += sql.SQL(' ON CONFLICT(key) DO UPDATE SET value=excluded.value') + for row in records: + connection.execute(query, row) + fixture.conn.close() + await fixture.metadata.close() + fixture.conn = Connection(dsn) + fixture.metadata = metadata diff --git a/rust/tests/postgres_transport_fixture.py b/rust/tests/postgres_transport_fixture.py new file mode 100644 index 00000000..d4620f84 --- /dev/null +++ b/rust/tests/postgres_transport_fixture.py @@ -0,0 +1,135 @@ +"""One local-test-only PostgreSQL proxy that loses a committed acknowledgement. + +The target must be an explicitly supplied loopback or Unix socket test instance. +It never sends authentication data anywhere other than that local test server. +""" + +from __future__ import annotations + +import socket +import threading +from contextlib import AbstractContextManager + +from psycopg.conninfo import conninfo_to_dict, make_conninfo + + +def exact(sock, n): + data = bytearray() + while len(data) < n: + block = sock.recv(n - len(data)) + if not block: + raise EOFError + data.extend(block) + return bytes(data) + + +class LoseCommitAck(AbstractContextManager): + def __init__(self, dsn): + self.config = conninfo_to_dict(dsn) + if not self.config.get('dbname', '').startswith('msg_rust_test_'): + raise ValueError('disposable database required') + host = self.config.get('host', '') + if not (host.startswith('/') or host in {'localhost', '127.0.0.1', '::1'}): + raise ValueError('local test host required') + self.listener = socket.socket() + self.listener.bind(('127.0.0.1', 0)) + self.listener.listen(8) + self.listener.settimeout(0.2) + self.dsn = make_conninfo( + dsn, host='127.0.0.1', port=self.listener.getsockname()[1], sslmode='disable' + ) + self.closed = threading.Event() + self.dropped = threading.Event() + self.sockets = [] + self.threads = [] + + def __enter__(self): + thread = threading.Thread(target=self.accept, daemon=True) + self.threads.append(thread) + thread.start() + return self + + def accept(self): + while not self.closed.is_set(): + try: + client, _ = self.listener.accept() + except TimeoutError: + continue + except OSError: + return + self.sockets.append(client) + thread = threading.Thread(target=self.connection, args=(client,), daemon=True) + self.threads.append(thread) + thread.start() + + def connection(self, client): + backend = None + try: + host = self.config['host'] + port = int(self.config.get('port', 5432)) + if host.startswith('/'): + backend = socket.socket(socket.AF_UNIX) + backend.connect(host + '/.s.PGSQL.' + str(port)) + else: + backend = socket.create_connection((host, port), timeout=5) + self.sockets.append(backend) + size = exact(client, 4) + n = int.from_bytes(size, 'big') + backend.sendall(size + exact(client, n - 4)) + wrote = threading.Event() + dropping = threading.Event() + + def responses(): + try: + while not self.closed.is_set(): + tag = exact(backend, 1) + size = exact(backend, 4) + n = int.from_bytes(size, 'big') + if not 4 <= n <= 2**24: + raise ValueError('test frame size') + body = exact(backend, n - 4) + if dropping.is_set(): + if tag == b'Z': + self.dropped.set() + client.shutdown(socket.SHUT_RDWR) + return + else: + client.sendall(tag + size + body) + except EOFError, OSError, ValueError: + pass + + reader = threading.Thread(target=responses, daemon=True) + self.threads.append(reader) + reader.start() + while not self.closed.is_set(): + tag = exact(client, 1) + size = exact(client, 4) + n = int.from_bytes(size, 'big') + if not 4 <= n <= 2**24: + raise ValueError('test frame size') + body = exact(client, n - 4) + if tag == b'P': + statement = body.split(b'\0', 2)[1].lstrip().upper() + if statement.startswith((b'INSERT ', b'UPDATE ', b'DELETE ')): + wrote.set() + if tag == b'Q' and body.rstrip(b'\0; \n').upper() == b'COMMIT' and wrote.is_set(): + dropping.set() + backend.sendall(tag + size + body) + except EOFError, OSError, ValueError: + pass + finally: + client.close() + if backend is not None: + backend.close() + + def __exit__(self, *_): + self.closed.set() + self.listener.close() + for sock in self.sockets: + try: + sock.shutdown(socket.SHUT_RDWR) + except OSError: + pass + sock.close() + for thread in self.threads: + thread.join(timeout=1) diff --git a/rust/tests/test_authorization_parity.py b/rust/tests/test_authorization_parity.py new file mode 100644 index 00000000..6ca8df8d --- /dev/null +++ b/rust/tests/test_authorization_parity.py @@ -0,0 +1,640 @@ +"""Exercise complete resource authorization against Python, using public fixtures. + +The helper authenticates every request before authorization. It never accepts a +serialized Principal, and no test publishes data or reaches a production server. +""" + +from __future__ import annotations + +import argparse +import asyncio +import json +import tempfile +from contextlib import contextmanager +from dataclasses import replace +from datetime import timedelta +from pathlib import Path +from types import SimpleNamespace + +from test_identity_parity import NOW, READ, WRITE, Check, Fixture + +from msg.core.codec import canonical, wire +from msg.core.errors import Failure +from msg.core.models import AccessRequirement, ResourceTypeSpec +from msg.security.authorization import AuthorizationService +from msg.security.capabilities import BASE_FAMILIES + +EXTRA = frozenset({ + 'communication.dm_send@1', + 'communication.dm_send@2', + 'query.save@1', + 'query.saved_archive@1', + 'sharing.link_create@1', + 'sharing.grant@2', + 'sharing.revoke@2', + 'identity.note_put@2', + 'identity.personal_put@2', + 'identity.todo_put@2', + 'identity.legacy_put@2', + 'content.archive@1', + 'content.move@1', + 'content.chmod@1', + 'content.topic_create@1', + 'content.file_put@1', + 'wiki.page_put@1', +}) + + +class PolicyFixture(Fixture): + def __init__(self, path): + super().__init__(path) + # Trusted test configuration, not part of a request. Use the real names + # of the built-in families; no synthetic bypass to ordinary checks. + self.primary = tuple(replace(g, operations=g.operations | EXTRA) for g in self.primary) + for key, spec in list(self.registry._capabilities.items()): + self.registry._capabilities[key] = replace(spec, operations=spec.operations | EXTRA) + for name, cred in self.credentials.items(): + cred = replace(cred, ceiling=self.primary) + self.credentials[name] = cred + self.conn.execute( + 'UPDATE credentials SET body=? WHERE id=?', (canonical(cred).decode(), cred.id) + ) + for kind in ( + 'space', + 'topic', + 'file', + 'post', + 'user', + 'organization', + 'tool', + 'csr', + 'certificate', + 'credential', + 'legacy_directive', + 'dm_conversation', + 'watch', + 'saved_query', + 'delegation', + 'todo', + ): + self.registry.add_resource_type( + ResourceTypeSpec( + name=kind, + version=1, + container=kind in {'space', 'topic', 'user', 'organization', 'dm_conversation'}, + content_schema=None, + operations=frozenset(), + relations=frozenset(), + ) + ) + self.policy = AuthorizationService(self.registry, self.validator) + self.put_resource('r_user', parent='r_root', kind='user', owner='alice') + self.put_resource('r_soul', parent='r_user', name='SOUL.md', owner='alice') + self.put_resource('r_notes', parent='r_user', name='notes', kind='topic', owner='alice') + self.put_resource('r_note', parent='r_notes', name='note.md', owner='alice') + self.put_resource('r_todos', parent='r_user', name='todos', kind='topic', owner='alice') + self.put_resource('r_todo', parent='r_todos', kind='todo', owner='alice') + self.put_resource('t_last_will', parent='r_root', kind='topic') + self.put_resource('r_legacy', parent='t_last_will', kind='legacy_directive', owner='alice') + self.put_resource('t_wiki', parent='r_root', kind='topic') + self.put_resource('r_wiki', parent='t_wiki', owner='alice') + self.put_resource('r_agents', parent='r_root', owner='alice') + self.put_resource('r_rules', parent='r_root', owner='alice') + self.put_resource('r_watch', parent='r_area', kind='watch', owner='alice') + self.put_resource('r_query', parent='r_area', kind='saved_query', owner='alice') + self.put_resource('r_about', parent='r_area', name='ABOUT.md', owner='alice') + self.put_resource('r_header', parent='r_area', name='HEADER.svg', owner='alice') + self.put_resource('t_tools', parent='r_root', kind='topic', mode=0o700) + self.put_resource('r_tool', parent='t_tools', kind='tool', mode=0o700) + self.put_resource('r_dm', parent='r_root', kind='topic', mode=0o777) + self.put_resource('r_message', parent='r_dm', kind='post', owner='alice', mode=0o777) + self.put_resource('g_test', parent='r_root', kind='organization') + self.conn.execute( + 'INSERT INTO identities VALUES (?,?,?,?)', + ( + 'g_test', + 'organization', + 0, + canonical({'resource_id': 'g_test', 'membership_version': 1}).decode(), + ), + ) + for member in ('alice', 'bob'): + self.conn.execute( + 'INSERT INTO memberships VALUES (?,?,?,?)', + ( + 'g_test', + self.users[member], + 1, + canonical({ + 'organization_id': 'g_test', + 'subject_id': self.users[member], + 'role': 'member', + 'version': 1, + }).decode(), + ), + ) + self.conn.execute( + """INSERT INTO dm_conversations + (pair,resource_id,participant_a,participant_b,initiator,conversation_kind,state,created_at,updated_at) + VALUES (?,?,?,?,?,'direct','active',?,?)""", + ( + 'pair_fixture', + 'r_dm', + *sorted((self.users['alice'], self.users['bob'])), + self.users['alice'], + wire(NOW), + wire(NOW), + ), + ) + + def put_resource(self, id, *, parent='r_area', kind='file', name=None, owner='bob', mode=0o755): + resource = replace( + self.resources['r_file'], + id=id, + parent=parent, + type=kind, + name=name or id, + owner=self.users.get(owner, owner), + mode=mode, + revision='v_' + id, + ) + self.resources[id] = resource + self.conn.execute( + 'INSERT INTO resources VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?)', + ( + id, + kind, + resource.name, + parent, + resource.owner, + resource.group, + mode, + 0, + resource.revision, + 'active', + wire(NOW), + wire(NOW), + canonical(resource).decode(), + ), + ) + return resource + + def context(self): + return { + **super().context(), + 'resource_types': wire(self.registry.resource_types()), + 'base_families': list(BASE_FAMILIES), + } + + @contextmanager + def resource_change(self, id, **fields): + with self.body_change('resources', id, replace(self.resources[id], **fields)): + yield + + @contextmanager + def row_change(self, table, key, updates): + # Table, columns, and predicates are test-owned constants, not request data. + where = ' AND '.join(f'{k}=?' for k in key) + columns = ','.join(updates) + old = self.conn.execute( + f'SELECT {columns} FROM {table} WHERE {where}', tuple(key.values()) + ).fetchone() + assert old is not None + assign = ','.join(f'{k}=?' for k in updates) + self.conn.execute( + f'UPDATE {table} SET {assign} WHERE {where}', (*updates.values(), *key.values()) + ) + try: + yield + finally: + self.conn.execute(f'UPDATE {table} SET {assign} WHERE {where}', (*old, *key.values())) + + +class PolicyCheck(Check): + async def require_case( + self, + group, + *, + id='r_file', + check='read', + actor='alice', + operation=READ + '@1', + certificates=(), + name=None, + checks=None, + entry='network', + delegated=False, + ): + f = self.fixture + opname, version = operation.rsplit('@', 1) + policy = f.spec(opname, version=int(version), entries=('network', 'local_admin', 'worker')) + request = f.request( + name=opname, + version=int(version), + actor=actor, + anonymous=actor is None, + subject=None if actor is None else f.users['bob'] if delegated else 'default', + certificates=certificates, + arguments={} if name is None else {'name': name}, + ) + requirements = checks or [ + AccessRequirement(resource_id=id, operation=operation, check=check) + ] + async with f.metadata.transaction(write=False) as tx: + try: + principal = await f.auth.authenticate(request, tx, entry=entry) + context = SimpleNamespace(principal=principal, now=f.now, entry=entry) + refs = await f.policy.require(context, request, requirements, tx) + expected = {'ok': True, 'data': {'refs': wire(refs)}} + except Failure as exc: + expected = {'ok': False, 'code': exc.code} + actual = self.native({ + 'action': 'authorize', + 'raw': canonical(request).decode(), + 'policy': policy, + 'entry': entry, + 'now': wire(f.now), + 'checks': wire(requirements), + 'request_name': name, + }) + self.compare(group, actual, expected) + return actual + + async def source_case(self, group, id='r_file', grant='s_main', subject='alice', reshare=False): + f = self.fixture + async with f.metadata.transaction(write=False) as tx: + try: + resource = await tx.resource(id) + result = await f.policy.share_source_active( + resource, grant, f.users.get(subject, subject), f.now, tx, reshare=reshare + ) + expected = {'ok': True, 'data': {'allowed': result}} + except Failure as exc: + expected = {'ok': False, 'code': exc.code} + self.compare( + group, + self.native({ + 'action': 'share_source', + 'id': id, + 'grant': grant, + 'subject': f.users.get(subject, subject), + 'now': wire(f.now), + 'reshare': reshare, + }), + expected, + ) + + async def link_case(self, group, id='r_file', grantor='bob', credential=None): + f = self.fixture + credential = credential or f.keys[grantor] + async with f.metadata.transaction(write=False) as tx: + try: + r = await tx.resource(id) + result = await f.policy.share_link_read_allowed( + f.users[grantor], credential, r, (*await tx.ancestors(id), r), f.now, tx + ) + expected = {'ok': True, 'data': {'allowed': result}} + except Failure as exc: + expected = {'ok': False, 'code': exc.code} + self.compare( + group, + self.native({ + 'action': 'share_link', + 'id': id, + 'grantor': f.users[grantor], + 'credential': credential, + 'now': wire(f.now), + }), + expected, + ) + + +def insert_share( + f, + id='s_main', + *, + resource='r_file', + grantor='bob', + grantee='alice', + parent=None, + kind='user', + reshare=True, +): + f.conn.execute( + """INSERT INTO share_grants_v2 VALUES (?,?,?,?,?,?,?,?,?,?,?,?)""", + ( + id, + resource, + f.users.get(grantor, grantor), + f.users.get(grantee, grantee), + kind, + parent, + '["read"]', + '{}', + int(reshare), + wire(NOW - timedelta(minutes=1)), + wire(NOW + timedelta(hours=1)), + None, + ), + ) + + +async def exercise(f, c): + for actor in ('alice', 'bob', None): + for mode in ( + 0o000, + 0o100, + 0o400, + 0o600, + 0o640, + 0o700, + 0o750, + 0o755, + 0o777, + 0o1777, + 0o2777, + 0o4777, + ): + with f.resource_change('r_file', mode=mode): + for check in ( + 'read', + 'list', + 'traverse', + 'write', + 'create', + 'remove', + 'chmod', + 'chgrp', + 'chown', + 'manage', + 'certgate', + 'purge', + 'tool_use', + ): + await c.require_case('resource_policy', actor=actor, check=check) + for group_kind in ('active', 'pending', 'invited', 'rejected'): + body = { + 'organization_id': 'g_test', + 'subject_id': f.users['alice'], + 'role': 'member', + 'version': 1, + 'status': group_kind, + } + with f.row_change( + 'memberships', + {'org': 'g_test', 'subject': f.users['alice']}, + {'body': canonical(body).decode()}, + ): + async with f.metadata.transaction(write=False) as tx: + expected = { + 'ok': True, + 'data': { + 'groups': sorted( + m.organization_id for m in await tx.memberships(f.users['alice']) + ) + }, + } + c.compare( + 'memberships', + c.native({'action': 'memberships', 'subject': f.users['alice']}), + expected, + ) + with f.resource_change('r_file', mode=0o040): + await c.require_case('group_mode') + for parentmode in (0, 0o100, 0o400, 0o700, 0o710, 0o755): + with f.resource_change('r_area', mode=parentmode): + for actor in ('alice', 'bob', None): + await c.require_case('ancestor_traversal', actor=actor) + for state in ('archived', 'purged'): + with f.resource_change('r_area', state=state): + await c.require_case('inactive_ancestor') + for id in ( + 'r_watch', + 'r_query', + 'r_soul', + 'r_note', + 'r_todo', + 'r_legacy', + 'r_wiki', + 'r_agents', + 'r_rules', + ): + for actor in ('alice', 'bob', None): + for check in ('read', 'write', 'manage'): + await c.require_case('managed_resources', id=id, actor=actor, check=check) + for id, operations in { + 'r_query': ('query.save@1', 'query.saved_archive@1'), + 'r_soul': ('identity.personal_put@2', 'sharing.grant@2'), + 'r_note': ('identity.note_put@2', 'sharing.grant@2'), + 'r_todo': ('identity.todo_put@2', 'sharing.grant@2'), + 'r_legacy': ('identity.legacy_put@2', 'content.archive@1'), + 'r_wiki': ('content.archive@1', 'content.move@1', 'content.chmod@1', 'wiki.page_put@1'), + }.items(): + for op in operations: + for actor in ('alice', 'bob'): + await c.require_case( + 'dedicated_operations', id=id, operation=op, actor=actor, check='write' + ) + for resource in ('r_about', 'r_header', 'r_area'): + for actor in ('alice', 'bob', None): + for check in ('read', 'write', 'create', 'manage'): + await c.require_case( + 'topic_presentation', id=resource, actor=actor, check=check, name='ABOUT.md' + ) + f.conn.execute( + """INSERT INTO topic_memberships(topic,subject,role,status,joined_at,invited_by) + VALUES (?,?,'admin','active',?,?)""", + ('r_area', f.users['alice'], wire(NOW), None), + ) + for id in ('r_about', 'r_header', 'r_area'): + await c.require_case('topic_admin', id=id, check='write') + for expires in ( + None, + wire(NOW - timedelta(seconds=1)), + wire(NOW), + wire(NOW + timedelta(seconds=1)), + ): + f.conn.execute( + """INSERT INTO topic_bans(topic,subject,actor,created_at,expires_at,reason,status) + VALUES (?,?,?,?,?,'test','active') ON CONFLICT(topic,subject) DO UPDATE SET expires_at=excluded.expires_at""", + ('r_area', f.users['alice'], f.users['bob'], wire(NOW), expires), + ) + await c.require_case('topic_bans', id='r_about', check='write') + f.conn.execute('DELETE FROM topic_bans') + for prefix, id in (('hosting_preview:', 'r_area'), ('hosting_preview_file:', 'r_file')): + for marker in (False, {}, {'owner': f.users['alice']}, {'owner': f.users['bob']}): + with f.setting_change(prefix + id, marker): + for actor in ('alice', 'bob', None): + await c.require_case('hosting_preview', actor=actor) + for actor in ('alice', 'bob', 'ca', None): + for check in ('read', 'list', 'traverse', 'write', 'create', 'manage', 'chmod'): + await c.require_case('dm_participants', id='r_dm', actor=actor, check=check) + await c.require_case('dm_messages', id='r_message', actor=actor, check=check) + for state in ('active', 'pending', 'rejected'): + with f.row_change('dm_conversations', {'resource_id': 'r_dm'}, {'state': state}): + for actor in ('alice', 'bob'): + await c.require_case( + 'dm_send', + id='r_dm', + actor=actor, + check='create', + operation='communication.dm_send@2', + ) + await c.require_case( + 'dm_edit', id='r_message', actor=actor, check='write', operation=WRITE + '@1' + ) + f.conn.execute('INSERT INTO dm_blocks VALUES (?,?)', (f.users['alice'], f.users['bob'])) + for actor in ('alice', 'bob'): + await c.require_case( + 'dm_blocked', + id='r_dm', + actor=actor, + check='create', + operation='communication.dm_send@2', + ) + await c.require_case( + 'dm_blocked', id='r_message', actor=actor, check='write', operation=WRITE + '@1' + ) + f.conn.execute('DELETE FROM dm_blocks') + for mode in (0o2000, 0o2777, 0o6755): + with f.resource_change('r_area', mode=mode): + for check in ('read', 'write', 'create', 'manage'): + await c.require_case('certificate_gate', id='r_file', check=check, actor='bob') + for actor in ('alice', 'bob', None): + for check in ('read', 'tool_use', 'write'): + await c.require_case('tool_gate', id='r_tool', check=check, actor=actor) + for certificates in (('c_leaf',), ('c_online_leaf',), ('c_delegation',)): + await c.require_case('attached_certificates', certificates=certificates) + # A share authorizes one read, never traversal, writes, or descendant reads. + insert_share(f) + with f.resource_change('r_file', mode=0o600), f.resource_change('r_area', mode=0o700): + for actor in ('alice', 'bob', 'ca', None): + for check in ('read', 'list', 'traverse', 'write'): + await c.require_case('share_exact_read', actor=actor, check=check) + for field, values in { + 'resource_id': ('r_file', 'r_other'), + 'grantor': (f.users['bob'], f.users['alice']), + 'grantee': (f.users['alice'], f.users['bob']), + 'grantee_kind': ('user', 'group'), + 'operations': ('["read"]', '["read","write"]', '[]', 'null', '{bad', '["read","read"]'), + 'constraints': ('{}', '{"name":"secret"}', '[]', 'null', 'bad'), + 'expires_at': ( + wire(NOW), + wire(NOW - timedelta(seconds=1)), + wire(NOW + timedelta(seconds=1)), + 'invalid', + ), + 'created_at': ( + wire(NOW), + wire(NOW + timedelta(seconds=1)), + wire(NOW - timedelta(days=1)), + 'invalid', + ), + 'revoked_at': (None, wire(NOW)), + 'allow_reshare': (0, 1), + }.items(): + for value in values: + with f.row_change('share_grants_v2', {'id': 's_main'}, {field: value}): + await c.source_case('share_facts') + await c.source_case('share_reshare', reshare=True) + with f.row_change( + 'share_grants_v2', {'id': 's_main'}, {'grantee': 'g_test', 'grantee_kind': 'group'} + ): + for subject in ('alice', 'bob', 'ca'): + await c.source_case('group_share', subject=subject) + insert_share(f, 's_child', grantor='alice', grantee='ca', parent='s_main') + await c.source_case('share_chain', grant='s_child', subject='ca') + for change in ( + {'revoked_at': wire(NOW)}, + {'allow_reshare': 0}, + {'expires_at': wire(NOW + timedelta(minutes=2))}, + {'parent_id': 's_child'}, + ): + with f.row_change('share_grants_v2', {'id': 's_main'}, change): + await c.source_case('share_chain', grant='s_child', subject='ca') + for id in ( + 'r_dm', + 'r_message', + 'r_soul', + 'r_todo', + 'r_legacy', + 'r_agents', + 'r_rules', + 'r_area', + 'r_tool', + 'c_leaf', + ): + with f.row_change('share_grants_v2', {'id': 's_main'}, {'resource_id': id}): + await c.source_case('protected_share', id=id) + for grantor in ('alice', 'bob'): + await c.link_case('share_link', grantor=grantor) + for change in ( + {'revoked_at': NOW}, + {'not_before': NOW + timedelta(seconds=1)}, + {'expires_at': NOW}, + {'ceiling': ()}, + ): + with f.body_change('credentials', f.keys['bob'], replace(f.credentials['bob'], **change)): + await c.link_case('share_link_credential') + for key, value in ( + ('recovery_quarantine', False), + ('recovery_quarantine', None), + ('recovery_quarantine', {}), + ): + with f.setting_change(key, value): + await c.require_case('quarantine', actor=None) + await c.link_case('quarantined_share') + # Once either authority service observed a replaced generation it remains + # fenced even if a later restore reinstates the old database value. + with f.setting_change('recovery_runtime_generation', 'replacement'): + await c.require_case('runtime_generation') + await c.require_case('runtime_generation_latched') + + +async def run(binary, report, postgres=False): + from contextlib import nullcontext + + from postgres_fixture import database, replace_fixture_storage + + with tempfile.TemporaryDirectory(prefix='msg-rust-policy-') as temp: + f = PolicyFixture(Path(temp) / 'metadata.sqlite3') + with database() if postgres else nullcontext(None) as dsn: + c = None + try: + if dsn: + await replace_fixture_storage(f, dsn) + c = PolicyCheck(binary, f) + await exercise(f, c) + data = { + 'backend': 'postgresql' if postgres else 'sqlite', + 'cases': sum(c.groups.values()), + 'groups': dict(c.groups), + 'failures': c.failures, + } + report.write_text(json.dumps(data, ensure_ascii=False, indent=2) + '\n') + print( + json.dumps({ + 'cases': data['cases'], + 'failures': len(c.failures), + 'report': str(report), + }) + ) + if c.failures: + print(json.dumps(c.failures[:8], indent=2)) + return bool(c.failures) + finally: + if c is not None: + c.process.stdin.close() + c.process.wait(timeout=10) + await f.close() + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('--binary', type=Path, required=True) + parser.add_argument('--report', type=Path, required=True) + parser.add_argument('--postgres', action='store_true') + options = parser.parse_args() + raise SystemExit( + asyncio.run(run(options.binary.resolve(), options.report.resolve(), options.postgres)) + ) diff --git a/rust/tests/test_content_parity.py b/rust/tests/test_content_parity.py new file mode 100644 index 00000000..7990699e --- /dev/null +++ b/rust/tests/test_content_parity.py @@ -0,0 +1,392 @@ +"""Real Python/Rust content interoperability on disposable, shared directories. + +Not an authenticated executor, full server acceptance or a recovery journal test. +All test identities, paths and hooks are synthetic. No production service access. +""" + +from __future__ import annotations + +import argparse +import asyncio +import base64 +import hashlib +import json +import os +import selectors +import subprocess +import tempfile +from dataclasses import replace +from datetime import UTC, datetime +from pathlib import Path + +from msg.core.codec import canonical, decode, wire +from msg.core.models import BlobRef, Revision +from msg.storage.git import GitContentStore + +REPORT = [] +NOW = datetime(2026, 10, 6, tzinfo=UTC) + + +def check(condition, name): + if not condition: + raise AssertionError(name) + REPORT.append(name) + + +def b64(value): + return base64.urlsafe_b64encode(value).rstrip(b'=').decode() + + +class Native: + def __init__(self, binary, root, *, env=None): + root.mkdir(exist_ok=True) + (root / '.msg-native-content-test').write_bytes(b'disposable test content\n') + self.process = subprocess.Popen( + [str(binary), str(root)], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + env=env, + ) + + def send(self, **value): + self.process.stdin.write(json.dumps(value) + '\n') + self.process.stdin.flush() + with selectors.DefaultSelector() as selector: + selector.register(self.process.stdout, selectors.EVENT_READ) + if not selector.select(timeout=20): + self.stop(kill=True) + raise AssertionError('native content fixture timed out') + line = self.process.stdout.readline() + if not line: + self.process.wait(timeout=5) + raise AssertionError(self.process.stderr.read()) + return json.loads(line) + + def call(self, **value): + result = self.send(**value) + if 'error' in result: + raise AssertionError(result) + return result['data'] + + def stop(self, *, kill=False): + if self.process.poll() is None: + if kill: + self.process.kill() + else: + self.process.stdin.close() + try: + self.process.wait(timeout=5) + except subprocess.TimeoutExpired: + self.process.kill() + self.process.wait(timeout=5) + raise + for stream in [self.process.stdin, self.process.stdout, self.process.stderr]: + stream.close() + + +def revision(blob, name, parents=()): + return Revision( + format_version=1, + id=name, + resource_id='r_content', + parents=list(parents), + content=blob, + relations=[], + actor='u_alice', + subject='u_alice', + author='u_alice', + created_at=NOW, + manifest_digest='sha256:' + 'a' * 64, + signature=None, + ) + + +def git(store, *args): + return subprocess.run( + ['git', '--git-dir', str(store.repo), *args], + check=True, + capture_output=True, + env=store.env, + timeout=20, + ).stdout + + +async def interoperate(binary, root): + store = GitContentStore(root) + native = Native(binary, root) + try: + kinds = [ + 'text/markdown', + 'application/json', + 'application/msg-template', + 'application/octet-stream', + ] + payloads = [b'', 'Hello\0中文\n'.encode(), bytes(range(251)) * 600] + for index, media in enumerate(kinds): + for number, data in enumerate(payloads): + label = f'{index}-{number}' + py_blob = await store.put_bytes(data, media) + check( + native.call(action='read', blob=wire(py_blob), range=None) == b64(data), + 'python-to-rust-' + label, + ) + native_blob = native.call( + action='put', bytes=b64(data), media_type=media, expected=None + ) + check(native_blob == wire(py_blob), 'blob-ref-' + label) + check( + await store.read_bytes(decode(BlobRef, native_blob)) == data, + 'rust-to-python-' + label, + ) + index_path = store.index / py_blob.digest.removeprefix('sha256:') + entry = json.loads(index_path.read_bytes()) + if entry['kind'] == 'git': + check( + git(store, 'cat-file', 'blob', entry['oid']) == data, 'git-object-' + label + ) + else: + check( + (store.binary / py_blob.digest.removeprefix('sha256:')).read_bytes() + == data, + 'binary-object-' + label, + ) + lease = '../../opaque/lease中文' + check( + native.call(action='pin', blob=native_blob, lease=lease), 'native-pin-' + label + ) + check(await store.pinned(py_blob, lease), 'python-sees-pin-' + label) + await store.unpin(py_blob, lease) + check( + not native.call(action='pinned', blob=native_blob, lease=lease), + 'native-sees-unpin-' + label, + ) + await store.pin(py_blob, lease) + check( + not native.call(action='unpin', blob=native_blob, lease=lease), + 'native-unpin-' + label, + ) + check(not await store.pinned(py_blob, lease), 'python-sees-unpin-' + label) + ranges = [(0, len(data)), (len(data), len(data))] + if len(data) > 65536: + ranges += [(65532, 65540), (5, 70000)] + for begin, end in ranges: + py_bytes = b''.join([ + chunk async for chunk in store.read(py_blob, (begin, end)) + ]) + check( + native.call(action='read', blob=native_blob, range=[begin, end]) + == b64(py_bytes), + f'range-{label}-{begin}-{end}', + ) + for media in ['text/plain', 'application/octet-stream']: + blob = await store.put_bytes(b'parent', media) + suffix = 'text' if media.startswith('text') else 'binary' + parent = revision(blob, 'v_python_' + suffix) + parent_oid = await store.commit_revision('t_fixture', parent) + check( + native.call(action='revision', topic='t_fixture', revision=wire(parent)) + == parent_oid, + 'reuse-python-commit-' + suffix, + ) + child = revision(blob, 'v_rust_' + suffix, [parent.id]) + child_oid = native.call(action='revision', topic='t_fixture', revision=wire(child)) + check( + git(store, 'show', f'{child_oid}:manifest.json') == canonical(child), + 'canonical-manifest-' + suffix, + ) + check( + git(store, 'rev-parse', f'{child_oid}^').strip().decode() == parent_oid, + 'native-parent-' + suffix, + ) + descendant = revision(blob, 'v_python_after_' + suffix, [child.id]) + descendant_oid = await store.commit_revision('t_fixture', descendant) + check( + git(store, 'rev-parse', f'{descendant_oid}^').strip().decode() == child_oid, + 'python-parent-' + suffix, + ) + conflict = replace(child, author='u_different') + check( + native.send(action='revision', topic='t_fixture', revision=wire(conflict)) + == {'error': 'revision_content_conflict'}, + 'immutable-revision-' + suffix, + ) + check( + native.call(action='revision', topic='t_fixture', revision=wire(child)) + == child_oid, + 'retry-native-commit-' + suffix, + ) + large = native.call( + action='generated', + length=4_194_305, + media_type='application/octet-stream', + hold_during=False, + hold_after=False, + ) + expected = 'sha256:' + hashlib.sha256(b'x' * large['size']).hexdigest() + check( + native.call(action='hash', blob=large) == {'digest': expected, 'size': large['size']}, + 'streaming-native-large-blob', + ) + py_hash = hashlib.sha256() + async for chunk in store.read(decode(BlobRef, large)): + py_hash.update(chunk) + check('sha256:' + py_hash.hexdigest() == expected, 'streaming-python-large-blob') + bad = native.send( + action='put', + bytes=b64(b'wrong'), + media_type='text/plain', + expected='sha256:' + '0' * 64, + ) + check(bad == {'error': 'digest_mismatch'}, 'digest-mismatch') + check(not (store.index / ('0' * 64)).exists(), 'digest-mismatch-no-index') + check(not list(store.staging.iterdir()), 'normal-paths-clean-staging') + finally: + native.stop() + + +async def hardlinks_and_corruption(binary, root): + store = GitContentStore(root) + blob = await store.put_bytes(b'correct', 'application/octet-stream') + path = store.binary / blob.digest.removeprefix('sha256:') + link = root / 'lfs-shared-link' + os.link(path, link) + before = path.stat().st_ino + native = Native(binary, root) + try: + native.call(action='put', bytes=b64(b'correct'), media_type=blob.media_type, expected=None) + check( + path.stat().st_ino == before and link.stat().st_nlink == 2, + 'binary-inode-and-lfs-hardlink-preserved', + ) + path.write_bytes(b'corrupt') + check( + native.send(action='read', blob=wire(blob), range=[0, 1]) + == {'error': 'content_digest_mismatch'}, + 'corrupt-range-rejected', + ) + check( + native.send( + action='put', bytes=b64(b'correct'), media_type=blob.media_type, expected=None + ) + == {'error': 'content_digest_mismatch'}, + 'corrupt-duplicate-not-replaced', + ) + check(link.read_bytes() == b'corrupt', 'existing-inode-never-rewritten') + path.unlink() + path.symlink_to(link) + check( + native.send(action='read', blob=wire(blob), range=None) + == {'error': 'unsafe_content_path'}, + 'binary-symlink-rejected', + ) + finally: + native.stop() + + +def hermetic_git(binary, root): + root.mkdir() + env = os.environ | { + 'GIT_DIR': str(root / 'wrong'), + 'GIT_OBJECT_DIRECTORY': str(root / 'wrong-objects'), + 'GIT_CONFIG_COUNT': '1', + 'GIT_CONFIG_KEY_0': 'core.bare', + 'GIT_CONFIG_VALUE_0': 'false', + } + native = Native(binary, root, env=env) + try: + native.call(action='put', bytes=b64(b'ready'), media_type='text/plain', expected=None) + hooks = root / 'private.git' / 'hooks' + hooks.mkdir(exist_ok=True) + marker = root / 'hook-executed' + hook = hooks / 'reference-transaction' + hook.write_text('#!/bin/sh\nprintf bad > "' + str(marker) + '"\n') + hook.chmod(0o700) + native.call( + action='put', bytes=b64(b'no inherited config'), media_type='text/plain', expected=None + ) + check(not marker.exists(), 'git-hooks-disabled') + check(not (root / 'wrong-objects').exists(), 'inherited-git-environment-ignored') + finally: + native.stop() + + +async def crashes(binary, root): + root.mkdir() + native = Native(binary, root) + try: + checkpoint = native.send( + action='generated', + length=131_072, + media_type='text/plain', + hold_during=True, + hold_after=False, + ) + check(checkpoint == {'checkpoint': 'held'}, 'crash-before-publication-checkpoint') + finally: + native.stop(kill=True) + check(not list((root / 'index').iterdir()), 'crash-before-publication-no-index') + # SIGKILL leaves staging bytes intentionally: no unsafe GC/recovery claim. + check(bool(list((root / 'staging').iterdir())), 'crash-staging-retained-for-recovery') + native = Native(binary, root) + length = 131_073 + blob = BlobRef( + digest='sha256:' + hashlib.sha256(b'x' * length).hexdigest(), + size=length, + media_type='text/plain', + ) + try: + checkpoint = native.send( + action='generated', + length=length, + media_type=blob.media_type, + hold_during=False, + hold_after=True, + ) + check(checkpoint == {'checkpoint': 'held'}, 'crash-after-publication-checkpoint') + finally: + native.stop(kill=True) + store = GitContentStore(root) + check(await store.read_bytes(blob) == b'x' * length, 'python-reads-after-lost-native-response') + native = Native(binary, root) + try: + check( + native.call( + action='generated', + length=length, + media_type=blob.media_type, + hold_during=False, + hold_after=False, + ) + == wire(blob), + 'retry-after-lost-response', + ) + finally: + native.stop() + + +async def main(binary): + with tempfile.TemporaryDirectory(prefix='msg-content-parity-') as name: + root = Path(name) + await interoperate(binary, root / 'interop') + await hardlinks_and_corruption(binary, root / 'hardlinks') + hermetic_git(binary, root / 'hermetic') + await crashes(binary, root / 'crashes') + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('binary', type=Path) + parser.add_argument('--report', type=Path) + args = parser.parse_args() + asyncio.run(main(args.binary.resolve())) + result = { + 'checks': len(REPORT), + 'passed': REPORT, + 'scope': 'content storage primitives; not a server or recovery journal', + } + if args.report: + args.report.parent.mkdir(parents=True, exist_ok=True) + args.report.write_text(json.dumps(result, ensure_ascii=False, indent=2) + '\n') + print(json.dumps(result, ensure_ascii=False)) diff --git a/rust/tests/test_identity_parity.py b/rust/tests/test_identity_parity.py new file mode 100644 index 00000000..6ffc9c83 --- /dev/null +++ b/rust/tests/test_identity_parity.py @@ -0,0 +1,1276 @@ +"""Check native authority reads against real Python services and Python-created SQLite. + +Only deterministic, public test keys and disposable databases are used. This is +not a production probe. The native JSONL helper has no network listener. +""" + +from __future__ import annotations + +import argparse +import asyncio +import json +import sqlite3 +import subprocess +import tempfile +from collections import Counter +from contextlib import contextmanager +from dataclasses import replace +from datetime import UTC, datetime, timedelta +from pathlib import Path + +from msg.browser_config import BrowserConfig +from msg.core.codec import b64, canonical, digest, wire +from msg.core.errors import Failure +from msg.core.models import ( + CapabilityGrant, + CapabilitySpec, + Certificate, + CertificateRequest, + Credential, + IssuancePolicy, + OperationResult, + OperationSpec, + Resource, + ResourceRef, + Scope, + SignatureProof, + Subject, + TokenProof, +) +from msg.core.packet import decode_packet +from msg.core.registry import Registry +from msg.core.requests import payload_fields, signing_bytes +from msg.core.schemas import NETWORK_CONSTRAINTS +from msg.security.authentication import AuthenticationService +from msg.security.certificates import ( + CertificateValidator, + certificate_body, + csr_body, + sign_certificate, +) +from msg.security.crypto import Ed25519Signer, key_id, subject_id +from msg.security.policy import allows, constraints_subset, scope_contains, scope_subset +from msg.security.token_delivery import recovery_verifier +from msg.storage.sqlite import SqliteMetadataStore + +NOW = datetime(2026, 10, 5, tzinfo=UTC) +SERVICE = 'https://msg.test.invalid' +READ = 'discovery.get' +WRITE = 'content.post_edit' +OPS = frozenset({ + READ + '@1', + WRITE + '@1', + 'identity.token_rotate@2', + 'identity.register@1', + 'identity.temporary@3', + 'identity.custodial_create@2', + 'identity.token_recover@1', + 'achievement.finish@1', + 'transfer.open@1', + 'tool.run@1', +}) + + +async def unused(*_args): + raise AssertionError('Conformance must never execute a business handler') + + +class Fixture: + def __init__(self, path: Path): + self.path = path + self.metadata = SqliteMetadataStore(path) + self.conn = sqlite3.connect(path, isolation_level=None) + self.conn.execute('PRAGMA foreign_keys=ON') + self.now = NOW + self.signers = { + name: Ed25519Signer.from_bytes(bytes([n]) * 32) + for n, name in enumerate(('root', 'alice', 'bob', 'ca', 'online'), 1) + } + self.users = {name: subject_id(s.public_key) for name, s in self.signers.items()} + self.users.update(root='u_root', online='u_online_ca') + self.keys = {name: key_id(s.public_key) for name, s in self.signers.items()} + self.registry = Registry() + self.grants = { + 'resource.basic': self.grant('resource.basic', OPS), + 'identity.basic': self.grant('identity.basic', OPS), + 'cert.issue': self.grant('cert.issue', {'cert.publish@1'}), + 'cert.ca.issue': self.grant('cert.ca.issue', {'cert.publish@1'}), + 'tool.use': self.grant( + 'tool.use', + {'tool.run@1'}, + constraints={ + 'hosts': ['example.test'], + 'schemes': ['https'], + 'ports': [443], + 'methods': ['GET', 'HEAD'], + 'max_response_bytes': 4096, + 'timeout_ms': 5000, + 'max_redirects': 2, + }, + ), + } + self.primary = tuple(self.grants.values()) + for name, grant in self.grants.items(): + self.registry.add_capability( + CapabilitySpec( + name=name, + version=1, + scope_types=frozenset({'space', 'topic', 'file', 'certificate', 'delegation'}), + operations=grant.operations, + replaces_checks=frozenset(), + delegatable=True, + ca_only=name.startswith('cert.'), + constraints_schema=ResourceRef(id='schema:network-constraints') + if name == 'tool.use' + else None, + ) + ) + self.registry.add_schema(ResourceRef(id='schema:network-constraints'), NETWORK_CONSTRAINTS) + self.resources = {} + for rid, parent, kind in [ + ('r_root', None, 'space'), + ('r_area', 'r_root', 'topic'), + ('r_file', 'r_area', 'file'), + ('r_other', 'r_root', 'topic'), + ('r_delegation', 'r_root', 'delegation'), + ('c_root', 'r_root', 'certificate'), + ('c_ca', 'r_root', 'certificate'), + ('c_leaf', 'r_root', 'certificate'), + ('c_delegation', 'r_root', 'certificate'), + ('c_online', 'r_root', 'certificate'), + ('c_online_leaf', 'r_root', 'certificate'), + ('c_network', 'r_root', 'certificate'), + ('csr_new', 'r_root', 'certificate'), + ('c_new', 'r_root', 'certificate'), + ]: + resource = Resource( + id=rid, + type=kind, + type_version=1, + name=rid, + parent=parent, + owner=self.users['bob'], + group='g_test', + mode=0o755, + generation=0, + revision='v_' + rid, + state='active', + created_at=NOW, + created_by=self.users['bob'], + modified_at=NOW, + modified_by=self.users['bob'], + ) + self.resources[rid] = resource + data = wire(resource) + self.conn.execute( + 'INSERT INTO resources VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?)', + ( + rid, + kind, + rid, + parent, + resource.owner, + resource.group, + resource.mode, + 0, + resource.revision, + 'active', + data['created_at'], + data['modified_at'], + canonical(resource).decode(), + ), + ) + self.subjects = {} + self.credentials = {} + for name, signer in self.signers.items(): + subject = Subject( + resource_id=self.users[name], + kind='system' if name in {'root', 'online'} else 'registered', + primary_group='g_test', + auth_version=1, + local_only=name == 'root', + ) + credential = Credential( + id=self.keys[name], + subject_id=subject.resource_id, + kind='signing_key', + verifier=signer.public_key, + ceiling=self.primary, + not_before=NOW - timedelta(days=7), + expires_at=NOW + timedelta(days=40), + revoked_at=None, + ) + self.subjects[name], self.credentials[name] = subject, credential + self.conn.execute( + 'INSERT INTO identities VALUES (?,?,?,?)', + (subject.resource_id, 'subject', 0, canonical(subject).decode()), + ) + self.conn.execute( + 'INSERT INTO credentials VALUES (?,?,?)', + (credential.id, subject.resource_id, canonical(credential).decode()), + ) + self.token = bytes(range(32)) + import hashlib + + self.credentials['token'] = Credential( + id='t_fixture', + subject_id=self.users['alice'], + kind='token', + verifier=hashlib.sha256(self.token).digest(), + ceiling=self.primary, + not_before=NOW - timedelta(days=1), + expires_at=NOW + timedelta(days=1), + revoked_at=None, + ) + self.conn.execute( + 'INSERT INTO credentials VALUES (?,?,?)', + ('t_fixture', self.users['alice'], canonical(self.credentials['token']).decode()), + ) + self.certs = {} + root_policy = IssuancePolicy( + issue_grants=self.primary, + max_cert_ttl_seconds=15 * 86400, + max_child_ca_depth=3, + max_delegation_depth=4, + ) + ca_policy = replace(root_policy, max_cert_ttl_seconds=7 * 86400, max_child_ca_depth=2) + issue = (self.grants['cert.issue'], self.grants['cert.ca.issue']) + self.add_cert( + 'c_root', + 'root', + 'root', + None, + kind='ca', + grants=issue, + issuance=root_policy, + before=NOW - timedelta(days=5), + expires=NOW + timedelta(days=30), + ) + self.add_cert( + 'c_ca', + 'ca', + 'root', + 'c_root', + kind='ca', + grants=issue, + issuance=ca_policy, + before=NOW - timedelta(days=2), + expires=NOW + timedelta(days=10), + ) + leaf = self.grant('resource.basic', {READ + '@1'}, scope='r_area', descendants=True) + self.add_cert('c_leaf', 'alice', 'ca', 'c_ca', grants=(leaf,)) + self.add_cert( + 'c_delegation', + 'alice', + 'ca', + 'c_ca', + kind='delegation', + grants=(leaf,), + sources=(ResourceRef(id='r_delegation', revision='v_r_delegation'),), + ) + self.fact = { + 'grantor': self.users['bob'], + 'grantee': self.users['alice'], + 'grants': wire((leaf,)), + 'expires_at': wire(NOW + timedelta(hours=2)), + 'parent_certificate': None, + 'grantor_credential_id': self.keys['bob'], + } + self.setting('delegation:r_delegation', self.fact) + online_policy = IssuancePolicy( + issue_grants=(self.grants['resource.basic'],), + max_cert_ttl_seconds=86400, + max_child_ca_depth=0, + max_delegation_depth=1, + ) + self.add_cert( + 'c_online', + 'online', + 'root', + 'c_root', + kind='ca', + grants=(self.grants['cert.issue'],), + issuance=online_policy, + ) + self.add_cert('c_online_leaf', 'alice', 'online', 'c_online', grants=(leaf,)) + self.add_cert('c_network', 'alice', 'ca', 'c_ca', grants=(self.grants['tool.use'],)) + self.validator = CertificateValidator( + self.registry, + self.certs['c_root'], + self.signers['root'].public_key, + SERVICE, + lambda: self.now, + ) + self.auth = AuthenticationService( + self.registry, + self.validator, + SERVICE, + lambda: self.now, + lambda: self.primary, + lambda: self.primary, + ) + self.browser = BrowserConfig(enabled=True) + self.auth.browser_config = self.browser + + def grant(self, name, operations, *, scope='r_root', descendants=True, constraints=None): + return CapabilityGrant( + capability=name, + version=1, + scope=Scope(resource_id=scope, descendants=descendants), + operations=frozenset(operations), + constraints=constraints or {}, + ) + + def add_cert( + self, + id, + subject, + issuer, + parent, + *, + kind='identity', + grants=(), + issuance=None, + sources=(), + before=None, + expires=None, + ): + cert = Certificate( + resource_id=id, + serial=id, + subject_id=self.users[subject], + key_id=self.keys[subject], + issuer_id=self.users[issuer], + parent_certificate_id=parent, + authority_sources=sources, + kind=kind, + grants=grants, + not_before=before or NOW - timedelta(hours=1), + expires_at=expires or NOW + timedelta(hours=1), + target_service=SERVICE, + delegation_depth=0, + issuance=issuance, + signature=self.signers[issuer].sign(b'', purpose='certificate'), + ) + cert = sign_certificate(cert, self.signers[issuer]) + self.certs[id] = cert + self.conn.execute( + 'INSERT INTO certificates VALUES (?,?,?,?,?)', + (id, cert.subject_id, parent, 0, canonical(cert).decode()), + ) + return cert + + def context(self): + return { + 'root': wire(self.certs['c_root']), + 'root_public': b64(self.signers['root'].public_key), + 'service': SERVICE, + 'capabilities': wire(self.registry.capabilities()), + 'primary_ceiling': wire(self.primary), + 'temporary_ceiling': wire(self.primary), + 'browser': wire(self.browser), + } + + def setting(self, key, value): + self.conn.execute( + 'INSERT INTO settings VALUES (?,?) ON CONFLICT(key) DO UPDATE SET value=excluded.value', + (key, canonical(value).decode()), + ) + + @contextmanager + def setting_change(self, key, value): + old = self.conn.execute('SELECT value FROM settings WHERE key=?', (key,)).fetchone() + self.setting(key, value) + try: + yield + finally: + if old is None: + self.conn.execute('DELETE FROM settings WHERE key=?', (key,)) + else: + self.conn.execute('UPDATE settings SET value=? WHERE key=?', (old[0], key)) + + @contextmanager + def body_change(self, table, id, body): + assert table in {'resources', 'credentials', 'certificates', 'identities'} + old = self.conn.execute(f'SELECT body FROM {table} WHERE id=?', (id,)).fetchone()[0] + self.conn.execute(f'UPDATE {table} SET body=? WHERE id=?', (canonical(body).decode(), id)) + try: + yield + finally: + self.conn.execute(f'UPDATE {table} SET body=? WHERE id=?', (old, id)) + + def spec( + self, + name=READ, + *, + version=1, + effect='read', + signature=False, + entries=('network',), + anonymous=False, + ): + spec = OperationSpec( + name=name, + version=version, + input_schema=ResourceRef(id='input'), + output_schema=ResourceRef(id='output'), + effect=effect, + entries=frozenset(entries), + require_signature=signature, + anonymous_only=anonymous, + handler=unused, + requirements=unused, + ) + self.registry._operations[(name, version)] = spec + return { + key: wire(getattr(spec, key)) + for key in ( + 'name', + 'version', + 'effect', + 'entries', + 'require_signature', + 'anonymous_only', + ) + } + + def request( + self, + *, + name=READ, + version=1, + actor='alice', + subject='default', + token=False, + token_id='t_fixture', + anonymous=False, + expires=60, + certificates=(), + arguments=None, + request_id='fixture-request', + resign=True, + **changes, + ): + subject = self.users[actor] if subject == 'default' else subject + fields = { + 'request_id': request_id, + 'protocol_version': 1, + 'operation': name, + 'contract_version': version, + 'target_service': SERVICE, + 'subject': None if anonymous else subject, + 'arguments': arguments or {'id': 'r_file'}, + 'expected_generations': [], + 'expires_at': None if expires is None else wire(NOW + timedelta(seconds=expires)), + 'payload_digest': 'sha256:' + '0' * 64, + 'proof': None, + 'source': 'mcp', + 'return_fields': [], + } + fields.update(changes) + request = decode_packet(canonical(fields)) + request = replace(request, payload_digest=digest(payload_fields(request))) + if not anonymous: + proof = ( + TokenProof(credential_id=token_id, token=self.token) + if token + else SignatureProof( + signature=self.signers[actor].sign(signing_bytes(request), purpose='request'), + certificates=certificates, + ) + ) + request = replace(request, proof=proof) + if not resign: + request = replace(request, target_service='https://wrong.test.invalid') + return request + + async def close(self): + self.conn.close() + await self.metadata.close() + + +class Check: + def __init__(self, binary, fixture): + self.fixture = fixture + self.process = subprocess.Popen( + [str(binary), str(fixture.path)], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + encoding='utf-8', + ) + self.groups = Counter() + self.failures = [] + self.send(fixture.context()) + assert self.receive() == {'ok': True, 'data': {'ready': True}} + + def send(self, value): + self.process.stdin.write( + json.dumps(value, ensure_ascii=False, separators=(',', ':')) + '\n' + ) + self.process.stdin.flush() + + def receive(self): + line = self.process.stdout.readline() + if not line: + raise AssertionError('native helper ended: ' + self.process.stderr.read()[-1000:]) + return json.loads(line) + + def native(self, command): + self.send(command) + return self.receive() + + def compare(self, group, actual, expected, *, count=1): + self.groups[group] += count + if actual != expected: + # Synthetic keys only, but reports still omit request/proof material. + self.failures.append({ + 'group': group, + 'index': self.groups[group], + 'actual': actual, + 'expected': expected, + }) + + async def auth_case( + self, + group, + request, + policy, + *, + entry='network', + replay=False, + ): + command = { + 'action': 'authenticate', + 'raw': canonical(request).decode(), + 'policy': policy, + 'entry': entry, + 'now': wire(self.fixture.now), + } + async with self.fixture.metadata.transaction(write=False) as tx: + try: + principal = await self.fixture.auth.authenticate(request, tx, entry=entry) + if replay: + previous = await tx.request_result( + principal.subject, request.request_id, request.payload_digest + ) + assert previous is not None + data = {'kind': 'replay', 'result': wire(previous)} + else: + data = {'kind': 'fresh', 'principal': wire(principal)} + expected = {'ok': True, 'data': data} + except Failure as exc: + expected = {'ok': False, 'code': exc.code} + self.compare(group, self.native(command), expected) + + async def cert_case(self, group, id='c_leaf'): + async with self.fixture.metadata.transaction(write=False) as tx: + try: + cert = await self.fixture.validator.validate(id, tx) + expected = { + 'ok': True, + 'data': { + 'canonical': canonical(cert).decode(), + 'signing_digest': digest(canonical(certificate_body(cert))), + }, + } + except Failure as exc: + expected = {'ok': False, 'code': exc.code} + self.compare( + group, + self.native({'action': 'certificate', 'id': id, 'now': wire(self.fixture.now)}), + expected, + ) + + def close(self): + self.process.stdin.close() + self.process.wait(timeout=15) + assert self.process.returncode == 0, self.process.stderr.read() + self.process.stdout.close() + self.process.stderr.close() + + +def ok(data): + return {'ok': True, 'data': data} + + +async def run(binary, report): + with tempfile.TemporaryDirectory(prefix='msg-native-authority-') as directory: + fixture = Fixture(Path(directory) / 'metadata.db') + check = Check(binary, fixture) + try: + await cases(fixture, check) + finally: + check.close() + await fixture.close() + summary = { + 'cases': sum(check.groups.values()), + 'groups': dict(sorted(check.groups.items())), + 'failures': check.failures, + 'failure_count': len(check.failures), + 'fixture': 'real Python SqliteMetadataStore schema; deterministic public test keys', + 'limitations': [ + 'Key-approved issuance and token delivery are not part of this read-only slice; retired OAuth authority is rejected', + 'Replay admission returns only the stored result, not a fresh principal', + 'Mode corpus counts each mode/class/permission decision separately', + ], + } + report.parent.mkdir(parents=True, exist_ok=True) + report.write_text(json.dumps(summary, ensure_ascii=False, indent=2) + '\n') + print(json.dumps({key: value for key, value in summary.items() if key != 'failures'})) + for failure in check.failures[:10]: + print(json.dumps(failure, ensure_ascii=False)) + if check.failures: + raise AssertionError(f'{len(check.failures)} native authority mismatches') + + +async def cases(f, check): + # Exhaustive mode decisions: 4096 modes x 3 classes x 5 permissions. + decisions = [] + resource = f.resources['r_file'] + for mode in range(0o10000): + for subject in (resource.owner, 'member', None): + for permission in ('read', 'write', 'execute', 'list', 'traverse'): + decisions.append( + allows(replace(resource, mode=mode), subject, {resource.group}, permission) + ) + check.compare( + 'mode_decisions', + check.native({'action': 'modes'}), + ok({'count': len(decisions), 'digest': digest(decisions)}), + count=len(decisions), + ) + + for kind, values in [ + ('subject', f.subjects.values()), + ('credential', f.credentials.values()), + ('certificate', f.certs.values()), + ('resource', f.resources.values()), + ]: + for record in values: + id = getattr(record, 'id', None) or record.resource_id + check.compare( + 'record_bytes', + check.native({'action': 'record_digest', 'kind': kind, 'id': id}), + ok({'digest': digest(record)}), + ) + + for parent in ('r_root', 'r_area', 'r_file', 'r_other'): + for child in ('r_root', 'r_area', 'r_file', 'r_other'): + for descendants in (False, True): + scope = Scope(resource_id=parent, descendants=descendants) + async with f.metadata.transaction(write=False) as tx: + contains = await scope_contains(scope, child, tx) + check.compare( + 'scope_contains', + check.native({'action': 'scope', 'scope': wire(scope), 'id': child}), + ok({'contains': contains}), + ) + for child_scope in ('r_area', 'r_file', 'r_other'): + for operations in ({READ + '@1'}, {WRITE + '@1'}, {READ + '@1', WRITE + '@1'}): + child = f.grant('resource.basic', operations, scope=child_scope) + parent = f.grant('resource.basic', {READ + '@1'}, scope='r_area') + async with f.metadata.transaction(write=False) as tx: + subset = child.operations <= parent.operations and await scope_subset( + child.scope, parent.scope, tx + ) + check.compare( + 'grant_subset', + check.native({'action': 'subset', 'child': wire(child), 'parent': wire(parent)}), + ok({'subset': subset}), + ) + parent = f.grants['tool.use'] + for constraints in ( + {}, + dict(parent.constraints), + {**parent.constraints, 'methods': ['GET']}, + {**parent.constraints, 'methods': ['POST']}, + {**parent.constraints, 'timeout_ms': 5001}, + {**parent.constraints, 'max_response_bytes': 4095}, + {**parent.constraints, 'ports': [80, 443]}, + {**parent.constraints, 'hosts': []}, + {**parent.constraints, 'max_redirects': True}, + ): + child = replace(parent, constraints=constraints) + check.compare( + 'constraint_subset', + check.native({'action': 'subset', 'child': wire(child), 'parent': wire(parent)}), + ok({'subset': constraints_subset(child.constraints, parent.constraints)}), + ) + + for child_value, parent_value in [ + (-2, -1), + (-1, -2), + (-1, 0), + (0, -1), + (10**80, 10**80 + 1), + (10**80 + 1, 10**80), + (2, 2.0), + (2.0, 2), + ]: + child = replace(parent, constraints={'max_redirects': child_value}) + upper = replace(parent, constraints={'max_redirects': parent_value}) + check.compare( + 'numeric_constraint_subset', + check.native({'action': 'subset', 'child': wire(child), 'parent': wire(upper)}), + ok({'subset': constraints_subset(child.constraints, upper.constraints)}), + ) + for ports in ([443.0], [443], [444.0]): + child = replace(parent, constraints={'ports': ports}) + upper = replace(parent, constraints={'ports': [443]}) + check.compare( + 'numeric_allowlist_subset', + check.native({'action': 'subset', 'child': wire(child), 'parent': wire(upper)}), + ok({'subset': constraints_subset(child.constraints, upper.constraints)}), + ) + network = f.certs['c_network'] + for constraints in [ + {**parent.constraints, 'ports': [443.0]}, + {**parent.constraints, 'ports': [0]}, + {**parent.constraints, 'ports': [65536]}, + {**parent.constraints, 'ports': [True]}, + {**parent.constraints, 'hosts': 'not-a-list'}, + {**parent.constraints, 'ports': 'not-a-list'}, + {**parent.constraints, 'max_redirects': -1}, + {**parent.constraints, 'timeout_ms': 0}, + {**parent.constraints, 'timeout_ms': 5000.0}, + ]: + proposed = sign_certificate( + replace(network, grants=(replace(parent, constraints=constraints),)), f.signers['ca'] + ) + with f.body_change('certificates', network.resource_id, proposed): + await check.cert_case('network_constraint_validation', network.resource_id) + + policy = f.spec() + request = f.request() + for cert in f.certs: + await check.cert_case('valid_chain', cert) + for ids in ((), ('c_leaf',), ('c_leaf', 'c_delegation'), ('c_missing',)): + await check.auth_case('signed_auth', f.request(certificates=ids), policy) + for expiry in (None, -300, -1, 0, 0.000001, 1, 299, 300, 300.000001, 301, 600): + for token in (False, True): + await check.auth_case('request_expiry', f.request(expires=expiry, token=token), policy) + await check.auth_case('anonymous_read', f.request(expires=expiry, anonymous=True), policy) + await check.auth_case('wrong_service', f.request(resign=False), policy) + await check.auth_case('digest_tamper', replace(request, arguments={'id': 'r_other'}), policy) + await check.auth_case('signature_tamper', replace(request, source='manual'), policy) + await check.auth_case('missing_proof', replace(request, proof=None), policy) + await check.auth_case('subject_spoof', f.request(subject=f.users['bob']), policy) + await check.auth_case( + 'delegated_subject', + f.request(subject=f.users['bob'], certificates=('c_delegation',)), + policy, + ) + await check.auth_case('entry_denied', request, policy, entry='worker') + await check.auth_case('root_network', f.request(actor='root'), policy) + local_policy = f.spec(entries=('network', 'local_admin')) + await check.auth_case('root_local', f.request(actor='root'), local_policy, entry='local_admin') + policy = f.spec() + for name, effect, version, anonymous in [ + ('content.post_edit', 'transaction', 1, False), + ('identity.link_claim', 'transaction', 1, True), + ('communication.internet_receive', 'transaction', 1, True), + ('identity.link_claim', 'transaction', 2, True), + ('identity.link_claim', 'external', 1, True), + ]: + p = f.spec(name, effect=effect, version=version, anonymous=anonymous) + await check.auth_case( + 'anonymous_write', f.request(name=name, version=version, anonymous=True), p + ) + policy = f.spec() + + for flag in (False, None, {}, True): + with f.setting_change('recovery_quarantine', flag): + await check.auth_case('quarantine_presence', request, policy) + p = f.spec(WRITE, effect='transaction') + await check.auth_case('quarantine_write', f.request(name=WRITE), p) + for key, value in [ + ('active_root_certificate', 'other-root'), + ('identity_archived:' + f.users['alice'], True), + ('identity_archived:' + f.users['alice'], False), + ]: + with f.setting_change(key, value): + await check.auth_case('authority_fences', request, policy) + + for name, field, value in [ + ('alice', 'revoked_at', NOW), + ('alice', 'expires_at', NOW), + ('alice', 'not_before', NOW + timedelta(microseconds=1)), + ('alice', 'expires_at', None), + ('alice', 'ceiling', ()), + ('alice', 'kind', 'ssh_key'), + ('token', 'verifier', b'wrong'), + ('token', 'revoked_at', NOW), + ('token', 'expires_at', NOW), + ]: + credential = f.credentials[name] + with f.body_change('credentials', credential.id, replace(credential, **{field: value})): + await check.auth_case('credential_state', f.request(token=name == 'token'), policy) + p = f.spec(signature=True) + await check.auth_case('signature_required', f.request(token=True), p) + policy = f.spec() + derived = replace(f.credentials['token'], source_credential_id=f.keys['alice']) + with f.body_change('credentials', derived.id, derived): + await check.auth_case('missing_token_binding', f.request(token=True), policy) + source = { + 'parent': f.keys['alice'], + 'subject': f.users['alice'], + 'auth_version': 1, + 'ceiling': wire(f.primary), + } + with f.body_change('credentials', derived.id, derived): + f.conn.execute( + 'INSERT INTO credential_states VALUES (?,?,?,?)', + ( + 'api:' + derived.id, + 'binding', + wire(NOW + timedelta(days=1)), + canonical(source).decode(), + ), + ) + await check.auth_case('key_approved_api_binding', f.request(token=True), policy) + for field, value in [('revoked_at', NOW), ('expires_at', NOW), ('ceiling', ())]: + parent = f.credentials['alice'] + with f.body_change('credentials', parent.id, replace(parent, **{field: value})): + await check.auth_case('binding_live_parent', f.request(token=True), policy) + with f.body_change( + 'identities', f.users['alice'], replace(f.subjects['alice'], auth_version=2) + ): + await check.auth_case('binding_auth_version', f.request(token=True), policy) + nested = {**source, 'session': 'session:fixture'} + f.conn.execute( + 'UPDATE credential_states SET body=? WHERE id=?', + (canonical(nested).decode(), 'api:' + derived.id), + ) + f.conn.execute( + 'INSERT INTO credential_states VALUES (?,?,?,?)', + ( + 'session:fixture', + 'session', + wire(NOW + timedelta(seconds=60)), + canonical(source).decode(), + ), + ) + await check.auth_case('binding_nested_session', f.request(token=True), policy) + f.conn.execute( + 'UPDATE credential_states SET body=? WHERE id=?', + (canonical({**source, 'revoked': True}).decode(), 'session:fixture'), + ) + await check.auth_case('binding_session_revoked', f.request(token=True), policy) + f.conn.execute( + 'UPDATE credential_states SET body=?,expires=? WHERE id=?', + (canonical(source).decode(), wire(NOW), 'session:fixture'), + ) + await check.auth_case('binding_session_expired', f.request(token=True), policy) + for field, value in [ + ('login_binding', True), + ('custodial', True), + ('ceiling', []), + ('subject', f.users['bob']), + ]: + f.conn.execute( + 'UPDATE credential_states SET body=? WHERE id=?', + (canonical({**source, field: value}).decode(), 'api:' + derived.id), + ) + await check.auth_case('binding_unapproved_source', f.request(token=True), policy) + f.conn.execute('DELETE FROM credential_states') + + # Positive browser case must have a genuine current session ID, never a + # restored OAuth access credential with a new-looking binding attached. + browser = replace(derived, id='t_session_fixture') + f.conn.execute( + 'INSERT INTO credentials VALUES (?,?,?)', + (browser.id, browser.subject_id, canonical(browser).decode()), + ) + try: + f.conn.execute( + 'INSERT INTO credential_states VALUES (?,?,?,?)', + ( + 'browser:' + browser.id, + 'binding', + wire(NOW + timedelta(days=1)), + canonical(source).decode(), + ), + ) + await check.auth_case( + 'key_approved_browser_binding', f.request(token=True, token_id=browser.id), policy + ) + for body in [ + {**source, 'login_binding': True}, + {**source, 'custodial': True}, + {**source, 'parent': f.keys['bob']}, + {**source, 'ceiling': []}, + ]: + f.conn.execute( + 'UPDATE credential_states SET body=? WHERE id=?', + (canonical(body).decode(), 'browser:' + browser.id), + ) + await check.auth_case( + 'browser_binding_mismatch', f.request(token=True, token_id=browser.id), policy + ) + f.conn.execute('DELETE FROM credential_states') + await check.auth_case( + 'browser_binding_missing', f.request(token=True, token_id=browser.id), policy + ) + finally: + f.conn.execute('DELETE FROM credentials WHERE id=?', (browser.id,)) + + # Even old authority rows restored beside a valid live parent are rejected. + f.conn.execute( + 'CREATE TABLE oauth_states(id TEXT PRIMARY KEY,kind TEXT,expires TEXT,body TEXT)' + ) + try: + for id in ('t_oauth_fixture', 't_browser_fixture'): + legacy = replace(derived, id=id) + f.conn.execute( + 'INSERT INTO credentials VALUES (?,?,?)', + (id, legacy.subject_id, canonical(legacy).decode()), + ) + f.conn.execute( + 'INSERT INTO oauth_states VALUES (?,?,?,?)', + ( + 'access:' + id, + 'access', + wire(NOW + timedelta(days=1)), + canonical(source).decode(), + ), + ) + f.conn.execute( + 'INSERT INTO credential_states VALUES (?,?,?,?)', + ('api:' + id, 'binding', wire(NOW + timedelta(days=1)), canonical(source).decode()), + ) + await check.auth_case( + 'restored_legacy_authority_denied', f.request(token=True, token_id=id), policy + ) + f.conn.execute('DELETE FROM credentials WHERE id=?', (id,)) + f.conn.execute('DELETE FROM credential_states') + finally: + f.conn.execute('DROP TABLE oauth_states') + + for actor, kind in [('alice', 'custodial'), ('alice', 'registered')]: + subject = f.subjects[actor] + with f.body_change('identities', subject.resource_id, replace(subject, kind=kind)): + for name, arguments in [ + (WRITE, {}), + ('tool.run', {}), + ('achievement.finish', {}), + ('transfer.open', {'direction': 'download'}), + ('transfer.open', {'direction': 'upload'}), + ]: + p = f.spec(name, effect='transaction', signature=name == 'achievement.finish') + await check.auth_case( + 'custodial_limits', f.request(name=name, token=True, arguments=arguments), p + ) + + for kind, version, nonce_size in [ + ('identity.temporary', 3, 23), + ('identity.temporary', 3, 24), + ('identity.custodial_create', 2, 32), + ('identity.temporary', 3, 64), + ('identity.temporary', 3, 65), + ]: + p = f.spec(kind, version=version, effect='transaction') + req = f.request( + name=kind, + version=version, + anonymous=True, + arguments={'nonce': b64(bytes(range(nonce_size)))}, + ) + await check.auth_case('bootstrap', req, p) + p = f.spec('identity.register', effect='transaction') + req = f.request( + name='identity.register', arguments={'public_key': b64(f.signers['alice'].public_key)} + ) + await check.auth_case('register', req, p) + with f.setting_change('delegated_identity:' + f.users['alice'], {'locked': True}): + await check.auth_case('register_delegated_denied', req, p) + + # Invalid child conditions are re-signed to test rules after cryptography. + leaf = f.certs['c_leaf'] + for field, value in [ + ('target_service', 'https://other.test.invalid'), + ('not_before', NOW + timedelta(seconds=1)), + ('expires_at', NOW), + ('issuer_id', f.users['alice']), + ('subject_id', 'u_root'), + ('delegation_depth', 5), + ('not_before', NOW - timedelta(days=3)), + ('expires_at', NOW + timedelta(days=11)), + ('expires_at', NOW + timedelta(days=8)), + ('issuance', f.certs['c_ca'].issuance), + ('grants', (f.grant('resource.basic', {'unknown.call@1'}),)), + ('grants', (f.grant('missing.capability', {READ + '@1'}),)), + ('grants', (replace(leaf.grants[0], constraints={'unknown': 'x'}),)), + ('subject_id', f.users['bob']), + ]: + changed = sign_certificate(replace(leaf, **{field: value}), f.signers['ca']) + with f.body_change('certificates', leaf.resource_id, changed): + await check.cert_case('invalid_child') + with f.body_change('certificates', 'c_leaf', replace(leaf, serial='tampered')): + await check.cert_case('invalid_signature') + with f.body_change('certificates', 'c_root', replace(f.certs['c_root'], serial='tampered')): + await check.cert_case('anchor_pin') + for id in ('c_leaf', 'c_ca', 'c_root'): + f.conn.execute('UPDATE certificates SET revoked=1 WHERE id=?', (id,)) + await check.cert_case('certificate_revoked') + f.conn.execute('UPDATE certificates SET revoked=0 WHERE id=?', (id,)) + for name in ('alice', 'ca', 'root'): + old = f.credentials[name] + with f.body_change('credentials', old.id, replace(old, revoked_at=NOW)): + await check.cert_case('issuer_key_revoked') + with f.body_change( + 'certificates', 'c_ca', replace(f.certs['c_ca'], parent_certificate_id='c_leaf') + ): + await check.cert_case('certificate_cycle') + for field, value in [ + ('kind', 'identity'), + ('issuance', None), + ('grants', (f.grants['cert.ca.issue'],)), + ('grants', (f.grants['cert.issue'],)), + ('issuance', replace(f.certs['c_ca'].issuance, max_child_ca_depth=3)), + ('issuance', replace(f.certs['c_ca'].issuance, max_cert_ttl_seconds=16 * 86400)), + ('issuance', replace(f.certs['c_ca'].issuance, max_delegation_depth=5)), + ]: + cert = sign_certificate(replace(f.certs['c_ca'], **{field: value}), f.signers['root']) + with f.body_change('certificates', 'c_ca', cert): + await check.cert_case('ca_policy') + for field, value in [('kind', 'capability'), ('kind', 'delegation')]: + cert = sign_certificate( + replace(f.certs['c_online_leaf'], **{field: value}), f.signers['online'] + ) + with f.body_change('certificates', 'c_online_leaf', cert): + await check.cert_case('online_ca_limits', 'c_online_leaf') + for field, value in [ + ('issuance', replace(f.certs['c_online'].issuance, max_child_ca_depth=1)), + ('grants', (f.grants['cert.issue'], f.grants['cert.ca.issue'])), + ]: + cert = sign_certificate(replace(f.certs['c_online'], **{field: value}), f.signers['root']) + with f.body_change('certificates', 'c_online', cert): + await check.cert_case('online_ca_limits', 'c_online_leaf') + + for field, value in [ + ('expires_at', wire(NOW)), + ('grantee', f.users['bob']), + ('grantor', f.users['alice']), + ('grants', []), + ('parent_certificate', 'c_delegation'), + ]: + with f.setting_change('delegation:r_delegation', {**f.fact, field: value}): + await check.cert_case('delegation_source', 'c_delegation') + resource = f.resources['r_delegation'] + for field, value in [('state', 'archived'), ('revision', 'v_changed')]: + with f.body_change('resources', resource.id, replace(resource, **{field: value})): + await check.cert_case('delegation_resource', 'c_delegation') + credential = f.credentials['bob'] + for field, value in [('revoked_at', NOW), ('ceiling', ())]: + with f.body_change('credentials', credential.id, replace(credential, **{field: value})): + await check.cert_case('delegation_live_source', 'c_delegation') + with f.setting_change('identity_archived:' + f.users['bob'], True): + await check.cert_case('delegation_live_source', 'c_delegation') + + # CSR and publication signed bytes use the same actual Python model. + csr = CertificateRequest( + resource_id='csr_new', + applicant=f.users['alice'], + subject_id=f.users['alice'], + requested_issuer=f.users['ca'], + public_key=f.signers['alice'].public_key, + kind='identity', + grants=leaf.grants, + issuance=None, + requested_ttl_seconds=7200, + target_service=SERVICE, + delegation_depth=0, + authority_sources=(), + request_digest='', + possession_proof=f.signers['alice'].sign(b'', purpose='csr'), + ) + csr = replace( + csr, + request_digest=digest(csr_body(csr)), + possession_proof=f.signers['alice'].sign(canonical(csr_body(csr)), purpose='csr'), + ) + proposed = sign_certificate(replace(leaf, resource_id='c_new', serial='new'), f.signers['ca']) + for field, val in [ + (None, None), + ('request_digest', 'sha256:' + '0' * 64), + ('requested_issuer', f.users['bob']), + ]: + candidate = csr if field is None else replace(csr, **{field: val}) + async with f.metadata.transaction(write=False) as tx: + try: + result = await f.validator.validate_publication(proposed, candidate, tx) + expected = ok({'canonical': canonical(result).decode()}) + except Failure as exc: + expected = {'ok': False, 'code': exc.code} + check.compare( + 'csr_publication', + check.native({ + 'action': 'publication', + 'certificate': wire(proposed), + 'csr': wire(candidate), + 'now': wire(f.now), + }), + expected, + ) + + # Rotation retries may read the exact saved result, never get a new principal. + policy = f.spec('identity.token_rotate', version=2, effect='transaction') + req = f.request( + name='identity.token_rotate', version=2, token=True, request_id='rotation-fixed' + ) + result = OperationResult( + request_id=req.request_id, + operation='identity.token_rotate', + status='ok', + actor=f.users['alice'], + subject=f.users['alice'], + committed_at=NOW, + data={'previous_credential': 't_fixture', 'credential_id': 't_next'}, + ) + successor = replace(f.credentials['token'], id='t_next') + f.conn.execute( + 'INSERT INTO credentials VALUES (?,?,?)', + ('t_next', f.users['alice'], canonical(successor).decode()), + ) + f.conn.execute( + 'INSERT INTO results VALUES (?,?,?,?)', + (f.users['alice'], req.request_id, req.payload_digest, canonical(result).decode()), + ) + with f.body_change('credentials', 't_fixture', replace(f.credentials['token'], revoked_at=NOW)): + await check.auth_case('rotation_exact_replay', req, policy, replay=True) + await check.auth_case( + 'rotation_new_request_denied', + f.request(name='identity.token_rotate', version=2, token=True, request_id='other'), + policy, + ) + changed = f.request( + name='identity.token_rotate', + version=2, + token=True, + request_id=req.request_id, + arguments={'new': 'content'}, + ) + await check.auth_case('rotation_digest_conflict', changed, policy) + with f.body_change('credentials', 't_next', replace(successor, revoked_at=NOW)): + await check.auth_case('rotation_dead_successor', req, policy) + check.compare( + 'ledger_lookup', + check.native({ + 'action': 'result', + 'subject': f.users['alice'], + 'id': req.request_id, + 'digest': req.payload_digest, + }), + ok({'result': wire(result)}), + ) + check.compare( + 'ledger_subject_isolation', + check.native({ + 'action': 'result', + 'subject': f.users['bob'], + 'id': req.request_id, + 'digest': req.payload_digest, + }), + ok({'result': None}), + ) + check.compare( + 'ledger_anonymous_isolation', + check.native({ + 'action': 'result', + 'subject': None, + 'id': req.request_id, + 'digest': req.payload_digest, + }), + ok({'result': None}), + ) + + recovery_secret = b64(bytes(range(32, 64))) + recovery_args = { + 'credential_id': 't_fixture', + 'recovery_secret': recovery_secret, + 'original_request_id': 'original-issue', + } + recover = f.request( + name='identity.token_recover', arguments=recovery_args, request_id='recovery-fixed' + ) + recover = replace(recover, proof=None) + policy = f.spec('identity.token_recover', effect='transaction') + f.conn.execute( + 'INSERT INTO token_deliveries VALUES (?,?,?,?,?,?,?,?)', + ( + 't_fixture', + f.users['alice'], + 'original-issue', + 'unused', + recovery_verifier(recovery_secret), + wire(NOW + timedelta(seconds=60)), + None, + None, + ), + ) + await check.auth_case('token_recovery', recover, policy) + for key, value in [ + ('recovery_secret', b64(bytes(32))), + ('original_request_id', 'other'), + ('recovery_secret', b64(bytes(31))), + ]: + bad = f.request(name='identity.token_recover', arguments={**recovery_args, key: value}) + await check.auth_case('token_recovery_binding', replace(bad, proof=None), policy) + for field, value in [('recovery_expires_at', wire(NOW)), ('consumed_at', wire(NOW))]: + old = f.conn.execute(f'SELECT {field} FROM token_deliveries').fetchone()[0] + f.conn.execute(f'UPDATE token_deliveries SET {field}=?', (value,)) + await check.auth_case('token_recovery_state', recover, policy) + f.conn.execute(f'UPDATE token_deliveries SET {field}=?', (old,)) + previous = replace(result, request_id=recover.request_id, operation=recover.operation) + f.conn.execute( + 'INSERT INTO results VALUES (?,?,?,?)', + ( + f.users['alice'], + recover.request_id, + recover.payload_digest, + canonical(previous).decode(), + ), + ) + f.conn.execute('UPDATE token_deliveries SET consumed_at=?', (wire(NOW),)) + with f.body_change('credentials', 't_fixture', replace(f.credentials['token'], revoked_at=NOW)): + await check.auth_case('token_recovery_replay', recover, policy, replay=True) + f.conn.execute('DELETE FROM token_deliveries') + + # WAL snapshots cannot combine old credential rows with new revocation facts. + f.setting('snapshot_probe', 'old') + check.send({'action': 'hold_snapshot'}) + check.compare('snapshot_before', check.receive(), ok({'before': 'old'})) + f.setting('snapshot_probe', 'new') + check.send({'resume': True}) + check.compare('snapshot_stable', check.receive(), ok({'after': 'old'})) + check.compare( + 'snapshot_fresh', + check.native({'action': 'setting', 'key': 'snapshot_probe'}), + ok({'exists': True, 'value': 'new'}), + ) + f.conn.execute('UPDATE resources SET parent=? WHERE id=?', ('r_root', 'r_root')) + # Body parent links, not the incidental SQL column, are the Python authority view. + with f.body_change('resources', 'r_area', replace(f.resources['r_area'], parent='r_file')): + command = { + 'action': 'scope', + 'scope': wire(Scope(resource_id='r_other', descendants=True)), + 'id': 'r_file', + } + check.compare('parent_cycle', check.native(command), {'ok': False, 'code': 'parent_cycle'}) + f.conn.execute('UPDATE resources SET parent=NULL WHERE id=?', ('r_root',)) + + # Runtime generation mismatch remains latched even when the old value returns. + policy = f.spec() + with f.setting_change('recovery_runtime_generation', 'after-restore'): + await check.auth_case('runtime_stale', request, policy) + await check.auth_case('runtime_stale_latched', request, policy) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('--binary', required=True, type=Path) + parser.add_argument('--report', required=True, type=Path) + args = parser.parse_args() + asyncio.run(run(args.binary.resolve(), args.report)) diff --git a/rust/tests/test_postgres_parity.py b/rust/tests/test_postgres_parity.py new file mode 100644 index 00000000..dc1fc03c --- /dev/null +++ b/rust/tests/test_postgres_parity.py @@ -0,0 +1,645 @@ +"""Native PostgreSQL writes against the real Python PostgreSQL implementation. + +Only new, randomly named databases on an explicitly selected local test server +are used. SQL inputs in the native helper are compiled constants, not JSON data. +""" + +from __future__ import annotations + +import argparse +import asyncio +import json +import os +import subprocess +from collections import Counter +from dataclasses import replace +from pathlib import Path + +import psycopg +from postgres_fixture import database +from psycopg import sql +from psycopg.conninfo import conninfo_to_dict +from test_storage_parity import ( + NOW, + TABLES as METADATA_TABLES, + D, + action, + audit, + insert, + metadata_cases, + python_steps as base_python_steps, + resource, + revision, +) + +from msg.core.codec import canonical, decode, wire +from msg.core.errors import Failure +from msg.core.identifiers import hex_id +from msg.core.models import ( + BlobRef, + Certificate, + CertificateRequest, + CertificateRequestState, + Credential, + EffectJob, + EmailSettings, + Membership, + Organization, + Principal, + ResourceRef, + Signature, + Subject, + TransferChunk, + TransferSession, +) +from msg.storage.postgres import PostgresMetadataStore + +TABLES = ( + *METADATA_TABLES, + 'identities', + 'memberships', + 'emails', + 'credentials', + 'csrs', + 'certificates', + 'transfers', + 'chunks', + 'jobs', +) +REPORT = [] + + +def check(name, okay, detail=None, group='postgres_parity'): + REPORT.append({ + 'name': name, + 'group': group, + 'ok': bool(okay), + **({'detail': detail} if not okay else {}), + }) + + +def require_disposable(dsn): + if not conninfo_to_dict(dsn)['dbname'].startswith('msg_rust_test_'): + raise RuntimeError('Refusing to modify any existing application database') + + +def reset(dsn, baseline): + require_disposable(dsn) + with psycopg.connect(dsn) as c: + c.execute( + sql.SQL('TRUNCATE {} RESTART IDENTITY CASCADE').format( + sql.SQL(',').join(map(sql.Identifier, TABLES)) + ) + ) + for table, (columns, rows) in baseline.items(): + query = sql.SQL('INSERT INTO {} ({}) VALUES ({})').format( + sql.Identifier(table), + sql.SQL(',').join(map(sql.Identifier, columns)), + sql.SQL(',').join(sql.Placeholder() for _ in columns), + ) + for row in rows: + c.execute(query, row) + + +def snapshot(dsn): + with psycopg.connect(dsn) as c: + result = {} + for table in TABLES: + rows = c.execute(sql.SQL('SELECT * FROM {}').format(sql.Identifier(table))) + result[table] = ([x.name for x in rows.description], sorted(rows.fetchall(), key=repr)) + return result + + +async def python_steps(tx, steps, store, effects): + out = [] + classes = { + 'subject': Subject, + 'organization': Organization, + 'membership': Membership, + 'email': EmailSettings, + } + for step in steps: + kind = step['action'] + if kind == 'identity': + await tx.update_identity( + decode(classes[step['record_kind']], step['record']), step['expected'] + ) + elif kind == 'subject': + out.append(wire(await tx.subject(step['id']))) + elif kind == 'credential': + await tx.save_credential(decode(Credential, step['record']), step['expected']) + elif kind == 'get_credential': + out.append(wire(await tx.credential(step['id']))) + elif kind == 'csr': + await tx.save_csr(decode(CertificateRequest, step['record'])) + elif kind == 'csr_state': + out.append(wire(await tx.csr_state(step['id']))) + elif kind == 'transition_csr': + await tx.transition_csr( + decode(CertificateRequestState, step['record']), step['expected'] + ) + elif kind == 'certificate': + await tx.register_certificate( + decode(Certificate, step['record']), step['csr'], step.get('expected', 0) + ) + elif kind == 'revoke_certificate': + await tx.revoke_certificate(step['id'], decode(type(audit()), step['event'])) + elif kind == 'get_certificate': + out.append(wire(await tx.certificate(step['id']))) + elif kind == 'transfer': + await tx.save_transfer(decode(TransferSession, step['record']), step['expected']) + elif kind == 'get_transfer': + out.append(wire(await tx.transfer(step['id']))) + elif kind == 'chunk': + await tx.put_chunk(decode(TransferChunk, step['record'])) + elif kind == 'missing': + out.append(wire(await tx.missing_ranges(step['id'], step['cursor'], step['limit']))) + elif kind == 'enqueue': + await tx.enqueue(decode(EffectJob, step['record'])) + elif kind == 'save_job': + await tx.save_job(decode(EffectJob, step['record'])) + elif kind == 'job': + out.append(wire(await tx.job(step['id']))) + elif kind == 'path': + out.append(await tx.path(step['id'])) + elif kind == 'resolve': + out.append( + await ( + tx.resolve_migrated(step['path']) + if step.get('migrated') + else tx.resolve(step['path']) + ) + ) + elif kind == 'children': + out.append(wire(await tx.children(step['id'], step['cursor'], step['limit']))) + elif kind == 'history': + out.append(wire(await tx.history(step['id'], step['cursor'], step['limit']))) + elif kind == 'get_revision': + out.append(wire(await tx.revision(decode(ResourceRef, step['reference'])))) + elif kind == 'savepoint': + try: + async with store.transaction(write=True) as inner: + data = await python_steps(inner, step['steps'], store, effects) + out.append(data) + except Failure as exc: + if not step.get('recover'): + raise + out.append({'error': exc.code}) + else: + out.extend(await base_python_steps(tx, [step], store, effects)) + return out + + +async def run_python(store, steps): + effects = [] + try: + async with store.transaction(write=True) as tx: + out = await python_steps(tx, steps, store, effects) + return {'status': 'ok', 'values': out}, effects + except Failure as exc: + return {'status': 'error', 'code': exc.code, 'cleanup': []}, effects + + +class Native: + def __init__(self, binary): + self.binary = str(binary) + + def start(self, dsn, steps, **kwargs): + env = {**os.environ, 'MSG_PARITY_POSTGRES_DSN': dsn} + p = subprocess.Popen( + [self.binary], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + env=env, + ) + p.stdin.write(json.dumps({'steps': steps, **kwargs}, ensure_ascii=False) + '\n') + p.stdin.flush() + return p + + def run(self, dsn, steps, **kwargs): + p = self.start(dsn, steps, **kwargs) + stdout, stderr = p.communicate(timeout=30) + if p.returncode: + raise AssertionError('Native helper failed: ' + stderr) + lines = [json.loads(s) for s in stdout.splitlines()] + result = lines[-1] + if result.get('uncertain') is False: + result.pop('uncertain') + return result, [item['effect'] for item in lines if 'effect' in item] + + +def extended_cases(): + subject = Subject( + resource_id='u_alice', kind='registered', primary_group='g_public', auth_version=0 + ) + organization = Organization(resource_id='g_team', membership_version=0) + membership = Membership( + organization_id='g_team', subject_id='u_alice', role='member', version=0 + ) + email = EmailSettings( + subject_id='u_alice', address='public-test@example.invalid', verified_at=None + ) + credential = Credential( + id='key_a', + subject_id='u_alice', + kind='signing_key', + verifier=bytes([3]) * 32, + ceiling=(), + not_before=NOW, + expires_at=None, + revoked_at=None, + ) + sig = Signature(key_id='key_a', algorithm='Ed25519', value=bytes([1]) * 64) + csr = CertificateRequest( + resource_id='csr_a', + applicant='u_alice', + subject_id='u_alice', + requested_issuer='u_root', + public_key=credential.verifier, + kind='capability', + grants=(), + issuance=None, + requested_ttl_seconds=3600, + target_service='https://fixture.invalid', + delegation_depth=0, + authority_sources=(), + request_digest=D, + possession_proof=sig, + ) + cert = Certificate( + resource_id='cert_a', + serial='fixture', + subject_id='u_alice', + key_id='key_a', + issuer_id='u_root', + parent_certificate_id=None, + authority_sources=(), + kind='capability', + grants=(), + not_before=NOW, + expires_at=NOW.replace(year=2027), + target_service='https://fixture.invalid', + delegation_depth=0, + issuance=None, + signature=sig, + ) + transfer = TransferSession( + id='transfer', + subject_id='u_alice', + direction='upload', + state='open', + target=None, + expected_size=100, + expected_digest=D, + expires_at=NOW.replace(year=2027), + generation=0, + ) + chunk = TransferChunk( + transfer_id='transfer', + offset=10, + content=BlobRef(digest=D, size=20, media_type='text/plain'), + ) + principal = Principal( + actor='u_alice', + subject='u_alice', + credential_id='key_a', + method='signature', + ceiling=(), + certificates=(), + ) + job = EffectJob( + id='job', + event_id='event', + kind='tool', + dedupe_key='dedupe', + principal=principal, + operation='tool.run@1', + arguments={}, + state='pending', + attempts=0, + next_attempt_at=NOW, + lease_until=None, + ) + + def identity(record, kind='subject', expected=-1): + return action('identity', record_kind=kind, record=record, expected=expected) + + cases = [] + for kind, r, field in [ + ('subject', subject, 'auth_version'), + ('organization', organization, 'membership_version'), + ('membership', membership, 'version'), + ]: + cases.extend([ + (kind + '-insert', [identity(r, kind)]), + (kind + '-update', [identity(r, kind), identity(replace(r, **{field: 1}), kind, 0)]), + (kind + '-conflict', [identity(r, kind), identity(replace(r, **{field: 2}), kind, 1)]), + ]) + cases.extend([ + ('subject-read', [identity(subject), action('subject', id='u_alice')]), + ( + 'email-update', + [ + identity(email, 'email'), + identity( + replace(email, verified_at=NOW, enabled_events=frozenset({'reply', 'message'})), + 'email', + 0, + ), + ], + ), + ( + 'credential-read', + [ + identity(subject), + action('credential', record=credential, expected=0), + action('get_credential', id=credential.id), + ], + ), + ( + 'credential-revoke', + [ + identity(subject), + action('credential', record=credential, expected=0), + action('credential', record=replace(credential, revoked_at=NOW), expected=0), + ], + ), + ( + 'credential-version', + [identity(subject), action('credential', record=credential, expected=1)], + ), + ( + 'credential-immutable', + [ + identity(subject), + action('credential', record=credential, expected=0), + action( + 'credential', record=replace(credential, verifier=bytes([9]) * 32), expected=0 + ), + ], + ), + ('csr-pending', [action('csr', record=csr), action('csr_state', id=csr.resource_id)]), + ( + 'csr-rejected', + [ + action('csr', record=csr), + action( + 'transition_csr', + record=CertificateRequestState( + request_id=csr.resource_id, status='rejected', generation=1 + ), + expected=0, + ), + ], + ), + ( + 'certificate-with-csr', + [ + action('csr', record=csr), + action('certificate', record=cert, csr=csr.resource_id, expected=0), + action('csr_state', id=csr.resource_id), + action('get_certificate', id=cert.resource_id), + ], + ), + ( + 'certificate-without-csr', + [ + action('certificate', record=cert, csr=None), + action('get_certificate', id=cert.resource_id), + ], + ), + ( + 'certificate-csr-failed', + [ + action('csr', record=csr), + action('certificate', record=cert, csr=csr.resource_id, expected=1), + ], + ), + ( + 'certificate-revoked', + [ + action('certificate', record=cert, csr=None), + action('revoke_certificate', id=cert.resource_id, event=audit()), + ], + ), + ( + 'transfer-roundtrip', + [ + action('transfer', record=transfer, expected=None), + action('get_transfer', id=transfer.id), + ], + ), + ( + 'transfer-transition', + [ + action('transfer', record=transfer, expected=None), + action( + 'transfer', + record=replace( + transfer, + state='sealed', + generation=1, + output=ResourceRef(id='r_doc', revision='v1'), + ), + expected=0, + ), + ], + ), + ( + 'transfer-conflict', + [ + action('transfer', record=transfer, expected=None), + action('transfer', record=replace(transfer, generation=2), expected=1), + ], + ), + ( + 'chunk-repeat', + [ + action('transfer', record=transfer, expected=None), + action('chunk', record=chunk), + action('chunk', record=chunk), + ], + ), + ( + 'chunk-conflict', + [ + action('transfer', record=transfer, expected=None), + action('chunk', record=chunk), + action('chunk', record=replace(chunk, offset=20)), + ], + ), + ( + 'chunk-content-conflict', + [ + action('transfer', record=transfer, expected=None), + action('chunk', record=chunk), + action( + 'chunk', + record=replace( + chunk, content=replace(chunk.content, digest='sha256:' + 'a' * 64) + ), + ), + ], + ), + ('job-roundtrip', [action('enqueue', record=job), action('job', id=job.id)]), + ( + 'job-update', + [ + action('enqueue', record=job), + action( + 'save_job', + record=replace( + job, state='running', attempts=1, lease_until=NOW.replace(hour=1) + ), + ), + action('job', id=job.id), + ], + ), + ( + 'job-dedupe', + [action('enqueue', record=job), action('enqueue', record=replace(job, id='other'))], + ), + ]) + for cursor in (None, '0', '30', '100'): + for limit in (1, 2, 50): + cases.append(( + f'missing-{cursor}-{limit}', + [ + action('transfer', record=transfer, expected=None), + action('chunk', record=chunk), + action('chunk', record=replace(chunk, offset=50)), + action('missing', id=transfer.id, cursor=cursor, limit=limit), + ], + )) + for path, migrated in [ + ('/', False), + ('/document', False), + ('/_id/r_doc', False), + ('/_id/' + hex_id('r_doc'), False), + ('/*' + hex_id('r_doc'), False), + ('/document', True), + ]: + cases.append(( + 'resolve-' + path, + [insert(), action('resolve', path=path, migrated=migrated)], + )) + cases.extend([ + ('path', [insert(), action('path', id='r_doc')]), + ('children', [insert(), action('children', id='r_root', cursor=None, limit=1)]), + ( + 'history', + [ + insert(), + action('revision', record=revision()), + action('history', id='r_doc', cursor=None, limit=1), + ], + ), + ( + 'revision-read', + [ + insert(), + action('revision', record=revision()), + action('get_revision', reference=ResourceRef(id='r_doc', revision='v1')), + ], + ), + ( + 'revision-hex', + [ + insert(), + action('revision', record=revision()), + action( + 'get_revision', reference=ResourceRef(id=hex_id('r_doc'), revision=hex_id('v1')) + ), + ], + ), + ( + 'read-after-rename', + [ + insert(), + action('replace', record=replace(resource(), name='new', generation=1), expected=0), + action('resolve', path='/document', migrated=True), + action('resolve', path='/new', migrated=False), + ], + ), + ]) + return cases + + +async def run(args): + native = Native(args.binary.resolve()) + with database() as py_dsn, database() as rs_dsn: + py_store = PostgresMetadataStore(py_dsn) + rs_store = PostgresMetadataStore(rs_dsn) + for store in [py_store, rs_store]: + async with store.transaction(write=True) as tx: + await tx.insert(resource('r_root', parent=None, name='')) + baseline = snapshot(py_dsn) + try: + for name, steps in metadata_cases() + extended_cases(): + reset(py_dsn, baseline) + reset(rs_dsn, baseline) + expected, py_effects = await run_python(py_store, steps) + actual, rs_effects = native.run(rs_dsn, steps) + py_db, rs_db = snapshot(py_dsn), snapshot(rs_dsn) + different = [t for t in TABLES if py_db[t] != rs_db[t]] + check( + name, + canonical(actual) == canonical(expected) + and not different + and py_effects == rs_effects, + { + 'python': expected, + 'rust': actual, + 'tables': different, + 'effects': [py_effects, rs_effects], + }, + ) + reset(rs_dsn, baseline) + actual, _ = native.run( + rs_dsn, [action('set', key='read-only', value=True)], read_only=True + ) + check( + 'read-only-write-rejected', + actual.get('code') == 'read_only_transaction' and snapshot(rs_dsn) == baseline, + actual, + group='postgres_safety', + ) + actual, _ = native.run( + rs_dsn, + [ + action('set', key='before', value=True), + action('ignore', steps=[insert(resource(parent='absent'))]), + ], + ) + check( + 'ignored-write-error-cannot-commit', + actual.get('status') == 'error' and snapshot(rs_dsn) == baseline, + actual, + group='postgres_safety', + ) + finally: + await py_store.close() + await rs_store.close() + report = { + 'backend': 'postgresql', + 'checks': len(REPORT), + 'failures': sum(not x['ok'] for x in REPORT), + 'groups': dict(Counter(x['group'] for x in REPORT)), + 'cases': REPORT, + } + args.report.write_text(json.dumps(report, ensure_ascii=False, indent=2) + '\n') + print(json.dumps({k: v for k, v in report.items() if k != 'cases'})) + return report['failures'] + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('--binary', type=Path, required=True) + parser.add_argument('--report', type=Path, required=True) + raise SystemExit(bool(asyncio.run(run(parser.parse_args())))) + + +if __name__ == '__main__': + main() diff --git a/rust/tests/test_postgres_process.py b/rust/tests/test_postgres_process.py new file mode 100644 index 00000000..9311145e --- /dev/null +++ b/rust/tests/test_postgres_process.py @@ -0,0 +1,268 @@ +"""Real process/transaction failure checks against a disposable PostgreSQL DB.""" + +from __future__ import annotations + +import argparse +import asyncio +import json +import selectors +import time +from collections import Counter +from pathlib import Path + +import psycopg +from postgres_fixture import database +from postgres_transport_fixture import LoseCommitAck +from test_postgres_parity import REPORT, Native, check, reset, snapshot +from test_storage_parity import action, audit, event, insert, ledger, once, resource + +from msg.storage.postgres import PostgresMetadataStore + + +def line(p): + with selectors.DefaultSelector() as selected: + selected.register(p.stdout, selectors.EVENT_READ) + if not selected.select(timeout=15): + raise AssertionError('missing process checkpoint') + return json.loads(p.stdout.readline()) + + +def finish(p): + p.stdin.write('continue\n') + p.stdin.flush() + out, err = p.communicate(timeout=15) + if p.returncode: + raise AssertionError('native process failed: ' + err) + return json.loads(out.splitlines()[-1]) + + +def kill(p): + if p.poll() is None: + p.kill() + p.communicate(timeout=10) + + +def waiting_writer(dsn): + deadline = time.monotonic() + 5 + with psycopg.connect(dsn, autocommit=True) as c: + while time.monotonic() < deadline: + if c.execute( + "SELECT EXISTS(SELECT 1 FROM pg_stat_activity WHERE datname=current_database() AND application_name='msgd-native' AND wait_event_type='Lock' AND wait_event='advisory')" + ).fetchone()[0]: + return True + time.sleep(0.01) + return False + + +async def run(args): + native = Native(args.binary.resolve()) + with database() as dsn: + store = PostgresMetadataStore(dsn) + async with store.transaction(write=True) as tx: + await tx.insert(resource('r_root', parent=None, name='')) + baseline = snapshot(dsn) + body = [insert(), action('event', record=event()), action('audit', record=audit())] + try: + for stage in ['before', 'after']: + reset(dsn, baseline) + p = native.start( + dsn, + [once(body + ([action('hold')] if stage == 'before' else []))], + hold_after_commit=stage == 'after', + ) + try: + got = line(p) + visible = snapshot(dsn) + check( + 'kill-' + stage + '-visibility', + got == {'checkpoint': 'transaction' if stage == 'before' else 'committed'} + and bool(visible['results'][1]) == (stage == 'after'), + got, + group='postgres_process', + ) + finally: + kill(p) + actual, _ = native.run(dsn, [once(body)]) + after = snapshot(dsn) + check( + 'kill-' + stage + '-retry-once', + actual['status'] == 'ok' + and len(after['resources'][1]) == 2 + and all(len(after[t][1]) == 1 for t in ['results', 'events', 'audit']), + actual, + group='postgres_process', + ) + reset(dsn, baseline) + first = native.start(dsn, [once(body + [action('hold')])]) + second = None + try: + check( + 'first-writer-acquired', + line(first) == {'checkpoint': 'transaction'}, + group='postgres_process', + ) + second = native.start(dsn, [once(body)]) + blocked = waiting_writer(dsn) + first_result = finish(first) + out, err = second.communicate(timeout=15) + check( + 'concurrent-writers-one-commit', + blocked + and second.returncode == 0 + and first_result == json.loads(out.splitlines()[-1]) + and len(snapshot(dsn)['audit'][1]) == 1, + err, + group='postgres_process', + ) + finally: + kill(first) + if second is not None: + kill(second) + reset(dsn, baseline) + second = None + async with store.transaction(write=True) as tx: + tx.set_setting('python-held', True) + second = native.start(dsn, [action('set', key='native-held', value=True)]) + check('python-writer-blocks-native', waiting_writer(dsn), group='postgres_process') + try: + out, err = second.communicate(timeout=15) + check( + 'native-continues-after-python-commit', + json.loads(out.splitlines()[-1])['status'] == 'ok', + err, + group='postgres_process', + ) + finally: + kill(second) + p = native.start(dsn, [action('set', key='native-held', value=True), action('hold')]) + try: + line(p) + + async def python_contender(): + async with store.transaction(write=True) as tx: + tx.set_setting('python-after', True) + + task = asyncio.create_task(python_contender()) + await asyncio.sleep(0.1) + check('native-writer-blocks-python', not task.done(), group='postgres_process') + finish(p) + await asyncio.wait_for(task, 15) + finally: + kill(p) + reset(dsn, baseline) + p = native.start( + dsn, + [ + action('effect', name='cleanup', hold=True), + action('set', key='abort', value=True), + action('fail'), + ], + ) + try: + got = line(p) + checkpoint = json.loads(p.stdout.readline()) + with psycopg.connect(dsn, autocommit=True) as c: + lock = c.execute( + 'SELECT pg_try_advisory_lock(725274758,1886265951)' + ).fetchone()[0] + if lock: + c.execute('SELECT pg_advisory_unlock(725274758,1886265951)') + check( + 'fence-survives-sql-rollback', + got == {'effect': 'cleanup'} + and checkpoint == {'checkpoint': 'compensation'} + and not lock + and snapshot(dsn) == baseline, + group='postgres_process', + ) + check( + 'compensation-completes', + finish(p).get('code') == 'fixture_failure', + group='postgres_process', + ) + finally: + kill(p) + reset(dsn, baseline) + with LoseCommitAck(dsn) as proxy: + actual, effects = native.run( + proxy.dsn, [action('effect', name='must-not-delete'), *body, ledger()] + ) + check( + 'lost-commit-response-is-uncertain', + proxy.dropped.is_set() + and actual.get('code') == 'storage_commit_uncertain' + and actual.get('uncertain') is True + and not effects, + actual, + group='postgres_process', + ) + after = snapshot(dsn) + actual, _ = native.run(dsn, [once(body)]) + check( + 'lost-commit-response-retry-no-duplicate', + actual['status'] == 'ok' and after == snapshot(dsn) and len(after['audit'][1]) == 1, + actual, + group='postgres_process', + ) + # Every schema mutation is isolated to this new test database and + # reverted, so later tests never inherit a deliberately broken DB. + reset(dsn, baseline) + schema_changes = [ + ( + 'missing-dedupe', + 'ALTER TABLE jobs DROP CONSTRAINT jobs_dedupe_key', + 'ALTER TABLE jobs ADD CONSTRAINT jobs_dedupe_key UNIQUE(dedupe)', + ), + ( + 'missing-root-index', + 'DROP INDEX one_root', + 'CREATE UNIQUE INDEX one_root ON resources((1)) WHERE parent IS NULL', + ), + ( + 'wrong-primary-key', + 'ALTER TABLE results DROP CONSTRAINT results_pkey; ALTER TABLE results ADD PRIMARY KEY(subject)', + 'ALTER TABLE results DROP CONSTRAINT results_pkey; ALTER TABLE results ADD PRIMARY KEY(subject,request_id)', + ), + ( + 'missing-foreign-key', + 'ALTER TABLE revisions DROP CONSTRAINT revisions_resource_id_fkey', + 'ALTER TABLE revisions ADD CONSTRAINT revisions_resource_id_fkey FOREIGN KEY(resource_id) REFERENCES resources(id)', + ), + ] + for name, break_sql, repair_sql in schema_changes: + with psycopg.connect(dsn, autocommit=True) as c: + c.execute(break_sql) + try: + actual, _ = native.run(dsn, [action('set', key='must-not-write', value=True)]) + check( + name + '-rejected', + actual.get('code') == 'unsupported_storage_schema' + and snapshot(dsn) == baseline, + actual, + group='postgres_schema', + ) + finally: + with psycopg.connect(dsn, autocommit=True) as c: + c.execute(repair_sql) + finally: + await store.close() + report = { + 'checks': len(REPORT), + 'failures': sum(not r['ok'] for r in REPORT), + 'groups': dict(Counter(r['group'] for r in REPORT)), + 'cases': REPORT, + } + args.report.write_text(json.dumps(report, ensure_ascii=False, indent=2) + '\n') + print(json.dumps({k: v for k, v in report.items() if k != 'cases'})) + return report['failures'] + + +def main(): + p = argparse.ArgumentParser() + p.add_argument('--binary', type=Path, required=True) + p.add_argument('--report', type=Path, required=True) + raise SystemExit(bool(asyncio.run(run(p.parse_args())))) + + +if __name__ == '__main__': + main() diff --git a/rust/tests/test_python_parity.py b/rust/tests/test_python_parity.py new file mode 100644 index 00000000..e8394414 --- /dev/null +++ b/rust/tests/test_python_parity.py @@ -0,0 +1,319 @@ +"""Run native wire code against the retained Python implementation, not a copy. + +Usage: PYTHONPATH=src python rust/tests/test_python_parity.py --binary PATH +Only deterministic public test keys are used. No server, database, or credentials. +""" + +from __future__ import annotations + +import argparse +import copy +import json +import math +import random +import struct +import subprocess +from collections import Counter +from datetime import UTC, datetime +from pathlib import Path + +from msg.core.codec import b64, canonical, decode, digest, loads, unb64, wire +from msg.core.models import Signature +from msg.core.packet import decode_packet +from msg.core.requests import payload_fields, request_for, signing_bytes +from msg.security.crypto import Ed25519Signer, key_id, subject_id, verify + +# This seed is public test material, shared with the Rust example only. +SIGNER = Ed25519Signer.from_bytes(bytes(range(32))) +CASES: list[tuple[str, dict, dict]] = [] + + +def add(group: str, command: dict, data: dict | None = None) -> None: + expected = {'ok': False} if data is None else {'ok': True, 'data': data} + CASES.append((group, command, expected)) + + +def canonical_case(raw: str) -> None: + command = {'action': 'canonical', 'raw': raw} + try: + encoded = canonical(loads(raw)) + except ValueError, TypeError, RecursionError: + add('canonical_reject', command) + else: + add('canonical', command, {'canonical': encoded.decode(), 'digest': digest(encoded)}) + + +def request_case(raw: str, *, public: bool = False) -> None: + command = {'action': 'request', 'raw': raw} + if public: + command['public'] = b64(SIGNER.public_key) + try: + packet = decode_packet(raw) + except ValueError, TypeError, RecursionError: + add('request_reject', command) + return + correct_digest = digest(payload_fields(packet)) + verified = None + if public: + verified = False + if packet.payload_digest == correct_digest and hasattr(packet.proof, 'signature'): + try: + verify( + SIGNER.public_key, + signing_bytes(packet), + packet.proof.signature, + purpose='request', + ) + except ValueError, TypeError: + pass + else: + verified = True + add( + 'request', + command, + { + 'canonical': canonical(packet).decode(), + 'payload_digest': correct_digest, + 'signing': b64(signing_bytes(packet)), + 'digest_valid': packet.payload_digest == correct_digest, + 'verified': verified, + }, + ) + + +def build_cases() -> None: + raw_values = [ + 'null', + 'true', + 'false', + '0', + '-0', + '-0.0', + '0.0', + '1e-7', + '1e-6', + '1e-5', + '1e-4', + '1e15', + '1e16', + '1e20', + '1e23', + '5e-324', + '-5e-324', + '1.7976931348623157e308', + '2.2250738585072014e-308', + '1e-999', + '-1e-999', + '184467440737095516160000000', + '-184467440737095516160000000', + '9' * 4300, + r'{"a":1,"\u0061":2}', + r'[{"nested":1,"nested":2}]', + 'NaN', + 'Infinity', + '-Infinity', + '1e999', + '[1,]', + '01', + 'true false', + r'"\ud800"', + r'"\udfff"', + '9' * 4301, + ] + raw_values += [ + json.dumps(value, ensure_ascii=True) + for value in [ + {'😀': 1, '\ue000': 2, 'a': '中文\n'}, + {'z': [], 'a': {'b': None}}, + ''.join(chr(i) for i in range(32)) + '"/\\\u007f\u2028\u2029', + {'é': 1, 'e\u0301': 2, '𐀀': 3}, + [True, 1, 1.0, -0.0], + ] + ] + for raw in raw_values: + canonical_case(raw) + rng = random.Random(20261005) + for _ in range(16_384): + value = struct.unpack('!d', rng.getrandbits(64).to_bytes(8, 'big'))[0] + if math.isfinite(value): + canonical_case(repr(value)) + for power in range(-323, 309): + value = float(f'1e{power}') + for neighbor in [value, math.nextafter(value, 0.0), math.nextafter(value, math.inf)]: + if math.isfinite(neighbor): + canonical_case(repr(neighbor)) + for size in [0, 1, 2, 3, 31, 32, 64, 1024]: + encoded = b64(bytes(rng.getrandbits(8) for _ in range(size))) + add( + 'base64', + {'action': 'base64', 'raw': encoded}, + {'encoded': encoded, 'digest': digest(unb64(encoded))}, + ) + for raw in ['YQ=', 'YQ==', 'YR', '+/', 'Y Q', 'a', '中文', '\nYQ']: + try: + unb64(raw) + except ValueError: + add('base64_reject', {'action': 'base64', 'raw': raw}) + else: + raise AssertionError('invalid base64 reference case was accepted') + + for purpose in ['request', 'receipt', 'revision', 'temporary-key-possession-v1']: + for payload in [b'', b'hello', '你好\x00世界'.encode(), canonical({'n': 1e-7, 'i': 2**80})]: + signature = SIGNER.sign(payload, purpose=purpose) + add( + 'sign', + {'action': 'sign', 'payload': b64(payload), 'purpose': purpose}, + { + 'public': b64(SIGNER.public_key), + 'key_id': key_id(SIGNER.public_key), + 'subject_id': subject_id(SIGNER.public_key), + 'signature': wire(signature), + }, + ) + command = { + 'action': 'verify', + 'public': b64(SIGNER.public_key), + 'payload': b64(payload), + 'signature': wire(signature), + 'purpose': purpose, + } + add('verify', command, {'verified': True}) + add('verify_reject', dict(command, payload=b64(payload + b'!'))) + add('verify_reject', dict(command, purpose=purpose + '-wrong')) + add('verify_reject', dict(command, public=b64(bytes(32)))) + + request = request_for( + 'discovery.get', + {'id': 'r_agents', 'fields': ['id', 'revision', 'content'], 'n': 1e-7, 'big': 2**90}, + 'https://msg.lmm.best', + signer=SIGNER, + subject=subject_id(SIGNER.public_key), + request_id='rust-wire-public-fixture', + expires_at=datetime(2026, 10, 5, tzinfo=UTC), + expected=(('r_agents', 0),), + source='msg', + ) + base = wire(request) + request_case(canonical(base).decode(), public=True) + for key, value in [ + ('source', 'mcp'), + ('expires_at', '2026-10-05T00:01:00Z'), + ('request_id', 'another'), + ('target_service', 'https://other.example'), + ('arguments', {'id': 'other'}), + ('expected_generations', [['r_agents', 1]]), + ('return_fields', ['data']), + ('subject', None), + ]: + changed = copy.deepcopy(base) + changed[key] = value + request_case(canonical(changed).decode(), public=True) + unsigned = dict(base, proof=None) + unsigned.pop('source') + unsigned.pop('return_fields') + request_case(canonical(unsigned).decode()) + for timestamp in [ + '2026-10-05T00:00:00Z', + '2026-10-05T00:00:00.1Z', + '2026-10-05T00:00:00.123456789Z', + ]: + request_case(canonical(dict(base, expires_at=timestamp)).decode(), public=True) + request_case( + canonical( + dict(base, proof={'credential_id': 'c_test', 'token': b64(b'test-only')}) + ).decode() + ) + invalid = [ + ('extra', 1), + ('protocol_version', True), + ('contract_version', 1.0), + ('operation', 'no_dot'), + ('operation', 'discovery..get'), + ('operation', 'Discovery.get'), + ('request_id', ''), + ('subject', 3), + ('source', 'ssh'), + ('arguments', []), + ('expected_generations', [['r_agents', True]]), + ('expected_generations', [['r_agents', -1]]), + ('expected_generations', [['r_agents', 0], ['r_agents', 1]]), + ('return_fields', ['id'] * 31), + ('proof', {'credential_id': 'c', 'token': 'YR'}), + ('expires_at', '2026-10-05T00:00:00+00:00'), + ('payload_digest', 'sha256:bad'), + ] + for key, value in invalid: + changed = copy.deepcopy(base) + changed[key] = value + request_case(canonical(changed).decode()) + for key in ['request_id', 'protocol_version', 'proof', 'arguments']: + missing = copy.deepcopy(base) + missing.pop(key) + request_case(canonical(missing).decode()) + # Bounded Rust rollout profile is intentionally stricter, never silently truncated. + for raw in [' ' * (1_048_576 + 1), '[' * 66 + '0' + ']' * 66]: + add('bounded_reject', {'action': 'canonical', 'raw': raw}) + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument('--binary', required=True, type=Path) + parser.add_argument('--report', type=Path) + args = parser.parse_args() + build_cases() + inputs = ''.join(json.dumps(command, ensure_ascii=True) + '\n' for _, command, _ in CASES) + result = subprocess.run( + [str(args.binary.resolve())], + input=inputs, + text=True, + capture_output=True, + timeout=120, + check=True, + ) + # JSONL is delimited by LF, not Unicode U+2028/U+2029 inside strings. + if not result.stdout.endswith('\n'): + raise AssertionError('native output is missing its final LF') + outputs = result.stdout.split('\n')[:-1] + if len(outputs) != len(CASES): + raise AssertionError(f'native output count {len(outputs)} != {len(CASES)}') + failures = [] + for index, ((group, command, expected), raw) in enumerate(zip(CASES, outputs, strict=True)): + actual = json.loads(raw) + passed = actual.get('ok') is expected['ok'] + if expected['ok']: + passed = passed and actual.get('data') == expected['data'] + if not passed: + failures.append({ + 'index': index, + 'group': group, + 'input': command, + 'expected': expected, + 'actual': actual, + }) + # Explicit reverse check of signatures produced by Rust with the Python verifier. + if passed and group == 'sign': + data = actual['data'] + verify( + unb64(data['public']), + unb64(command['payload']), + decode(Signature, data['signature']), + purpose=command['purpose'], + ) + report = { + 'cases': len(CASES), + 'groups': dict(Counter(group for group, _, _ in CASES)), + 'failures': len(failures), + 'first_failures': failures[:12], + } + if args.report: + args.report.parent.mkdir(parents=True, exist_ok=True) + args.report.write_text( + json.dumps(report, ensure_ascii=True, indent=2) + '\n', encoding='utf-8' + ) + print(json.dumps(report, ensure_ascii=True)) + if failures: + raise SystemExit(1) + + +if __name__ == '__main__': + main() diff --git a/rust/tests/test_storage_parity.py b/rust/tests/test_storage_parity.py new file mode 100644 index 00000000..669f98de --- /dev/null +++ b/rust/tests/test_storage_parity.py @@ -0,0 +1,880 @@ +"""Native writes against real Python storage; all data and keys are disposable. + +This tests storage primitives, not an authenticated operation executor. The +local `once` helper deliberately models only the atomic ledger boundary. +""" + +from __future__ import annotations + +import argparse +import asyncio +import json +import selectors +import sqlite3 +import subprocess +import tempfile +from collections import Counter +from contextlib import closing +from dataclasses import replace +from datetime import UTC, datetime +from pathlib import Path + +from msg.core.codec import canonical, decode, digest, wire +from msg.core.errors import Failure +from msg.core.models import ( + AuditEvent, + BlobRef, + Event, + OperationError, + OperationResult, + Relation, + Resource, + ResourceRef, + Revision, +) +from msg.security.crypto import Ed25519Signer +from msg.storage.sqlite import SqliteMetadataStore + +NOW = datetime(2026, 10, 5, tzinfo=UTC) +D = digest({'task': 'writer-test'}) +TABLES = ( + 'resources', + 'resource_tags', + 'resource_path_aliases', + 'revisions', + 'relations', + 'settings', + 'results', + 'events', + 'audit', +) +REPORT = [] + + +def resource(id='r_doc', *, parent='r_root', name='document', **kwargs): + return Resource( + id=id, + type='file' if parent else 'space', + type_version=1, + name=name, + parent=parent, + owner='u_alice', + group='g_public', + mode=0o0644, + generation=0, + revision=None, + state='active', + created_at=NOW, + created_by='u_alice', + modified_at=NOW, + modified_by='u_alice', + **kwargs, + ) + + +def result(subject='u_alice', request_id='req', **kwargs): + return OperationResult( + request_id=request_id, + operation='fixture.write', + status='ok', + actor=subject, + subject=subject, + committed_at=NOW, + **kwargs, + ) + + +def event(id='event-1', **kwargs): + return Event( + id=id, + type='fixture.changed', + time=NOW, + request_id='req', + actor='u_alice', + subject='u_alice', + resources=(ResourceRef(id='r_doc'),), + data=kwargs, + ) + + +def audit(id='event-1'): + return AuditEvent( + event=event(id, n=10**60), + authority=(), + before_digest=None, + after_digest=D, + previous_digest='ignored by appender', + entry_digest='ignored by appender', + result='ok', + ) + + +def revision(id='v1', **kwargs): + return Revision( + format_version=1, + id=id, + resource_id='r_doc', + parents=(), + content=BlobRef(digest=D, size=123, media_type='text/plain'), + relations=(), + actor='u_alice', + subject='u_alice', + author='u_alice', + created_at=NOW, + manifest_digest=D, + **kwargs, + ) + + +def action(kind, **kwargs): + return {'action': kind, **wire(kwargs)} + + +def insert(record=None): + return action('insert', record=record or resource()) + + +def set_value(key, value=1): + return action('set', key=key, value=value) + + +def ledger(record=None, **kwargs): + record = record or result() + return action('result', record=record, subject=record.subject, digest=D, **kwargs) + + +def once(body, record=None): + record = record or result() + return action('once', subject=record.subject, digest=D, record=record, steps=body) + + +def check(name, ok, detail=None, group='parity'): + REPORT.append({ + 'name': name, + 'group': group, + 'ok': bool(ok), + **({'detail': detail} if not ok else {}), + }) + + +def dump(path): + with closing(sqlite3.connect(path)) as db: + return { + table: sorted(db.execute(f'SELECT * FROM {table}').fetchall(), key=repr) + for table in TABLES + } + + +def schema(path): + with closing(sqlite3.connect(path)) as db: + return db.execute( + 'SELECT type,name,tbl_name,sql FROM sqlite_schema ORDER BY name' + ).fetchall() + + +def backup(source, target): + with closing(sqlite3.connect(source)) as db, closing(sqlite3.connect(target)) as dst: + db.backup(dst) + + +async def fixture(path): + store = SqliteMetadataStore(path, busy_timeout=0.08) + async with store.transaction(write=True) as tx: + await tx.insert(resource('r_root', parent=None, name='')) + return store + + +async def python_steps(tx, values, store, effects): + output = [] + for step in values: + kind = step['action'] + if kind == 'set': + tx.set_setting(step['key'], step['value']) + elif kind == 'read': + output.append(tx.setting(step['key'])) + elif kind == 'resource': + output.append(wire(await tx.resource(step['id']))) + elif kind == 'insert': + await tx.insert(decode(Resource, step['record'])) + elif kind == 'replace': + await tx.replace(decode(Resource, step['record']), step['expected']) + elif kind == 'revision': + await tx.append_revision(decode(Revision, step['record'])) + elif kind == 'result': + await tx.save_result( + step['subject'], step['digest'], decode(OperationResult, step['record']) + ) + elif kind == 'lookup': + output.append( + wire(await tx.request_result(step['subject'], step['id'], step['digest'])) + ) + elif kind == 'event': + await tx.append_event(decode(Event, step['record'])) + elif kind == 'audit': + await tx.append_audit(decode(AuditEvent, step['record'])) + elif kind == 'savepoint': + try: + async with store.transaction(write=True) as inner: + data = await python_steps(inner, step['steps'], store, effects) + output.append(data) + except Failure as exc: + if not step.get('recover'): + raise + output.append({'error': exc.code}) + elif kind == 'effect': + + async def cleanup(name=step['name']): + effects.append(name) + + tx.on_rollback(cleanup) + elif kind == 'fail': + raise Failure('fixture_failure') + elif kind == 'once': + record = decode(OperationResult, step['record']) + stored = await tx.request_result(step['subject'], record.request_id, step['digest']) + if stored is None: + await python_steps(tx, step['steps'], store, effects) + await tx.save_result(step['subject'], step['digest'], record) + stored = record + output.append(wire(stored)) + else: + raise AssertionError(f'Unsupported Python fixture action: {kind}') + return output + + +async def run_python(store, steps): + effects = [] + try: + async with store.transaction(write=True) as tx: + output = await python_steps(tx, steps, store, effects) + return {'status': 'ok', 'values': output}, effects + except Failure as exc: + return {'status': 'error', 'code': exc.code, 'cleanup': []}, effects + + +class Native: + def __init__(self, binary): + self.binary = str(binary) + + def run(self, path, steps, **kwargs): + completed = subprocess.run( + [self.binary, str(path)], + input=json.dumps({'steps': steps, **kwargs}) + '\n', + capture_output=True, + text=True, + timeout=15, + check=True, + ) + lines = [json.loads(line) for line in completed.stdout.splitlines()] + return lines[-1], [line['effect'] for line in lines if 'effect' in line] + + def start(self, path, steps, **kwargs): + process = subprocess.Popen( + [self.binary, str(path)], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + process.stdin.write(json.dumps({'steps': steps, **kwargs}) + '\n') + process.stdin.flush() + return process + + +def line(process): + with selectors.DefaultSelector() as selector: + selector.register(process.stdout, selectors.EVENT_READ) + if not selector.select(10): + raise AssertionError('Native process did not reach its checkpoint') + raw = process.stdout.readline() + if not raw: + raise AssertionError('Native exited before checkpoint: ' + process.stderr.read()) + return json.loads(raw) + + +def finish(process): + process.stdin.write('continue\n') + process.stdin.flush() + stdout, stderr = process.communicate(timeout=15) + if process.returncode: + raise AssertionError(f'Native failed: {process.returncode}: {stderr}') + return json.loads(stdout.splitlines()[-1]) + + +def kill(process): + process.kill() + process.communicate(timeout=10) + + +def metadata_cases(): + doc = resource() + rev = revision() + cases = [ + ('insert-read-resource', [insert(), action('resource', id=doc.id)]), + ( + 'canonical-settings', + [ + set_value('numbers', {'n': 10**80, 'f': -0.0, 'text': '中文😀é'}), + action('read', key='numbers'), + ], + ), + ('unicode-tags', [insert(resource(tags=('café', '安全', '苹果')))]), + ( + 'rename-alias', + [ + insert(), + action('replace', record=replace(doc, name='renamed', generation=1), expected=0), + ], + ), + ( + 'tags-update', + [ + insert(), + action( + 'replace', + record=replace(doc, tags=('rust', 'storage'), generation=1), + expected=0, + ), + ], + ), + ( + 'move-alias', + [ + insert(resource('r_other', name='other')), + insert(), + action('replace', record=replace(doc, parent='r_other', generation=1), expected=0), + ], + ), + ( + 'generation-conflict', + [insert(), action('replace', record=replace(doc, generation=2), expected=1)], + ), + ( + 'invalid-generation', + [insert(), action('replace', record=replace(doc, generation=2), expected=0)], + ), + ( + 'immutable-type', + [ + insert(), + action('replace', record=replace(doc, type='topic', generation=1), expected=0), + ], + ), + ( + 'immutable-creator', + [ + insert(), + action( + 'replace', record=replace(doc, created_by='u_bob', generation=1), expected=0 + ), + ], + ), + ( + 'immutable-time', + [ + insert(), + action( + 'replace', + record=replace(doc, created_at=NOW.replace(hour=1), generation=1), + expected=0, + ), + ], + ), + ('missing-parent', [insert(resource(parent='missing'))]), + ('duplicate-child', [insert(), insert(resource('r_duplicate'))]), + ('second-root', [insert(resource('second-root', parent=None, name=''))]), + ( + 'parent-cycle', + [ + insert(), + insert(resource('r_child', parent=doc.id, name='child')), + action('replace', record=replace(doc, parent='r_child', generation=1), expected=0), + ], + ), + ('append-revision', [insert(), action('revision', record=rev)]), + ( + 'revision-chain', + [ + insert(), + action('revision', record=rev), + action( + 'revision', + record=replace( + rev, + id='v2', + parents=('v1',), + summary='changed', + source_kind='user', + source_version=2, + ), + ), + ], + ), + ( + 'revision-relations', + [ + insert(), + action( + 'revision', + record=replace( + rev, + relations=( + Relation( + type='reference', target=ResourceRef(id='r_root'), excerpt=(0, 12) + ), + ), + ), + ), + ], + ), + ( + 'revision-parent-missing', + [insert(), action('revision', record=replace(rev, parents=('missing',)))], + ), + ( + 'revision-wrong-resource', + [ + insert(), + insert(resource('r_other', name='other')), + action('revision', record=replace(rev, resource_id='r_other')), + action('revision', record=replace(rev, id='v2', parents=('v1',))), + ], + ), + ( + 'revision-duplicate', + [insert(), action('revision', record=rev), action('revision', record=rev)], + ), + ('result-read', [ledger(), action('lookup', subject='u_alice', id='req', digest=D)]), + ( + 'result-anonymous', + [ledger(result(None)), action('lookup', subject=None, id='req', digest=D)], + ), + ( + 'result-namespace', + [ + ledger(), + ledger(result('u_bob')), + action('lookup', subject='u_charlie', id='req', digest=D), + ], + ), + ('result-duplicate', [insert(), ledger(), ledger()]), + ( + 'digest-conflict', + [ + ledger(), + action('lookup', subject='u_alice', id='req', digest=digest({'different': True})), + ], + ), + ('event', [action('event', record=event())]), + ( + 'duplicate-event', + [insert(), action('event', record=event()), action('event', record=event())], + ), + ( + 'audit-chain', + [action('audit', record=audit()), action('audit', record=audit('event-2'))], + ), + ( + 'atomic-resource-event-audit-result', + [ + insert(), + action('revision', record=rev), + action('event', record=event()), + action('audit', record=audit()), + ledger(), + ], + ), + ( + 'atomic-failure-at-result', + [ + ledger(), + insert(), + action('event', record=event()), + action('audit', record=audit()), + ledger(), + ], + ), + ( + 'rollback-all', + [ + insert(), + ledger(), + action('event', record=event()), + action('audit', record=audit()), + action('fail'), + ], + ), + ( + 'savepoint-recover', + [ + set_value('before'), + action('savepoint', steps=[set_value('inner'), action('fail')], recover=True), + set_value('after'), + ], + ), + ( + 'savepoint-outer-failure', + [action('savepoint', steps=[insert(), ledger()]), action('fail')], + ), + ( + 'nested-savepoints', + [ + action( + 'savepoint', + steps=[ + set_value('first'), + action( + 'savepoint', steps=[set_value('second'), action('fail')], recover=True + ), + set_value('third'), + ], + ) + ], + ), + ( + 'rollback-effects', + [ + action('effect', name='outer'), + action('savepoint', steps=[action('effect', name='inner')]), + action('effect', name='last'), + action('fail'), + ], + ), + ( + 'savepoint-effects', + [ + action('effect', name='outer'), + action( + 'savepoint', + steps=[action('effect', name='inner'), action('fail')], + recover=True, + ), + ], + ), + ( + 'same-unit-retry', + [ + once([insert(), action('audit', record=audit())]), + once([insert(), action('audit', record=audit())]), + ], + ), + ] + for name, field, value in [ + ('owner', 'owner', 'u_bob'), + ('group', 'group', 'g_other'), + ('mode', 'mode', 0o0740), + ('state', 'state', 'archived'), + ]: + cases.append(( + f'authorization-epoch-{name}', + [ + insert(), + action('replace', record=replace(doc, generation=1, **{field: value}), expected=0), + action('read', key='authorization_epoch'), + ], + )) + for status in ('ok', 'accepted', 'error', 'uncertain'): + record = replace( + result(data={'large': 10**90, 'text': '安全'}), + status=status, + error=OperationError(code='fixture', retryable=False) if status == 'error' else None, + ) + cases.append((f'result-status-{status}', [ledger(record)])) + signer = Ed25519Signer.from_bytes(bytes([7]) * 32) + cases.append(( + 'signed-receipt-bytes', + [ledger(result(receipt=signer.sign(b'fixture-only', purpose='receipt')))], + )) + cases.append(( + 'historical-error-message', + [ + ledger( + replace( + result(), + status='error', + error=OperationError(code='fixture', retryable=True, message='可重试'), + ) + ) + ], + )) + return cases + + +async def parity_cases(native, directory): + cases = metadata_cases() + for index, (name, steps) in enumerate(cases): + py = directory / f'py-{index}.db' + rs = directory / f'rs-{index}.db' + store = await fixture(py) + backup(py, rs) + before_schema = schema(rs) + expected, py_effects = await run_python(store, steps) + actual, rs_effects = native.run(rs, steps) + same = ( + canonical(actual) == canonical(expected) + and dump(py) == dump(rs) + and py_effects == rs_effects + and schema(rs) == before_schema + ) + check( + name, + same, + { + 'python': expected, + 'rust': actual, + 'tables_differ': [t for t in TABLES if dump(py)[t] != dump(rs)[t]], + 'effects': [py_effects, rs_effects], + }, + ) + await store.close() + + +async def boundary_cases(native, directory): + path = directory / 'boundaries.db' + store = await fixture(path) + baseline_settings = dump(path)['settings'] + actual, _ = native.run(path, [action('result', subject='u_bob', digest=D, record=result())]) + check( + 'mismatched-result-subject', + actual.get('code') == 'storage_record_mismatch' and not dump(path)['results'], + group='native-safety', + ) + actual, _ = native.run(path, [ledger(result(''))]) + check( + 'empty-subject-not-anonymous-alias', + actual.get('code') == 'storage_record_mismatch', + group='native-safety', + ) + actual, _ = native.run( + path, [action('result', subject='u_alice', digest='not-a-digest', record=result())] + ) + check('malformed-result-digest', actual.get('code') == 'invalid_digest', group='native-safety') + with closing(sqlite3.connect(path)) as db: + db.execute( + "CREATE TRIGGER fixture_abort BEFORE INSERT ON settings WHEN NEW.key='reject' BEGIN SELECT RAISE(ABORT,'fixture'); END" + ) + db.commit() + actual, _ = native.run( + path, [set_value('before'), action('ignore', steps=[set_value('reject')])] + ) + check( + 'swallowed-write-fails-closed', + actual.get('code') == 'constraint_conflict' and dump(path)['settings'] == baseline_settings, + group='native-safety', + ) + with closing(sqlite3.connect(path)) as db: + db.execute('DROP TRIGGER fixture_abort') + db.execute( + "CREATE TRIGGER fixture_abort BEFORE INSERT ON settings WHEN NEW.key='reject' BEGIN SELECT RAISE(ROLLBACK,'fixture'); END" + ) + db.commit() + actual, _ = native.run( + path, + [set_value('before'), action('ignore', steps=[set_value('reject')]), set_value('escape')], + ) + check( + 'implicit-abort-no-autocommit-write', + actual.get('code') == 'constraint_conflict' and dump(path)['settings'] == baseline_settings, + group='native-safety', + ) + with closing(sqlite3.connect(path)) as db: + db.execute('DROP TRIGGER fixture_abort') + db.execute( + "CREATE TRIGGER fixture_deferred AFTER INSERT ON settings WHEN NEW.key='deferred' BEGIN INSERT INTO resources SELECT 'orphan',type,'orphan','missing',owner,grp,mode,generation,revision,state,created_at,modified_at,body FROM resources WHERE id='r_root'; END" + ) + db.commit() + actual, effects = native.run( + path, [action('effect', name='cleanup'), set_value('deferred'), ledger()] + ) + check( + 'commit-failure-undoes-result-and-data', + actual.get('code') == 'constraint_conflict' + and effects == ['cleanup'] + and not dump(path)['results'] + and dump(path)['settings'] == baseline_settings + and len(dump(path)['resources']) == 1, + group='native-safety', + ) + actual, effects = native.run( + path, + [ + action('effect', name='first'), + action('effect', name='bad', fail=True), + action('effect', name='panic', panic=True), + action('fail'), + ], + ) + check( + 'compensation-errors-preserve-primary', + actual.get('code') == 'fixture_failure' + and actual.get('cleanup') == ['rollback_effect_panicked', 'fixture_cleanup'] + and effects == ['panic', 'bad', 'first'], + group='native-safety', + ) + await store.close() + + capacity = directory / 'capacity.db' + cap_store = await fixture(capacity) + with closing(sqlite3.connect(capacity)) as db: + db.execute( + "WITH RECURSIVE n(x) AS (VALUES(1) UNION ALL SELECT x+1 FROM n WHERE x<100000) INSERT INTO results SELECT 'fixture', CAST(x AS TEXT), 'fixture', '{}' FROM n" + ) + db.commit() + expected, _ = await run_python(cap_store, [insert(), ledger()]) + actual, _ = native.run(capacity, [insert(), ledger()]) + check( + 'result-capacity-rolls-back-resource', + actual == expected + and actual.get('code') == 'storage_capacity_exceeded' + and len(dump(capacity)['resources']) == 1, + group='parity', + ) + await cap_store.close() + + +async def process_cases(native, directory): + for stage in ('before', 'after'): + path = directory / f'crash-{stage}.db' + store = await fixture(path) + body = [insert(), action('event', record=event()), action('audit', record=audit())] + command = once(body + ([action('hold')] if stage == 'before' else [])) + process = native.start(path, [command], hold_after_commit=stage == 'after') + try: + got = line(process) + checkpoint = 'transaction' if stage == 'before' else 'committed' + check(f'crash-{stage}-checkpoint', got == {'checkpoint': checkpoint}, group='process') + visible = dump(path) + check( + f'crash-{stage}-visibility', + bool(visible['results']) == (stage == 'after') + and len(visible['resources']) == (2 if stage == 'after' else 1), + group='process', + ) + finally: + kill(process) + actual, _ = native.run(path, [once(body)]) + rows = dump(path) + check( + f'crash-{stage}-retry-single-commit', + actual['status'] == 'ok' + and len(rows['resources']) == 2 + and len(rows['results']) == len(rows['events']) == len(rows['audit']) == 1, + group='process', + ) + await store.close() + + path = directory / 'competing.db' + store = await fixture(path) + body = [insert(), action('audit', record=audit())] + first = native.start(path, [once(body + [action('hold')])]) + second = None + try: + check( + 'competing-writer-first-checkpoint', + line(first) == {'checkpoint': 'transaction'}, + group='process', + ) + second = native.start(path, [once(body)], timeout_ms=3000) + one = finish(first) + stdout, stderr = second.communicate(timeout=10) + two = json.loads(stdout.splitlines()[-1]) + check( + 'competing-writers-share-one-result', + second.returncode == 0 and one == two and len(dump(path)['audit']) == 1, + stderr, + group='process', + ) + finally: + if first.poll() is None: + kill(first) + if second is not None and second.poll() is None: + kill(second) + + async with store.transaction(write=True) as tx: + tx.set_setting('python-holds', 1) + actual, _ = native.run(path, [set_value('must-not-write')], timeout_ms=50) + check( + 'python-fence-blocks-rust', actual.get('code') == 'server_busy', actual, group='process' + ) + rust = native.start(path, [set_value('rust-holds'), action('hold')]) + try: + line(rust) + try: + async with store.transaction(write=True): + raise AssertionError('Python crossed native writer fence') + except Failure as exc: + check('rust-fence-blocks-python', exc.code == 'server_busy', group='process') + finish(rust) + finally: + if rust.poll() is None: + kill(rust) + + rust = native.start( + path, [action('effect', name='cleanup', hold=True), set_value('abort'), action('fail')] + ) + try: + check('compensation-started', line(rust) == {'effect': 'cleanup'}, group='process') + # The next checkpoint is line-buffered; read directly after its known + # predecessor rather than asking select about already buffered data. + checkpoint = json.loads(rust.stdout.readline()) + check( + 'compensation-checkpoint', checkpoint == {'checkpoint': 'compensation'}, group='process' + ) + with closing(sqlite3.connect(path, isolation_level=None, timeout=0)) as db: + db.execute('BEGIN IMMEDIATE') + db.execute('ROLLBACK') + try: + async with store.transaction(write=True): + raise AssertionError('Python crossed compensation fence') + except Failure as exc: + check('fence-held-after-sql-rollback', exc.code == 'server_busy', group='process') + outcome = finish(rust) + check( + 'compensation-finishes-before-release', + outcome.get('code') == 'fixture_failure', + group='process', + ) + finally: + if rust.poll() is None: + kill(rust) + actual, _ = native.run(path, [set_value('after-compensation')]) + check('writer-usable-after-clean-rollback', actual['status'] == 'ok', group='process') + await store.close() + + +async def main_async(args): + native = Native(args.binary.resolve()) + with tempfile.TemporaryDirectory(prefix='msg-writer-parity-') as root: + directory = Path(root) + await parity_cases(native, directory) + await boundary_cases(native, directory) + await process_cases(native, directory) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--binary', type=Path, required=True) + parser.add_argument('--report', type=Path) + args = parser.parse_args() + try: + asyncio.run(main_async(args)) + except Exception as exc: + REPORT.append({'name': 'harness', 'group': 'harness', 'ok': False, 'detail': repr(exc)}) + failures = [case for case in REPORT if not case['ok']] + report = { + 'cases': len(REPORT), + 'groups': dict(Counter(c['group'] for c in REPORT)), + 'failures': len(failures), + 'first_failures': failures[:12], + 'checks': REPORT, + } + if args.report: + args.report.parent.mkdir(parents=True, exist_ok=True) + args.report.write_text(json.dumps(report, ensure_ascii=True, indent=2) + '\n') + print(json.dumps({k: v for k, v in report.items() if k != 'checks'}, ensure_ascii=True)) + if failures: + raise SystemExit(1) + + +if __name__ == '__main__': + main()