diff --git a/.github/workflows/architecture-boundaries.yml b/.github/workflows/architecture-boundaries.yml index 25a9b25a..70501828 100644 --- a/.github/workflows/architecture-boundaries.yml +++ b/.github/workflows/architecture-boundaries.yml @@ -73,7 +73,7 @@ jobs: python -m pytest tests/test_tool_runner_contract.py tests/test_tools.py tests/test_effect_completion_fencing.py --junitxml=artifacts/runner-contract.xml - - uses: actions/upload-artifact@v4 + - uses: actions/upload-artifact@v7.0.1 if: always() with: name: architecture-boundaries diff --git a/.github/workflows/architecture-ports.yml b/.github/workflows/architecture-ports.yml index 7782b432..318f36e8 100644 --- a/.github/workflows/architecture-ports.yml +++ b/.github/workflows/architecture-ports.yml @@ -74,7 +74,7 @@ jobs: tests/test_batch_secret_release.py tests/test_tools.py tests/test_webhook_domain_events.py tests/test_read_projection_cache_transports.py tests/test_read_projection_cache_unit.py tests/issue_78_80 --junitxml=artifacts/architecture-ports.xml - - uses: actions/upload-artifact@v4 + - uses: actions/upload-artifact@v7.0.1 if: always() with: name: architecture-ports-evidence diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5143e5a8..f63db70c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -16,7 +16,7 @@ jobs: strategy: fail-fast: false matrix: - shard: [0, 1, 2, 3] + shard: [0, 1, 2, 3, 4, 5, 6, 7] services: postgres: # Recovery jobs explicitly install matching PostgreSQL 16 client tools. @@ -69,13 +69,17 @@ jobs: bwrap --version sudo apparmor_parser -r /usr/share/apparmor/extra-profiles/bwrap-userns-restrict - name: Install the complete dependency set - run: python -m pip install -e '.[dev]' + run: | + python -m pip install uv==0.12.20 + uv sync --locked --extra dev --python "$(command -v python)" + echo "$GITHUB_WORKSPACE/.venv/bin" >> "$GITHUB_PATH" + uv pip check - name: Check real tool isolation before running shards run: python scripts/check_tool_sandbox.py - name: Compile all new modules - run: python -m compileall -q src + run: python -W error -m compileall -q src - name: Run core, business and security tests - run: python scripts/ci_shards.py run ${{ matrix.shard }} 4 + run: python scripts/ci_shards.py run ${{ matrix.shard }} 8 - name: Require every transport and actual tokenizer if: matrix.shard == 0 env: @@ -86,7 +90,7 @@ jobs: run: | python -m build python scripts/check_package_artifacts.py dist - - uses: actions/upload-artifact@v4 + - uses: actions/upload-artifact@v7.0.1 if: always() with: name: rewrite-verification-${{ matrix.shard }} @@ -100,12 +104,12 @@ jobs: timeout-minutes: 5 steps: - uses: actions/checkout@v7 - - uses: actions/download-artifact@v4 + - uses: actions/download-artifact@v8.0.1 with: pattern: rewrite-verification-* merge-multiple: true - name: Require complete disjoint test evidence - run: python scripts/ci_shards.py check 4 + run: python scripts/ci_shards.py check 8 - name: Require all tests, conformance and build jobs to succeed env: SHARD_RESULT: ${{ needs.shards.result }} diff --git a/.github/workflows/client-boundary.yml b/.github/workflows/client-boundary.yml index 383239b6..db09ab13 100644 --- a/.github/workflows/client-boundary.yml +++ b/.github/workflows/client-boundary.yml @@ -63,7 +63,7 @@ jobs: "$client_python" -I "$GITHUB_WORKSPACE/scripts/check_client_install.py" --minimal-install \ > "$GITHUB_WORKSPACE/artifacts/client-install.json" "$client_python" -m pip list --format=json > "$GITHUB_WORKSPACE/artifacts/client-packages.json" - - uses: actions/upload-artifact@v4 + - uses: actions/upload-artifact@v7.0.1 if: always() with: name: client-boundary diff --git a/.github/workflows/client-part-bytes.yml b/.github/workflows/client-part-bytes.yml index 94162d60..8003aae9 100644 --- a/.github/workflows/client-part-bytes.yml +++ b/.github/workflows/client-part-bytes.yml @@ -68,7 +68,7 @@ jobs: python -m pytest -q tests/test_client_part_bytes.py tests/test_client.py --junitxml=artifacts/client-part-bytes.xml - - uses: actions/upload-artifact@v4 + - uses: actions/upload-artifact@v7.0.1 if: always() with: name: client-part-bytes-evidence diff --git a/.github/workflows/collaboration-list-bounds.yml b/.github/workflows/collaboration-list-bounds.yml index 82f2c388..b45a05a4 100644 --- a/.github/workflows/collaboration-list-bounds.yml +++ b/.github/workflows/collaboration-list-bounds.yml @@ -64,7 +64,7 @@ jobs: tests/test_collaboration_resources.py tests/test_collaboration_resource_surfaces.py tests/test_proposals.py --junitxml=artifacts/work-list-regressions.xml - - uses: actions/upload-artifact@v4 + - uses: actions/upload-artifact@v7.0.1 if: always() with: name: collaboration-list-evidence diff --git a/.github/workflows/deployment-rehearsal.yml b/.github/workflows/deployment-rehearsal.yml new file mode 100644 index 00000000..9560ced3 --- /dev/null +++ b/.github/workflows/deployment-rehearsal.yml @@ -0,0 +1,103 @@ +name: Deployment rehearsal +on: + push: + branches: [main] + pull_request: + workflow_dispatch: +permissions: + contents: read +concurrency: + group: deployment-rehearsal-${{ github.head_ref || github.ref_name }} + cancel-in-progress: true +jobs: + installed-server: + runs-on: ubuntu-24.04 + timeout-minutes: 15 + steps: + - uses: actions/checkout@v7 + with: + persist-credentials: false + - uses: actions/setup-python@v7 + with: + python-version: '3.15' + allow-prereleases: true + - name: Install isolated rehearsal prerequisites + run: | + sudo apt-get update -qq + sudo env NEEDRESTART_MODE=l apt-get install -y -qq postgresql-16 postgresql-client-16 openssh-server age git-lfs bubblewrap apparmor-profiles + echo /usr/lib/postgresql/16/bin >> "$GITHUB_PATH" + sudo apparmor_parser -r /usr/share/apparmor/extra-profiles/bwrap-userns-restrict + sudo install -d -m 0755 /run/sshd + python -m pip install uv==0.12.20 + uv sync --locked --extra dev --python "$(command -v python)" + - name: Build and install the exact wheel with locked server dependencies + run: | + mkdir -p artifacts + uv run --locked --extra dev python -m build + uv run --locked --extra dev python scripts/check_package_artifacts.py dist + uv export --locked --extra server --no-dev --no-emit-project --format requirements-txt --output-file artifacts/server-dependencies.lock + python -m venv "$RUNNER_TEMP/msg-installed-server" + server_python="$RUNNER_TEMP/msg-installed-server/bin/python" + uv pip sync --python "$server_python" --require-hashes artifacts/server-dependencies.lock + uv pip install --python "$server_python" --no-deps dist/*.whl + uv pip check --python "$server_python" + "$server_python" -m pip list --format=json > artifacts/server-packages.json + sha256sum dist/* artifacts/server-dependencies.lock > artifacts/release-sha256.txt + echo "MSG_REHEARSAL_PYTHON=$server_python" >> "$GITHUB_ENV" + - name: Install the same wheel as a locked client-only distribution + run: | + uv export --locked --no-dev --no-emit-project --format requirements-txt --output-file artifacts/client-dependencies.lock > /dev/null + python -m venv "$RUNNER_TEMP/msg-installed-client" + client_python="$RUNNER_TEMP/msg-installed-client/bin/python" + uv pip sync --python "$client_python" --require-hashes artifacts/client-dependencies.lock + uv pip install --python "$client_python" --no-deps dist/*.whl + uv pip check --python "$client_python" + "$client_python" -m pip list --format=json > artifacts/client-packages.json + sha256sum artifacts/client-dependencies.lock >> artifacts/release-sha256.txt + mkdir -p "$RUNNER_TEMP/msg-rehearsal-outside" + cd "$RUNNER_TEMP/msg-rehearsal-outside" + "$RUNNER_TEMP/msg-installed-client/bin/msg" --help > "$GITHUB_WORKSPACE/artifacts/client-help.txt" + "$client_python" -I "$GITHUB_WORKSPACE/scripts/check_client_install.py" --minimal-install > "$GITHUB_WORKSPACE/artifacts/client-install.json" + - name: Rehearse startup, restart, signed clients, read-only diagnostics and worker + run: | + mkdir -p "$RUNNER_TEMP/msg-rehearsal-outside" + cd "$RUNNER_TEMP/msg-rehearsal-outside" + "$MSG_REHEARSAL_PYTHON" -I "$GITHUB_WORKSPACE/scripts/check_installed_server.py" > "$GITHUB_WORKSPACE/artifacts/installed-server.json" + - name: Exercise real isolated OpenSSH and Git revocation boundaries + run: | + cd "$RUNNER_TEMP/msg-rehearsal-outside" + "$MSG_REHEARSAL_PYTHON" -I "$GITHUB_WORKSPACE/scripts/check_isolated_sshd.py" > "$GITHUB_WORKSPACE/artifacts/isolated-sshd.json" + - name: Run the installed official isolated selftest without development dependencies + run: | + cd "$RUNNER_TEMP/msg-rehearsal-outside" + "$MSG_REHEARSAL_PYTHON" -I -m msg.daemon selftest > "$GITHUB_WORKSPACE/artifacts/installed-selftest.json" + "$MSG_REHEARSAL_PYTHON" -I - <<'PY' + import json, os + from pathlib import Path + report = json.loads((Path(os.environ['GITHUB_WORKSPACE']) / 'artifacts/installed-selftest.json').read_text()) + assert report['ok'] is True and report['cleaned_up'] is True + assert report['checks']['market_e2e'] is True + PY + - name: Record the exact source and interpreter + if: always() + run: | + mkdir -p artifacts + python - <<'PY' + import json, os, platform, subprocess + from pathlib import Path + def git(*args): + return subprocess.check_output(['git', *args], text=True).strip() + Path('artifacts/source.json').write_text(json.dumps({ + 'commit': git('rev-parse', 'HEAD'), 'tree': git('rev-parse', 'HEAD^{tree}'), + 'run_id': os.environ['GITHUB_RUN_ID'], 'python': platform.python_version(), + 'scope': 'disposable installed wheel; no production access', + }, indent=2) + '\n') + PY + - uses: actions/upload-artifact@v7.0.1 + if: always() + with: + name: deployment-rehearsal + path: | + artifacts/ + dist/ + if-no-files-found: error diff --git a/.github/workflows/git-transport-boundaries.yml b/.github/workflows/git-transport-boundaries.yml index 2f66e02b..9a736678 100644 --- a/.github/workflows/git-transport-boundaries.yml +++ b/.github/workflows/git-transport-boundaries.yml @@ -67,7 +67,7 @@ jobs: - name: Preserve real HTTP, SSH, bundle, LFS and publication fences run: | python -m pytest tests/test_git_http_push.py tests/test_native_git_read_only.py tests/test_git_lfs.py tests/test_lfs_shared_gc_quota.py tests/test_ssh_rss.py tests/test_extensions.py tests/test_storage_layout.py tests/test_effect_completion_fencing.py tests/issue_78_80/test_lfs_integrity.py tests/issue_78_80/test_lfs_migration_contract.py --junitxml=artifacts/git-regressions.xml - - uses: actions/upload-artifact@v4 + - uses: actions/upload-artifact@v7.0.1 if: always() with: name: git-transport-boundaries-${{ github.sha }} diff --git a/.github/workflows/hosting-authority.yml b/.github/workflows/hosting-authority.yml index d5756fc3..1ff159b3 100644 --- a/.github/workflows/hosting-authority.yml +++ b/.github/workflows/hosting-authority.yml @@ -60,7 +60,7 @@ jobs: python -m pip install -e '.[dev]' - name: Exercise revoked keys and restorable ownership rights run: python -m pytest tests/test_hosting_runtime_authority.py --junitxml=artifacts/authority.xml - - uses: actions/upload-artifact@v4 + - uses: actions/upload-artifact@v7.0.1 if: always() with: name: hosting-authority-evidence diff --git a/.github/workflows/hosting-readonly.yml b/.github/workflows/hosting-readonly.yml index 6ed927d9..7b5d68bc 100644 --- a/.github/workflows/hosting-readonly.yml +++ b/.github/workflows/hosting-readonly.yml @@ -67,7 +67,7 @@ jobs: python -m pytest tests/test_hosting*.py tests/test_cli_hosting.py tests/test_extensions.py::test_hosting_publish_is_explicit_versioned_on_service_origin --junitxml=artifacts/hosting.xml - - uses: actions/upload-artifact@v4 + - uses: actions/upload-artifact@v7.0.1 if: always() with: name: hosting-readonly-evidence diff --git a/.github/workflows/integration-regressions.yml b/.github/workflows/integration-regressions.yml index 6d4d96b7..dd494593 100644 --- a/.github/workflows/integration-regressions.yml +++ b/.github/workflows/integration-regressions.yml @@ -92,7 +92,7 @@ jobs: 'requested_database_locale': os.environ['TEST_DATABASE_LOCALE']}, indent=2) + '\n') PY - name: Preserve reviewed public source even when tests fail - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7.0.1 with: name: integration-regression-source-${{ matrix.locale }} path: | @@ -151,7 +151,7 @@ jobs: tests/test_webhook_overlap.py tests/test_webhook_domain_events.py tests/test_webhook_delivery.py tests/test_webhook_identity.py --junitxml=artifacts/integration-regressions.xml - - uses: actions/upload-artifact@v4 + - uses: actions/upload-artifact@v7.0.1 if: always() with: name: integration-regression-evidence-${{ matrix.locale }} diff --git a/.github/workflows/issue-recovery.yml b/.github/workflows/issue-recovery.yml index 97645555..3e6607f2 100644 --- a/.github/workflows/issue-recovery.yml +++ b/.github/workflows/issue-recovery.yml @@ -76,7 +76,7 @@ jobs: tests/test_backup_retirement_cli.py tests/test_recovery_policy_v2.py tests/test_recovery_authority_reconcile.py tests/test_complete_recovery_proof.py --junitxml=artifacts/recovery-current.xml - - uses: actions/upload-artifact@v4 + - uses: actions/upload-artifact@v7.0.1 if: always() with: name: issue-recovery-evidence diff --git a/.github/workflows/lease-list-bounds.yml b/.github/workflows/lease-list-bounds.yml index f9bbac6e..cad6da41 100644 --- a/.github/workflows/lease-list-bounds.yml +++ b/.github/workflows/lease-list-bounds.yml @@ -65,7 +65,7 @@ jobs: tests/test_collaboration_surfaces.py tests/test_collaboration_resources.py tests/test_collaboration_resource_surfaces.py --junitxml=artifacts/lease-list-regressions.xml - - uses: actions/upload-artifact@v4 + - uses: actions/upload-artifact@v7.0.1 if: always() with: name: lease-list-evidence diff --git a/.github/workflows/market-boundaries.yml b/.github/workflows/market-boundaries.yml index 87ee288f..2aadc863 100644 --- a/.github/workflows/market-boundaries.yml +++ b/.github/workflows/market-boundaries.yml @@ -59,7 +59,7 @@ jobs: - name: Preserve all published checkout and recovery semantics run: | python -m pytest tests/test_orders.py tests/test_managed_checkout.py tests/test_checkout_decision_integrity.py tests/test_market_contracts.py tests/test_market_integration.py tests/test_market_recovery.py tests/test_market_account_scope.py tests/test_market_71.py tests/test_market_72.py tests/test_money_core.py tests/test_market_redemption.py tests/test_bounty.py tests/test_store_foundation.py --junitxml=artifacts/market-regressions.xml - - uses: actions/upload-artifact@v4 + - uses: actions/upload-artifact@v7.0.1 if: always() with: name: market-boundaries-${{ github.sha }} diff --git a/.github/workflows/metadata-session-boundary.yml b/.github/workflows/metadata-session-boundary.yml index 5a3bf086..3b446f9e 100644 --- a/.github/workflows/metadata-session-boundary.yml +++ b/.github/workflows/metadata-session-boundary.yml @@ -58,7 +58,7 @@ jobs: - name: Preserve PostgreSQL locks, migration and recovery regressions run: | python -m pytest tests/test_postgres.py tests/test_security_storage_rollback.py tests/test_security_rollback_fences.py tests/test_ledger_accounts.py tests/test_ledger_account_isolation.py tests/test_ledger_migration_json.py tests/test_ledger_migration_references.py tests/test_recovery_safety_units.py tests/test_recovery_metadata.py --junitxml=artifacts/metadata-regressions.xml - - uses: actions/upload-artifact@v4 + - uses: actions/upload-artifact@v7.0.1 if: always() with: name: metadata-session-${{ github.sha }} diff --git a/.github/workflows/python-quality.yml b/.github/workflows/python-quality.yml new file mode 100644 index 00000000..d110442f --- /dev/null +++ b/.github/workflows/python-quality.yml @@ -0,0 +1,67 @@ +name: Python quality +on: + push: + branches: [main] + pull_request: + workflow_dispatch: +permissions: + contents: read +concurrency: + group: python-quality-${{ github.head_ref || github.ref_name }} + cancel-in-progress: true +jobs: + lint: + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@v7 + with: + persist-credentials: false + - uses: actions/setup-python@v7 + with: + python-version: '3.15' + allow-prereleases: true + - name: Install the locked development environment + run: | + python -m pip install uv==0.12.20 + uv lock --check --python "$(command -v python)" + uv sync --locked --extra dev --python "$(command -v python)" + uv pip check + - name: Require zero Ruff errors and warnings, and exact formatting + run: | + mkdir -p artifacts + uv run --locked --extra dev ruff check . --output-format json > artifacts/ruff.json 2> artifacts/ruff.stderr + test ! -s artifacts/ruff.stderr + uv run --locked --extra dev ruff format --check . > artifacts/format.txt 2> artifacts/format.stderr + test ! -s artifacts/format.stderr + uv run --locked --extra dev python -W error -m compileall -q src tests conformance scripts + - name: Record source and toolchain evidence + if: always() + run: | + mkdir -p artifacts + uv run --locked --extra dev python - <<'PYTHON' + import importlib.metadata, json, os, pathlib, platform, subprocess + def git(*args): + return subprocess.check_output(['git', *args], text=True).strip() + pathlib.Path('artifacts/quality-source.json').write_text(json.dumps({ + 'commit': git('rev-parse', 'HEAD'), + 'tree': git('rev-parse', 'HEAD^{tree}'), + 'run_id': os.environ['GITHUB_RUN_ID'], + 'python': platform.python_version(), + 'ruff': importlib.metadata.version('ruff'), + 'uv': subprocess.check_output(['uv', '--version'], text=True).strip(), + 'uv_build_requirement': 'uv_build==0.12.20', + }, indent=2) + '\n') + PYTHON + - name: Preserve a suggested safe Ruff patch after failure + if: failure() + run: | + uv run --locked --extra dev ruff check . --fix --no-unsafe-fixes > artifacts/fix-suggestions.txt 2>&1 || true + uv run --locked --extra dev ruff format . >> artifacts/fix-suggestions.txt 2>&1 + git diff --binary > artifacts/quality-fixes.patch + - uses: actions/upload-artifact@v7.0.1 + if: always() + with: + name: python-quality + path: artifacts/ + if-no-files-found: error diff --git a/.github/workflows/recovery-safety.yml b/.github/workflows/recovery-safety.yml index 2d71e0e2..2ab8562b 100644 --- a/.github/workflows/recovery-safety.yml +++ b/.github/workflows/recovery-safety.yml @@ -69,7 +69,7 @@ jobs: tests/test_backup_drill.py \ tests/test_backup_daemon.py \ --junitxml=artifacts/recovery-safety.xml - - uses: actions/upload-artifact@v4 + - uses: actions/upload-artifact@v7.0.1 if: always() with: name: recovery-safety-verification diff --git a/.github/workflows/storage-session.yml b/.github/workflows/storage-session.yml index b3dd3955..4b6ba9be 100644 --- a/.github/workflows/storage-session.yml +++ b/.github/workflows/storage-session.yml @@ -77,7 +77,7 @@ jobs: tests/test_ledger_migration_references.py tests/test_recovery_safety_units.py tests/test_recovery_metadata.py --junitxml=artifacts/storage-session.xml - - uses: actions/upload-artifact@v4 + - uses: actions/upload-artifact@v7.0.1 if: always() with: name: storage-session-evidence diff --git a/.github/workflows/token-delivery-boundary.yml b/.github/workflows/token-delivery-boundary.yml index 5178b251..5acc125d 100644 --- a/.github/workflows/token-delivery-boundary.yml +++ b/.github/workflows/token-delivery-boundary.yml @@ -74,7 +74,7 @@ jobs: tests/test_credential_recovery_window.py tests/test_client_token_delivery.py tests/test_batch_secret_release.py tests/test_security_patch_regressions.py --junitxml=artifacts/token-delivery-boundary.xml - - uses: actions/upload-artifact@v4 + - uses: actions/upload-artifact@v7.0.1 if: always() with: name: token-delivery-boundary-evidence diff --git a/.github/workflows/transfer-status.yml b/.github/workflows/transfer-status.yml index c37d4f84..d561753a 100644 --- a/.github/workflows/transfer-status.yml +++ b/.github/workflows/transfer-status.yml @@ -73,7 +73,7 @@ jobs: tests/test_transfer_status_cursors.py tests/test_transfer.py tests/test_transfer_endpoint.py tests/test_read_only_evidence.py --junitxml=artifacts/transfer-status.xml - - uses: actions/upload-artifact@v4 + - uses: actions/upload-artifact@v7.0.1 if: always() with: name: transfer-status-evidence diff --git a/.gitignore b/.gitignore index d48dfcff..3c66f7ad 100644 --- a/.gitignore +++ b/.gitignore @@ -15,3 +15,4 @@ dist/ .coverage htmlcov/ .legacy-ledger/ +/artifacts/ diff --git a/README.md b/README.md index 978ca324..5ae103f9 100644 --- a/README.md +++ b/README.md @@ -30,7 +30,7 @@ uv sync ``` 基础安装仅包含签名客户端及传输所需的 Python 依赖，不要求本机运行 PostgreSQL 或 Valkey。 -安装发行包时使用 `python -m pip install msg-lmm-best`。`age` 加密/备份等操作仍需相应系统工具， +安装发行包时使用 `python -m pip install msgctl`。`age` 加密/备份等操作仍需相应系统工具， 并不因为依赖分组而获得空实现或降级执行。 部署完整服务端请使用 `uv sync --extra server`，或在源码目录安装 `python -m pip install '.[server]'`。 diff --git a/conformance/test_token_budget.py b/conformance/test_token_budget.py index faa6e60e..9f3fbe3d 100644 --- a/conformance/test_token_budget.py +++ b/conformance/test_token_budget.py @@ -1,51 +1,74 @@ """Measure actual tokenizer tokens; byte counts are not a token estimate.""" + import json import os from pathlib import Path + import httpx import pytest + +from msg.admin.root import _approve_csr, _provision from msg.application import Application -from msg.admin.root import _provision,_approve_csr +from msg.client import ClientState, MsgClient from msg.config import write_example -from msg.client import ClientState,MsgClient -from msg.core.codec import canonical,wire +from msg.core.codec import canonical, wire from msg.core.executor import result_wire from msg.transports.client import HTTPTransport from msg.transports.http import create_app + @pytest.mark.asyncio -async def test_real_token_and_roundtrip_budget(tmp_path,pg_dsn): +async def test_real_token_and_roundtrip_budget(tmp_path, pg_dsn): import tiktoken - encoding=tiktoken.get_encoding('cl100k_base') - app=Application(write_example(tmp_path/'etc',tmp_path/'data','http://testserver', - postgres_dsn=pg_dsn)) - csr,root=await _provision(app,'token-budget-test-passphrase') - await _approve_csr(app,csr,root,expected_digest=None,operator='conformance-fixture') - records=[] + + encoding = tiktoken.get_encoding('cl100k_base') + app = Application( + write_example(tmp_path / 'etc', tmp_path / 'data', 'http://testserver', postgres_dsn=pg_dsn) + ) + csr, root = await _provision(app, 'token-budget-test-passphrase') + await _approve_csr(app, csr, root, expected_digest=None, operator='conformance-fixture') + records = [] try: - async with httpx.AsyncClient(transport=httpx.ASGITransport(app=create_app(app)),base_url=app.settings.service_url) as http: - transport=HTTPTransport(app.settings.service_url,http=http) - client=MsgClient(ClientState(tmp_path/'client',server=app.settings.service_url),transport) + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=create_app(app)), base_url=app.settings.service_url + ) as http: + transport = HTTPTransport(app.settings.service_url, http=http) + client = MsgClient( + ClientState(tmp_path / 'client', server=app.settings.service_url), transport + ) await client.register('budget-agent') - async def measured(name,args,budget=900): - before=transport.calls - packet=client.prepare(name,args) - result=client.checked(await client.send(packet)) - text=canonical(result_wire(result)).decode() - request_text=canonical(packet).decode() - record={'operation':name,'input_tokens':len(encoding.encode(request_text)), - 'output_tokens':len(encoding.encode(text)),'output_bytes':len(text.encode()), - 'roundtrips':transport.calls-before} - assert record['roundtrips']==1,record - assert record['output_tokens']<=budget,record + + async def measured(name, args, budget=900): + before = transport.calls + packet = client.prepare(name, args) + result = client.checked(await client.send(packet)) + text = canonical(result_wire(result)).decode() + request_text = canonical(packet).decode() + record = { + 'operation': name, + 'input_tokens': len(encoding.encode(request_text)), + 'output_tokens': len(encoding.encode(text)), + 'output_bytes': len(text.encode()), + 'roundtrips': transport.calls - before, + } + assert record['roundtrips'] == 1, record + assert record['output_tokens'] <= budget, record records.append(record) return result - post=await measured('content.post_create',{'parent':'/main','body':'A small coordination message.'}) - await measured('discussion.reply',{'target':wire(post.resources[0]),'body':'Received; ready for the next step.'}) - await measured('discussion.thread',{'id':post.resources[0].id},budget=2500) - await measured('transfer.open',{'direction':'upload','size':0}) - target=Path(os.getenv('MSG_BENCHMARK_PATH',str(tmp_path/'token-budget.json'))) - target.parent.mkdir(parents=True,exist_ok=True) - target.write_text(json.dumps({'tokenizer':'cl100k_base','measurements':records},indent=2)) + + post = await measured( + 'content.post_create', {'parent': '/main', 'body': 'A small coordination message.'} + ) + await measured( + 'discussion.reply', + {'target': wire(post.resources[0]), 'body': 'Received; ready for the next step.'}, + ) + await measured('discussion.thread', {'id': post.resources[0].id}, budget=2500) + await measured('transfer.open', {'direction': 'upload', 'size': 0}) + target = Path(os.getenv('MSG_BENCHMARK_PATH', str(tmp_path / 'token-budget.json'))) + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text( + json.dumps({'tokenizer': 'cl100k_base', 'measurements': records}, indent=2) + ) finally: await app.close() diff --git a/conformance/test_transports.py b/conformance/test_transports.py index 60f0cd6d..399e9fee 100644 --- a/conformance/test_transports.py +++ b/conformance/test_transports.py @@ -3,125 +3,211 @@ No importorskip, fake GraphQL adapter, or expected-failure marker is used here. The local delivery environment cannot execute the GraphQL case; CI must do so. """ + import asyncio import os -from pathlib import Path import socket import sys +from pathlib import Path from uuid import uuid4 -import httpx + import pytest import uvicorn + +from msg.admin.root import _approve_csr, _provision from msg.application import Application -from msg.admin.root import _provision, _approve_csr -from msg.config import write_example from msg.client import ClientState, MsgClient -from msg.core.codec import b64,unb64,canonical,digest,decode,loads,wire -from msg.core.models import OperationResult +from msg.config import write_example +from msg.core.codec import b64, canonical, digest, loads, unb64 from msg.transports.client import TRANSPORTS, HTTPTransport from msg.transports.http import create_app def test_release_dependencies_are_available(): - assert sys.version_info[:2]>=(3,15), 'Release verification requires Python 3.15' + assert sys.version_info[:2] >= (3, 15), 'Release verification requires Python 3.15' import graphql import tiktoken + assert graphql.__version__ and tiktoken.__version__ @pytest.mark.asyncio -@pytest.mark.parametrize('mode',['http','path_get','graphql','mcp_http','cli','mcp_stdio']) -async def test_transfer_conformance_for_every_transport(mode,tmp_path,pg_dsn): - sock=socket.socket();sock.bind(('127.0.0.1',0));sock.setblocking(False) - url='http://127.0.0.1:'+str(sock.getsockname()[1]) - app=Application(write_example(tmp_path/'etc',tmp_path/'data',url,postgres_dsn=pg_dsn)) - csr,root=await _provision(app,'conformance-test-passphrase') - await _approve_csr(app,csr,root,expected_digest=None,operator='conformance-fixture') - server=uvicorn.Server(uvicorn.Config(create_app(app),log_level='critical',access_log=False,ws='none')) - serving=asyncio.create_task(server.serve(sockets=[sock])) +@pytest.mark.parametrize('mode', ['http', 'path_get', 'graphql', 'mcp_http', 'cli', 'mcp_stdio']) +async def test_transfer_conformance_for_every_transport(mode, tmp_path, pg_dsn): + sock = socket.socket() + sock.bind(('127.0.0.1', 0)) + sock.setblocking(False) + url = 'http://127.0.0.1:' + str(sock.getsockname()[1]) + app = Application(write_example(tmp_path / 'etc', tmp_path / 'data', url, postgres_dsn=pg_dsn)) + csr, root = await _provision(app, 'conformance-test-passphrase') + await _approve_csr(app, csr, root, expected_digest=None, operator='conformance-fixture') + server = uvicorn.Server( + uvicorn.Config(create_app(app), log_level='critical', access_log=False, ws='none') + ) + serving = asyncio.create_task(server.serve(sockets=[sock])) for _ in range(100): - if server.started:break - await asyncio.sleep(.02) + if server.started: + break + await asyncio.sleep(0.02) assert server.started - state=ClientState(tmp_path/'client',server=url) - transport=TRANSPORTS.get(mode,HTTPTransport)(url) - client=MsgClient(state,transport) - peer_transport=HTTPTransport(url) - peer=MsgClient(state,peer_transport) + state = ClientState(tmp_path / 'client', server=url) + transport = TRANSPORTS.get(mode, HTTPTransport)(url) + client = MsgClient(state, transport) + peer_transport = HTTPTransport(url) + peer = MsgClient(state, peer_transport) await peer.register('conformance-agent') - process=None - environment={**os.environ,'PYTHONPATH':str(Path(__file__).parents[1]/'src')} - command=[sys.executable,'-m','msg.cli','--config-dir',str(state.directory),'--server',url] - if mode=='mcp_stdio': - process=await asyncio.create_subprocess_exec(*command,'mcp',env=environment, - stdin=asyncio.subprocess.PIPE,stdout=asyncio.subprocess.PIPE,stderr=asyncio.subprocess.PIPE) - process.stdin.write(canonical({'jsonrpc':'2.0','id':'init','method':'initialize', - 'params':{'protocolVersion':'2025-11-25'}})+b'\n') + process = None + environment = {**os.environ, 'PYTHONPATH': str(Path(__file__).parents[1] / 'src')} + command = [ + sys.executable, + '-m', + 'msg.cli', + '--config-dir', + str(state.directory), + '--server', + url, + ] + if mode == 'mcp_stdio': + process = await asyncio.create_subprocess_exec( + *command, + 'mcp', + env=environment, + stdin=asyncio.subprocess.PIPE, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + process.stdin.write( + canonical({ + 'jsonrpc': '2.0', + 'id': 'init', + 'method': 'initialize', + 'params': {'protocolVersion': '2025-11-25'}, + }) + + b'\n' + ) await process.stdin.drain() - assert 'result' in loads(await asyncio.wait_for(process.stdout.readline(),15)) - async def invoke(operation,args,rid=None): - rid=rid or uuid4().hex - if mode=='cli': - child=await asyncio.create_subprocess_exec(*command,'call',operation,canonical(args).decode(), - '--request-id',rid,env=environment,stdout=asyncio.subprocess.PIPE,stderr=asyncio.subprocess.PIPE) - stdout,stderr=await asyncio.wait_for(child.communicate(),20) - assert child.returncode in {0,1},stderr.decode() + assert 'result' in loads(await asyncio.wait_for(process.stdout.readline(), 15)) + + async def invoke(operation, args, rid=None): + rid = rid or uuid4().hex + if mode == 'cli': + child = await asyncio.create_subprocess_exec( + *command, + 'call', + operation, + canonical(args).decode(), + '--request-id', + rid, + env=environment, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + stdout, stderr = await asyncio.wait_for(child.communicate(), 20) + assert child.returncode in {0, 1}, stderr.decode() return loads(stdout) - if mode=='mcp_stdio': - process.stdin.write(canonical({'jsonrpc':'2.0','id':rid,'method':'tools/call','params':{ - 'name':operation,'arguments':args,'_meta':{'msg/request_id':rid}}})+b'\n') + if mode == 'mcp_stdio': + process.stdin.write( + canonical({ + 'jsonrpc': '2.0', + 'id': rid, + 'method': 'tools/call', + 'params': { + 'name': operation, + 'arguments': args, + '_meta': {'msg/request_id': rid}, + }, + }) + + b'\n' + ) await process.stdin.drain() - result=loads(await asyncio.wait_for(process.stdout.readline(),15)) - assert 'result' in result,result + result = loads(await asyncio.wait_for(process.stdout.readline(), 15)) + assert 'result' in result, result return result['result']['structuredContent'] - result=await client.call(operation,args,request_id=rid) + result = await client.call(operation, args, request_id=rid) from msg.core.executor import result_wire + return result_wire(result) + try: - data=b'UTF8-exact\r\n'+bytes(range(128)) - opened=await invoke('transfer.open',{'direction':'upload','size':len(data),'digest':digest(data),'requested_part_bytes':97}) - assert opened['status']=='ok',opened - tid=opened['data']['transfer_id'] - chunks=[(offset,data[offset:offset+97]) for offset in range(0,len(data),97)] - offset,part=chunks[-1] - args={'transfer_id':tid,'offset':offset,'data':b64(part),'digest':digest(part)} - first=await invoke('transfer.part_put',args,'same-chunk') - again=await invoke('transfer.part_put',args,'same-chunk') - assert first['status']=='ok' and again['replayed'] - bad=await invoke('transfer.part_put',{**args,'digest':digest(b'wrong')}) - assert bad['status']=='error' - incomplete=await invoke('transfer.seal',{'transfer_id':tid,'final_size':len(data),'final_digest':digest(data)}) - assert incomplete['status']=='error' - for offset,part in reversed(chunks[:-1]): - piece={'transfer_id':tid,'offset':offset,'data':b64(part),'digest':digest(part)} + data = b'UTF8-exact\r\n' + bytes(range(128)) + opened = await invoke( + 'transfer.open', + { + 'direction': 'upload', + 'size': len(data), + 'digest': digest(data), + 'requested_part_bytes': 97, + }, + ) + assert opened['status'] == 'ok', opened + tid = opened['data']['transfer_id'] + chunks = [(offset, data[offset : offset + 97]) for offset in range(0, len(data), 97)] + offset, part = chunks[-1] + args = {'transfer_id': tid, 'offset': offset, 'data': b64(part), 'digest': digest(part)} + first = await invoke('transfer.part_put', args, 'same-chunk') + again = await invoke('transfer.part_put', args, 'same-chunk') + assert first['status'] == 'ok' and again['replayed'] + bad = await invoke('transfer.part_put', {**args, 'digest': digest(b'wrong')}) + assert bad['status'] == 'error' + incomplete = await invoke( + 'transfer.seal', + {'transfer_id': tid, 'final_size': len(data), 'final_digest': digest(data)}, + ) + assert incomplete['status'] == 'error' + for offset, part in reversed(chunks[:-1]): + piece = { + 'transfer_id': tid, + 'offset': offset, + 'data': b64(part), + 'digest': digest(part), + } # Recover the same subject-bound session using another real adapter. - if offset==0: - assert (await peer.call('transfer.part_put',piece)).status=='ok' + if offset == 0: + assert (await peer.call('transfer.part_put', piece)).status == 'ok' else: - assert (await invoke('transfer.part_put',piece))['status']=='ok' - status=await invoke('transfer.status',{'transfer_id':tid}) - assert status['status']=='ok' - sealed=await invoke('transfer.seal',{'transfer_id':tid,'final_size':len(data),'final_digest':digest(data)}) - assert sealed['status']=='ok',sealed - repeated=await invoke('transfer.seal',{'transfer_id':tid,'final_size':len(data),'final_digest':digest(data)}) - assert repeated['output']==sealed['output'] - download=await invoke('transfer.open',{'direction':'download','target':sealed['output'],'requested_part_bytes':83}) - assert download['status']=='ok' - restored=bytearray() - for offset in range(0,len(data),83): - result=await invoke('transfer.part_get',{'transfer_id':download['data']['transfer_id'],'offset':offset,'length':min(83,len(data)-offset)}) - assert result['status']=='ok',result - chunk=unb64(result['data']['data']) - assert digest(chunk)==result['data']['chunk']['content']['digest'] + assert (await invoke('transfer.part_put', piece))['status'] == 'ok' + status = await invoke('transfer.status', {'transfer_id': tid}) + assert status['status'] == 'ok' + sealed = await invoke( + 'transfer.seal', + {'transfer_id': tid, 'final_size': len(data), 'final_digest': digest(data)}, + ) + assert sealed['status'] == 'ok', sealed + repeated = await invoke( + 'transfer.seal', + {'transfer_id': tid, 'final_size': len(data), 'final_digest': digest(data)}, + ) + assert repeated['output'] == sealed['output'] + download = await invoke( + 'transfer.open', + {'direction': 'download', 'target': sealed['output'], 'requested_part_bytes': 83}, + ) + assert download['status'] == 'ok' + restored = bytearray() + for offset in range(0, len(data), 83): + result = await invoke( + 'transfer.part_get', + { + 'transfer_id': download['data']['transfer_id'], + 'offset': offset, + 'length': min(83, len(data) - offset), + }, + ) + assert result['status'] == 'ok', result + chunk = unb64(result['data']['data']) + assert digest(chunk) == result['data']['chunk']['content']['digest'] restored.extend(chunk) - assert bytes(restored)==data and digest(bytes(restored))==digest(data) - cancelled=await invoke('transfer.cancel',{'transfer_id':download['data']['transfer_id']}) - assert cancelled['status']=='ok' + assert bytes(restored) == data and digest(bytes(restored)) == digest(data) + cancelled = await invoke( + 'transfer.cancel', {'transfer_id': download['data']['transfer_id']} + ) + assert cancelled['status'] == 'ok' finally: if process: process.stdin.close() - await asyncio.wait_for(process.wait(),15) - await transport.close();await peer_transport.close() - server.should_exit=True + await asyncio.wait_for(process.wait(), 15) + await transport.close() + await peer_transport.close() + server.should_exit = True await serving sock.close() diff --git a/conftest.py b/conftest.py index 0a5a6428..a74ac741 100644 --- a/conftest.py +++ b/conftest.py @@ -15,43 +15,60 @@ from psycopg import sql -@pytest.fixture(scope="session") +@pytest.fixture(scope='session') def pg_cluster(): - configured = os.environ.get("MSG_TEST_POSTGRES_URL_TEMPLATE") + configured = os.environ.get('MSG_TEST_POSTGRES_URL_TEMPLATE') if configured: yield configured return - root = Path(tempfile.mkdtemp(prefix="msg-test-pg-", dir="/tmp")) - data = root / "data" - socket = root / "socket" + root = Path(tempfile.mkdtemp(prefix='msg-test-pg-', dir='/tmp')) + data = root / 'data' + socket = root / 'socket' socket.mkdir() try: subprocess.run( - ["initdb", "-D", str(data), "-A", "trust", "--no-instructions"], - check=True, capture_output=True, text=True, + ['initdb', '-D', str(data), '-U', 'msgtest', '-A', 'trust', '--no-instructions'], + check=True, + capture_output=True, + text=True, ) subprocess.run( - ["pg_ctl", "-D", str(data), "-l", str(root / "server.log"), - "-o", f"-k {socket} -h '' -p 5432", "-w", "start"], - check=True, capture_output=True, text=True, + [ + 'pg_ctl', + '-D', + str(data), + '-l', + str(root / 'server.log'), + '-o', + f"-k {socket} -h '' -p 5432", + '-w', + 'start', + ], + check=True, + capture_output=True, + text=True, ) - yield f'postgresql://lightjunction@localhost:5432/{{database}}?host={quote(str(socket),safe="")}' + yield f'postgresql://msgtest@localhost:5432/{{database}}?host={quote(str(socket), safe="")}' finally: if data.exists(): subprocess.run( - ["pg_ctl", "-D", str(data), "-m", "immediate", "-w", "stop"], - check=False, capture_output=True, text=True, + ['pg_ctl', '-D', str(data), '-m', 'immediate', '-w', 'stop'], + check=False, + capture_output=True, + text=True, ) shutil.rmtree(root) @pytest.fixture def pg_dsn(pg_cluster): - name = "msg_test_" + uuid.uuid4().hex - with psycopg.connect(pg_cluster.format(database="postgres"), autocommit=True) as connection: - connection.execute(sql.SQL("CREATE DATABASE {}").format(sql.Identifier(name))) + name = 'msg_test_' + uuid.uuid4().hex + with psycopg.connect(pg_cluster.format(database='postgres'), autocommit=True) as connection: + connection.execute(sql.SQL('CREATE DATABASE {}').format(sql.Identifier(name))) try: yield pg_cluster.format(database=name) finally: - with psycopg.connect(pg_cluster.format(database="postgres"), autocommit=True) as connection: - connection.execute(sql.SQL("DROP DATABASE {} WITH (FORCE)").format(sql.Identifier(name))) + with psycopg.connect(pg_cluster.format(database='postgres'), autocommit=True) as connection: + connection.execute( + sql.SQL('DROP DATABASE {} WITH (FORCE)').format(sql.Identifier(name)) + ) diff --git a/docs/CLIENT_INSTALLATION.md b/docs/CLIENT_INSTALLATION.md index 8b013e83..e9e4b43a 100644 --- a/docs/CLIENT_INSTALLATION.md +++ b/docs/CLIENT_INSTALLATION.md @@ -4,9 +4,9 @@ ## 安装角色 -仅连接远程 msg 服务的客户端，源码安装使用 `python -m pip install .` 或 `uv sync`。安装已经构建的发行文件时，使用 `python -m pip install /path/to/msg_lmm_best--py3-none-any.whl`。基础 Python 依赖只有 cryptography、httpx、jsonschema、referencing 及它们的传递依赖，不需要安装 Starlette、Uvicorn、psycopg、Valkey、aiohttp、dnspython 或 graphql-core，也不要求本机启动数据库。 +仅连接远程 msg 服务的客户端，源码安装使用 `python -m pip install .` 或 `uv sync`。安装已经构建的发行文件时，使用 `python -m pip install /path/to/msgctl--py3-none-any.whl`。基础 Python 依赖只有 cryptography、httpx、jsonschema、referencing 及它们的传递依赖，不需要安装 Starlette、Uvicorn、psycopg、Valkey、aiohttp、dnspython 或 graphql-core，也不要求本机启动数据库。 -完整服务端使用 `python -m pip install '.[server]'` 或 `uv sync --extra server`。发行包安装使用 `python -m pip install 'msg-lmm-best[server]'`，应选用已发布且与服务端部署匹配的版本。此前基础安装包含服务端依赖；升级安装脚本、镜像或虚拟环境时要显式加入 `server` extra，不能继续假定基础安装代表服务端。现有 `.[dev]` 仍拉入完整服务端依赖，原全量 CI 安装路径不变。 +完整服务端使用 `python -m pip install '.[server]'` 或 `uv sync --extra server`。发行包安装使用 `python -m pip install 'msgctl[server]'`，应选用已发布且与服务端部署匹配的版本。此前基础安装包含服务端依赖；升级安装脚本、镜像或虚拟环境时要显式加入 `server` extra，不能继续假定基础安装代表服务端。现有 `.[dev]` 仍拉入完整服务端依赖，原全量 CI 安装路径不变。 `msg`、`msgd` 两个命令入口都保留。`--help` 不要求服务端依赖。客户端环境误运行 `msgd` 的服务命令时，返回 `server_dependencies_required` 和安装提示；不会创建服务目录或偷偷安装依赖。安装 extra 只安装 Python 包，不启动数据库、不部署服务、不授予证书权限。`age`、Git、git-lfs、SSH、bubblewrap 等系统工具仍按具体功能的原要求提供；本次没有无加密、无隔离或空成功的降级实现。 diff --git a/docs/DEPLOYMENT.md b/docs/DEPLOYMENT.md index 5aed900e..7cc895fc 100644 --- a/docs/DEPLOYMENT.md +++ b/docs/DEPLOYMENT.md @@ -26,6 +26,14 @@ service_keys = "/var/lib/msgd/service" 服务端必须安装 `server` extra；裸 wheel 只含客户端依赖，不能用于 `msgd`。固定通过验收的源码提交、wheel SHA-256、Python 精确版本及服务依赖锁，不能在目标机重新解析一套未测依赖。用应用独立的 Python 3.15 和 venv，保留系统 Python、旧服务 venv 和数据用于回滚。若采用预发行 Python，记录完整版本并在同一版本验收。 +优先使用已通过最终 CI 的 `deployment-rehearsal` 产物：`dist/` 下的 wheel/sdist、 +`artifacts/server-dependencies.lock`、`client-dependencies.lock`、 +`release-sha256.txt` 与 `source.json`。这两份依赖锁来自同一 `uv.lock`，不含项目本身； +分别在全新 server/client venv 中执行 `uv pip sync --require-hashes`，核验 wheel 摘要后 +执行 `uv pip install --no-deps` 安装该 wheel，再执行 `uv pip check`。不要用裸 wheel +触发未测依赖重新解析，也不要把 client-only venv 用于服务器。产物范围及现场闸门见 +[发布验收入口](RELEASE_ACCEPTANCE.md)。 + 例如，先在与目标平台和解释器一致的构建/验收环境中，用实际 wheel 路径生成并测试锁文件（示例文件名须替换）： ```bash @@ -33,7 +41,7 @@ service_keys = "/var/lib/msgd/service" sudo install -d -m 0755 /opt/msgd sudo /absolute/path/to/verified/python3.15 -m venv /opt/msgd/venv # 下面的锁生成只在验收环境运行；目录需由构建操作者可写。 -printf '%s\n' 'msg-lmm-best[server] @ file:///protected/staged/msg_lmm_best-1.0.0a1-py3-none-any.whl' > /protected/staged/server.in +printf '%s\n' 'msgctl[server] @ file:///protected/staged/msgctl-0.1.0a1-py3-none-any.whl' > /protected/staged/server.in uv pip compile --python /opt/msgd/venv/bin/python --generate-hashes \ /protected/staged/server.in --output-file /protected/staged/server.lock sudo uv pip sync --python /opt/msgd/venv/bin/python --require-hashes /protected/staged/server.lock diff --git a/docs/LOCAL_VERIFICATION.md b/docs/LOCAL_VERIFICATION.md new file mode 100644 index 00000000..03adf4a8 --- /dev/null +++ b/docs/LOCAL_VERIFICATION.md @@ -0,0 +1,62 @@ +# 在本地验证未提交代码 + +验证脚本不提交、不推送、不触发 GitHub Actions，不部署生产。 +它会运行项目代码和隔离测试；数据库、Valkey 必须是可销毁的本机测试实例， +不能填写生产连接。源码记录使用未提交文件的真实内容指纹，而不把 HEAD 当作测试源码。 + +## 准备 + +使用 Python 3.15，uv 0.12.20。安装锁定的开发依赖： + +```sh +uv sync --locked --extra dev --python 3.15 +``` + +系统工具包括 PostgreSQL 客户端、initdb/pg_ctl（自行启动临时集群时）、 +Git/Git-LFS、age、OpenSSL、Nginx、bubblewrap。以普通用户运行。 +本机 bwrap 必须能够启动隔离子进程，不能为了通过测试更改生产安全策略。 + +`MSG_TEST_POSTGRES_URL_TEMPLATE` 可指向专用本机测试 PostgreSQL， +库名位置保留 `{database}`，测试将创建并清理随机数据库。 +不提供该变量时，由 fixture 使用 initdb/pg_ctl 启动仅 Unix socket 可访问的临时集群。 +`MSG_TEST_VALKEY_URL` 必须指向专用本机测试 Valkey，不能复用生产缓存实例。 + +旧版本测试需要准确的固定源码，本工作区已附 `.legacy-ledger`； +在其他 checkout 中可以仅从已有 Git 对象恢复，无需提交： + +```sh +mkdir -p .legacy-ledger +git archive ac083b3666b1b5268d90115129e2cc4e0dd11521 | tar -x -C .legacy-ledger +``` + +也可通过 `MSG_TEST_LEGACY_SOURCE` 指定等价目录。 +预检与既有迁移测试会校验 manifest 中的文件哈希。 + +## 运行 + +```sh +uv run --locked --extra dev --python 3.15 python scripts/verify_local.py --preflight-only +uv run --locked --extra dev --python 3.15 python scripts/verify_local.py --shards 8 +``` + +日志位于 `artifacts/local-<时间>/`。目录必须全新，不覆盖旧证据。 +本地八个分片默认串行，避免共享一个 Valkey 时出现跨分片干扰。 +脚本复用 `ci_shards.py` 的实际 pytest node-ID 并集检查，不用测试数量代替身份校验。 +之后执行 conformance、打包校验和独立客户端 wheel 安装/签名传输检查。 +任何阶段失败即返回非零状态；单独预检成功也不会标记整个验证通过。 + +`result.json` 包含基线 commit、dirty 状态、实际文件清单/摘要、命令、日志和退出码。 +验证前后源码改变会使整个结果失败。所有这些结果仍然不等于生产部署验收。 + +## 本次受限环境中的补充测试 + +当前会话只有 Python 3.13，不能运行目标版本业务模块。实际通过的是以下 +工具与配置测试，不加载 PostgreSQL 全局 fixture，也不代替上面的完整命令： + +```sh +PYTEST_DISABLE_PLUGIN_AUTOLOAD=1 python -m pytest --noconftest \ + -p pytest_asyncio.plugin \ + tests/test_release_version.py tests/test_release_artifacts.py \ + tests/test_ci_shards.py tests/test_local_verification.py \ + tests/test_local_postgres_fixtures.py +``` diff --git a/docs/NEW_ROOT_LEGACY_IMPORT_RUNBOOK.md b/docs/NEW_ROOT_LEGACY_IMPORT_RUNBOOK.md index 82b8df82..a8f86f21 100644 --- a/docs/NEW_ROOT_LEGACY_IMPORT_RUNBOOK.md +++ b/docs/NEW_ROOT_LEGACY_IMPORT_RUNBOOK.md @@ -50,29 +50,38 @@ from msg.client import ClientState, MsgClient from msg.transports.client import HTTPTransport from msg.transports.http import create_app + async def main(): app = Application(load_settings(Path(os.environ['CFG']))) try: await app.load() state = ClientState(Path(os.environ['CLIENT']), server=app.settings.service_url) - async with httpx.AsyncClient(transport=httpx.ASGITransport(app=create_app(app)), - base_url=app.settings.service_url) as http: + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=create_app(app)), base_url=app.settings.service_url + ) as http: client = MsgClient(state, HTTPTransport(state.server, http=http)) registered = await client.register(os.environ['NEW_HANDLE']) assert registered.status == 'ok', registered.error print('HOLDER', state.subject) for name in ('legacy-content', 'legacy-git', 'legacy-identities'): - result = await client.call('content.topic_create', {'parent':'/main','name':name}) + result = await client.call( + 'content.topic_create', {'parent': '/main', 'name': name} + ) assert result.status == 'ok', result.error ref = result.resources[0] async with app.metadata.transaction(write=False) as tx: generation = (await tx.resource(ref.id)).generation - result = await client.call('content.chmod', {'id':ref.id,'mode':'0700'}, - expected=((ref.id, generation),)) + result = await client.call( + 'content.chmod', + {'id': ref.id, 'mode': '0700'}, + expected=((ref.id, generation),), + ) assert result.status == 'ok', result.error print(name, ref.id) finally: await app.close() + + asyncio.run(main()) ``` @@ -141,47 +150,70 @@ from msg.config import load_settings from msg.core.codec import digest from msg.admin.legacy_approval import write_new + async def main(): - snapshot = Path(os.environ['MATERIALS'])/'offline-legacy-0221.sqlite' + snapshot = Path(os.environ['MATERIALS']) / 'offline-legacy-0221.sqlite' sha = os.environ['SQLITE_SHA256'] with snapshot.open('rb') as stream: assert hashlib.file_digest(stream, 'sha256').hexdigest() == sha - with sqlite3.connect(snapshot.resolve().as_uri()+'?mode=ro', uri=True) as db: + with sqlite3.connect(snapshot.resolve().as_uri() + '?mode=ro', uri=True) as db: db.execute('PRAGMA query_only=ON') candidates = [ (table, row[0], mode) for table, query, mode in ( - ('posts','SELECT id FROM posts ORDER BY id','0644'), - ('attachments','SELECT a.id FROM attachments a JOIN posts p ON p.id=a.post_id ORDER BY a.id','0644'), - ('boards','SELECT name FROM boards ORDER BY name','0755')) + ('posts', 'SELECT id FROM posts ORDER BY id', '0644'), + ( + 'attachments', + 'SELECT a.id FROM attachments a JOIN posts p ON p.id=a.post_id ORDER BY a.id', + '0644', + ), + ('boards', 'SELECT name FROM boards ORDER BY name', '0755'), + ) for row in db.execute(query) ] app = Application(load_settings(Path(os.environ['CFG']))) try: await app.load() async with app.metadata.transaction(write=False) as tx: - report = tx.setting('legacy-import:'+sha) + report = tx.setting('legacy-import:' + sha) assert report and report['source_sha256'] == sha entries = [] for table, old_id, mode in candidates: - rid = 'legacy_'+sha[:24]+'_'+digest((table,old_id))[7:31] + rid = 'legacy_' + sha[:24] + '_' + digest((table, old_id))[7:31] resource = await tx.resource(rid) assert resource.owner == os.environ['HOLDER'] and resource.state == 'active' - entries.append({'id':rid,'generation':resource.generation,'mode':mode,'source_table':table}) + entries.append({ + 'id': rid, + 'generation': resource.generation, + 'mode': mode, + 'source_table': table, + }) parent = await tx.resource(os.environ['CONTENT_PARENT']) assert parent.owner == os.environ['HOLDER'] and parent.mode == 0o700 for item in entries: assert parent.id in {r.id for r in await tx.ancestors(item['id'])} - entries.append({'id':parent.id,'generation':parent.generation,'mode':'0755','source_table':'publication_parent_last'}) - plan = {'source_sha256':sha,'holder':os.environ['HOLDER'],'entries':entries, - 'restores':'read-only-public-candidates','review_required':True, - 'source_code_commit':'e12eea32764656eec82452fee111e5b4fb6d8a42', - 'classification':'application-anonymous-read-candidate', - 'deployment_acl_review':None} - write_new(Path(os.environ['PUBLICATION_PLAN']),plan) - print({'candidate_count':len(entries)-1,'plan_digest':digest(plan)}) + entries.append({ + 'id': parent.id, + 'generation': parent.generation, + 'mode': '0755', + 'source_table': 'publication_parent_last', + }) + plan = { + 'source_sha256': sha, + 'holder': os.environ['HOLDER'], + 'entries': entries, + 'restores': 'read-only-public-candidates', + 'review_required': True, + 'source_code_commit': 'e12eea32764656eec82452fee111e5b4fb6d8a42', + 'classification': 'application-anonymous-read-candidate', + 'deployment_acl_review': None, + } + write_new(Path(os.environ['PUBLICATION_PLAN']), plan) + print({'candidate_count': len(entries) - 1, 'plan_digest': digest(plan)}) finally: await app.close() + + asyncio.run(main()) ``` diff --git a/docs/QUALITY_PROGRESS.md b/docs/QUALITY_PROGRESS.md new file mode 100644 index 00000000..1258fb0a --- /dev/null +++ b/docs/QUALITY_PROGRESS.md @@ -0,0 +1,117 @@ +# Python 3.15 质量与部署收口 · 2026-09-29 + +当前入口 PR #185，起点 main `25cf518a` / tree `61cc47b7`。上轮补丁 +SHA256 `118915fe14d589cf9f1b6e512f528c3a4af1fe2b76c8bdf3aab8d9930d955798` +精确匹配基线，全部本地修改保留。当前未部署生产。 +发布与现场交接入口见 [RELEASE_ACCEPTANCE.md](RELEASE_ACCEPTANCE.md)。 + +## 当前实际证据与修复 + +- 第一轮 e5238e77/tree874f78ed，Python3.15.0rc2、Ruff0.16.9、uv0.12.20。 + 质量运行36620632240实际15项诊断（14 I001、1 C420），stderr为空； + ZIP SHA256 `2894295a48ac07ad83bb60c68d32168cf9d0ae1e28efab7f760fb9d854b72059`。 + source.json及全部安全建议补丁已独立核对并应用，没有unsafe fix。 +- 完整运行36620632032八分片均在collection失败，37项ImportError共同原因是 + Ruff清理删掉msg.plugins.money._post_entry兼容导出。Storage36620631860 + 同因4项collection error，Recovery36620632239在全量collection拒绝。 + 没有将这些运行计为通过或完整执行。 +- 已恢复_post_entry、_amount和Delivery的_package为明确兼容导出，继续指向 + 唯一market owner；既有完整同对象矩阵、真实账本和receipt回归保持。 +- 新增Deployment rehearsal：源码逐字节wheel/sdist、uv.lock导出的server哈希锁、 + 干净安装、checkout外实际daemon启动/重启、四传输只读、签名客户端与幂等、 + doctor只读、默认禁外发worker及已有真实OpenSSH/Git/撤销脚本。 + 全部使用可销毁回环实例和独立Test Root，不访问生产或真实收件人。 +- 第二轮06d5e803/tree6bba0332：Recovery/Storage及工具、客户端、hosting、协作 + 专项已通过。部署36621959536的干净server安装、启动/重启、签名客户端、 + 四传输/doctor只读和worker步骤通过，真实sshd在认证阶段失败，未计为全通过。 + 将隔离daemon的PAM策略对齐现有部署配置，并补认证诊断与直接key lookup。 + 质量36621959251实际1个I001，已核验对应tree并应用安全格式补丁； + ZIP SHA256 `f3bd6265b6a7d657af3bf5d40269afaecc71af919fbb62a732a1f470d4d2aafe`。 +- e96e812b/tree91420337 的质量36622907866已通过，独立核验Ruff JSON为空、 + 两份stderr为空、581文件format通过；ZIP SHA256 + `915bc4e4c855a2a71caf83f3e09877bff71bad348eeb2643258c249fc2f81300`。 + 部署36622907569实际通过9项installed-server及7项真实sshd/Git检查； + ZIP SHA256 `57c366c56b39edd07becd293060ec3966ce86561e7dbf8f6481afe1505dafd4e`。 + source tree、wheel/sdist和server依赖锁摘要均已读回比对；不冒称全量测试已完成。 +- 补齐同一wheel的locked client-only安装和无dev依赖的正式selftest步骤。 + 这两项仍须在包含步骤的最终head执行，不能借上一轮安装成功直接记通过。 +- e96e812b全量运行36622907534已实际收集2518项，但八分片均失败： + 8个receipt effect-order参数例在hosting预探测业务数据时返回500； + 旧SQLite夹具仅提交事务而未关闭连接，严格ResourceWarning门禁报告泄漏。 + 将receipt只读effect门禁提前到hosting探测前，保持外层runtime freshness优先； + 旧夹具显式closing并保留提交/回滚语义，SQLite连接初始化和backup目标打开 + 失败也关闭已取得连接。补回归验证stale runtime不触达路由或hosting， + 以及连接初始化、目标打开和复制失败的完整关闭。未放宽警告或已有断言。 + +最终头仍须通过fresh Ruff/format/compile、精确node-ID八分片互斥并集、 +conformance/实际tokenizer、真实wheel/sdist/干净安装及适用专项；新失败继续修复。 +实际日志链、旧快照来源/冻结、新Root真实控制台、独立current pin、备份退役和 +目标机容量/持久性证据分别属于#64/#65/#68/#69/#70/#84，不能由隔离CI替代。 + +## 上轮本地证据归档 + +以下内容只对应提交前的本地交付阶段，不是当前分支状态或新CI结论。 + +## 当前状态 + +本地工作区基于已经合并的 main： +`25cf518a64ca2b6af820367caa4f5bb46dbc2f02`， +原始 tree 为 `61cc47b7abc25ccdcf163d59804e323ea6a10649`。 +已从已有 Git bundle 恢复并核对远端已发布 commit/tree 的准确哈希。 +本轮没有创建业务提交、推送、触发云端工作流或执行生产部署。 +所有代码修改均留在工作区，HEAD 仍是以上基线。 + +此前质量候选只作为本地修复来源，没有套用相对旧 head 的补丁： +先重建准确 main，再重新生成相对该 main 的差异。 +Ruff 的自动修复/格式化来源是此前真实云端运行；后续人工修改 +和本轮修改尚未经过最终 Ruff 0.16.9 验证，不能标记为零诊断。 + +## 本轮新增修复 + +- `src/msg/__init__.py` 的运行时版本从错误的 `1.0.0a1` 对齐至发行版本 `0.1.0a1`。 +- 构建校验从仅检查资源/命令名称扩展到完整包文件集合、逐字节 Python 源码、 + Name/Version/Requires-Python、准确 console target、sdist 当前源码和锁文件。 + 旧代码、额外的已移除模块和错误元数据不能再被校验脚本当作合格产物。 +- 本地 PostgreSQL 测试显式创建并使用 `msgtest` 临时角色，不依赖开发者的系统用户名。 +- 安装测试 fixture 使用 finally 回收应用和临时数据库；初始化失败也清理， + session 级 seed 数据库有明确 teardown，不再永久残留在配置的测试集群中。 +- `scripts/verify_local.py` 提供本地检查入口，记录实际未提交文件指纹、退出码与日志； + 缺依赖、非零退出、要求清空的 stderr、跳过的 conformance、源码漂移都不会标为通过。 +- uv 的版本要求写入 `pyproject.toml`；旧 uv 的真实拒绝结果已记录。 +- 收拢重复的显式 import，保持导入源、名称、别名及绑定顺序，减少 695 行展开代码。 + +## 实际运行结果及范围 + +现有可执行环境是 Python 3.13.5、uv 0.10.0，不是目标环境。 +新增发布回归先得到 8 个失败/2 个通过；修复后通过。 +新增 fixture 生命周期回归先得到 4 个失败；修复后通过。 + +最终本地补充回归：54 passed，0 failed / 0 error / 0 skipped。 +包含 29 个既有 CI 分片校验测试和 25 个新增工具/配置/fixture 编排测试。 +测试通过 `--noconftest` 运行指定的五个不依赖真实数据库的测试文件； +fixture 生命周期测试使用模拟进程/应用检查编排，不是 PostgreSQL 实测。 +产物校验测试使用合成档案，不是本项目真实 wheel/sdist 构建成功的证据。 +没有把这些结果当作 Python 3.15 全量业务测试。 + +附加静态源码审查没有发现未绑定的全局名称或额外语法错误。 +该审查仅为了让 Python 3.13 的解析器读取源码，临时规范化了 +18 个文件中的 Python 3.14+ except 语法，没有修改生产文件，也没有运行这些模块。 +它不代替 Ruff、目标版本编译或业务测试。 + +固定旧版源代码 `ac083b3666b1b5268d90115129e2cc4e0dd11521` +已恢复至本地 `.legacy-ledger`，按既有 manifest 验证了 8 个文件的 SHA-256； +没有因此宣称数据库迁移/回滚已执行。 + +## 仍未完成 + +本环境没有 Python 3.15、Ruff、build、psycopg、graphql-core、valkey、 +tiktoken 和若干系统测试工具。pip/uv 下载及 GitHub 域名解析未成功。 +`verify_local.py --preflight-only` 实际退出 1，结果是 blocked。 + +因此最终 Ruff/format、锁定环境安装、Python 3.15 全量测试、真实 +PostgreSQL/Valkey/Git-LFS/age/bwrap 验收、conformance/tokenizer、 +真实 wheel/sdist 和干净安装仍未完成。当前不能标记为可部署或全部完成。 +生产日志、旧库、Root、本机控制台、恢复授权和备份退役仍有各自验收边界。 + +本地执行入口及步骤见 `LOCAL_VERIFICATION.md`。会话交付包的 evidence/ +保存真实日志、JUnit 和环境阻塞记录；不借用此前其他提交的 CI 成功。 diff --git a/docs/RELEASE_ACCEPTANCE.md b/docs/RELEASE_ACCEPTANCE.md new file mode 100644 index 00000000..3aaf8da4 --- /dev/null +++ b/docs/RELEASE_ACCEPTANCE.md @@ -0,0 +1,69 @@ +# 发布验收与现场交接 + +本页是 PR #185 的当前入口，取代把历史进度数字当作当前部署结论的做法。 +源码基线为已合并 #176 的 `25cf518a`；#182/#183/#184 均已在该历史。 +每次最终验证以产物 `source.json` 中的 commit/tree、对应 workflow 和完整 JUnit +node-ID 为准。PR 的模拟 merge commit 与分支 head 可不同，必须核对实际 tree。 +本页索引测试，不单凭测试文件名声称原设计所有分支或生产已经验收。 + +## 必须同时通过的发布闸门 + +| 闸门 | 可复查证据 | 拒绝条件 | +| --- | --- | --- | +| Python quality | Ruff 0.16.9 JSON、空 stderr、format、Python 3.15 compile、工具链/源码记录 | 任一诊断、warning、格式/编译失败 | +| Rewrite contracts | 八分片 manifest/JUnit、完整 node-ID 互斥并集、conformance、实际 tokenizer、wheel/sdist | 失败、错误、跳过、重复、漏项或不一致源码 | +| Deployment rehearsal | 同一 wheel 的哈希锁 server/client 干净安装、真实启动/重启、四传输、doctor、worker、真实 sshd/Git、正式 selftest | 包代码/元数据不符、未测依赖、安装/执行/清理失败 | +| 适用专项 | Recovery、Client、Hosting、Tool、Storage、Credential、架构、协作与 Transfer 的当前运行 | 不用旧分支 success 替代当前结果 | + +部署演练仅用可销毁 PostgreSQL、回环监听器、独立 Test Root 与测试 SSH keys。 +server 包在 checkout 外运行，Root 目录确实不可由网络账号读取；重启后同请求 +返回原结果。四传输读和 doctor 比对所有表行值与 sequence，不只比数量。 +真实 sshd 不改管理员 SSH：保留 PAM 账户检查、StrictModes、公钥认证、禁 shell/ +转发；验证多 ref push/clone，撤销同时作用于新连接和已有 ControlMaster 的新通道。 +client-only 包不得安装或导入 server runtime；其传输检查使用 MockTransport， +不冒称真实远程服务器连接。安装包正式 selftest 的 Test Root 也不代表真实物理控制台。 + +通过结果与 ZIP/wheel/锁文件摘要附在 PR #185 和 #83，不为写入成功数字再改代码树。 + +## 开放 issue 的源码与验收入口 + +下面是具体已有断言的定位，不是“文件存在即关闭”的判定；完整条款仍按各 issue +与权威设计审查。跨表/跨存储、当前授权、历史签署字节和只读零副作用的断言不能放宽。 + +| issue | 当前核对入口 | 仍独立需要的证据 | +| --- | --- | --- | +| #64 | `test_url_secrets`、`test_request_target_safety`、真实 `test_nginx_secret_logs` | 实际所有监听器/CDN/代理/APM/日志链及部署配置来源 | +| #65 | 固定旧源码 fixture、`test_legacy_ledger_snapshot`、`test_ledger_migration_references/json` | 真实存量快照的合法来源、保护摘要、冻结时间与该快照迁移/回滚 | +| #68 | `test_custodial_lifecycle/inventory_ack/historical_migration`、`test_custody_exit_evidence`、`test_backup_retirement_attestation` | 真实 retained 密文逐项解密；独立备份范围退役，不推导自动销毁 | +| #69 | `test_complete_recovery_proof`、`test_recovery_authority_reconcile/current_authority/runtime_generation` | rollback 集之外的当前可信 pin、真实恢复状态及受控本机 promotion | +| #70 | `test_online_ca_policy`、`test_root_rotation/resume`、`test_task_a_root_files`、`test_money_admin` | 真实 VT/串口控制台、有限 CA 清单与明确重签/撤销批准 | +| #72 | `test_market_71/72`、`test_bounty`、`test_market_explicit_acceptance/recovery/notification_reuse`、正式 `market_e2e` | 新/旧政策分别核验，不将自动 settled 写成买家已 claimed/accepted；无生产资金操作 | +| #76 | `test_authorization_sources/matrix/recovery_boundaries`、`test_share_transport_matrix`、`test_read_projection_cache_transports` | 全条款逐来源/表示审查；现场启用方式归 #84 | +| #77 | `test_personal_content_signatures`、`test_notes_todos`、`test_legacy_directive`、`test_achievements/achievement_pins/honor_diagnostics` | 全保留历史和当前失效事实；荣誉不授权限、不宣称非人认证 | +| #78 | `test_read_entry_contract_matrix`、`test_read_query_keyset_boundary`、`test_search_source_relation_filters`、`issue_78_80`、cursor/Sync 回归 | 完整查询/输出/成本矩阵；不将概念名称偷偷作为新 wire 别名 | +| #79 | `test_file_operations/copy_integrity`、`test_text_patch_rebase_batch`、`test_patch_rebase_identity`、`test_personal_content_signatures`、`issue_78_80` | 完整文件/patch/Revision 矩阵；DB 原子提交不冒称物理文件回滚 | +| #80 | `test_transfer_status_cursors`、真实 `test_git_http_push/git_lfs`、`test_storage_commit_faults/lfs_shared_gc_quota`、真实 sshd 演练 | 目标共享卷/多实例拓扑的容量、断电、GC 与一致恢复 | +| #81 | `test_smtp_socket_acceptance`、`test_webhook_*`、`test_effect_completion_fencing`、`test_collaboration_*`、`test_lease_list_bounds/deadline` | 受控外发接收端及实际部署开关；lease 不授权或充当排他锁 | +| #82 | CLI/TUI/stdio 回归、`test_tool_sandbox_acceptance`、真实 sshd 演练、client-only 安装 | 目标机实际路径/PAM/systemd/隔离；TUI 只读导航，正式写入走签名 CLI | +| #83 | `DESIGN_CONTRACT_INVENTORY`、`test_configuration_field_contracts`、`test_manifest_feature_linkage`、Registry/route 矩阵、正式 selftest | 导航/正则匹配不等于逐条断言；现场闸门保留 | +| #84 | 本页三层闸门、`test_capacity_bounds`、`test_restore_physical_failure`、只读 preflight | 真实主机、旧数据、Root、日志、容量、备份/切流与回滚批准 | +| #85 | `test_market_boundaries`、`test_architecture_ports`、`test_client_boundary`、session/storage 边界、严格 Ruff | 保留唯一 owner 和兼容导出；不以删断言/unsafe fix/skip 换通过 | + +旧 issue 中“#103 未合并”“完整 recovery promotion 未实现”“工具没有 concurrency +闸门”“没有 receipts 路径”等记录只归属其当时基线，不能覆盖当前已合并源码。 +新默认显式验收与旧自动结算分别通过版本/policy 保持，不重写历史 receipt。 + +## 现场放行所需材料 + +1. 目标主机与只读运维入口、准确的部署版本/解释器/配置摘要、服务与数据库角色、 + 所有 listener/vhost/upstream/日志和 APM 链清单。公开 health 200 不能证明新版本就绪。 +2. 真实旧安装的受保护备份及 manifest、来源/冻结时间、旧 Root/CA 公开清单和 + 已批准的新 Root 导入/身份映射。不要将备份、PIN、私钥或凭据贴到公开 issue。 +3. 授权操作者在真实 VT/串口完成 Root/CA 审核；独立 current pin 和实际备份 + 退役证明。SSH、TTY 伪装、数据库布尔值或删除 quarantine 文件均不能代替。 +4. 目标拓扑的隔离容量/崩溃/恢复演练、适用 SMTP/Webhook 的受控接收端；未启用 + 渠道仍验证零外发。提供切流/停止/回滚条件；产生新写入后不得直接退回旧 SQLite。 + +`python -m msg.admin.preflight --config /explicit/target/config` 仅执行读事务，输出 +来源、账本/CA 与缺失证据。其 `decision=blocked`/退出 2 是明确未放行，不是可忽略错误。 +没有实际材料时保留对应 issue 开放；CI 全绿仅允许交付这个准确源码树的发布包。 diff --git a/docs/SSH_GIT_LOCAL_ACCEPTANCE.md b/docs/SSH_GIT_LOCAL_ACCEPTANCE.md index 91d8dce3..eb1476ff 100644 --- a/docs/SSH_GIT_LOCAL_ACCEPTANCE.md +++ b/docs/SSH_GIT_LOCAL_ACCEPTANCE.md @@ -20,6 +20,15 @@ wrapper and Root directory, and destroys the disposable installation/database. The script is deliberately separate from ordinary pytest: it requires sudo and real OpenSSH and fails if they are unavailable. +The release workflow now runs this script with the installed wheel outside the +checkout on Ubuntu 24.04. Its dedicated daemon uses `UsePAM yes`, matching +`deploy/sshd_config`, so the host PAM account/session policy applies while +authentication remains public-key only. It does not unlock or modify the service +account. `StrictModes yes` is explicit, and a direct dynamic key lookup is checked +before connecting. The disposable daemon logs authentication diagnostics using +test keys only; this does not change the deployed logging policy. The artifact +records the actual Python/OpenSSH versions and the source tree. + ## Real local checks - Registered SSH possession proof and capability ceiling; dynamic diff --git a/docs/VERIFICATION.md b/docs/VERIFICATION.md index 5e16c544..137ae01b 100644 --- a/docs/VERIFICATION.md +++ b/docs/VERIFICATION.md @@ -4,17 +4,25 @@ ## 本地与 CI -项目要求 Python 3.15。`tests/` 使用 PostgreSQL；不设置 `MSG_TEST_POSTGRES_URL_TEMPLATE` 时，根目录 `conftest.py` 尝试启动临时 PostgreSQL。CI 在 `.github/workflows/ci.yml` 配置 PostgreSQL、Valkey、age、Nginx、OpenSSL 和 Git LFS，并使用四个测试分片。 +项目要求 Python 3.15。`tests/` 使用 PostgreSQL；不设置 `MSG_TEST_POSTGRES_URL_TEMPLATE` 时，根目录 `conftest.py` 尝试启动临时 PostgreSQL。CI 在 `.github/workflows/ci.yml` 配置 PostgreSQL、Valkey、age、Nginx、OpenSSL 和 Git LFS，并使用八个测试分片。 ```bash -python3.15 -m pip install -e '.[dev]' -python3.15 -m compileall -q src -python3.15 -m pytest tests -python3.15 -m pytest conformance -python3.15 -m build +uv sync --locked --extra dev --python 3.15 +uv run --locked --extra dev ruff check . --output-format json +uv run --locked --extra dev ruff format --check . +uv run --locked --extra dev python -W error -m compileall -q src tests conformance scripts +uv run --locked --extra dev python -m pytest tests +uv run --locked --extra dev python -m pytest conformance +uv run --locked --extra dev python -m build +uv run --locked --extra dev python scripts/check_package_artifacts.py dist ``` -完整 CI 还要求四个分片的 JUnit 测试节点互不重复且覆盖完整收集清单；不能用一组定向测试或旧提交的成功运行代替当前提交的结果。具体命令和环境以 [CI 工作流](../.github/workflows/ci.yml) 为准。 +完整 CI 还要求八个分片的 JUnit 测试节点互不重复且覆盖完整收集清单；不能用一组定向测试或旧提交的成功运行代替当前提交的结果。具体命令和环境以 [CI 工作流](../.github/workflows/ci.yml) 为准。 + +安装包的独立演练见 [deployment-rehearsal 工作流](../.github/workflows/deployment-rehearsal.yml)。 +它从同一 uv.lock 导出哈希依赖锁，分别验收 client-only/server 安装和正式 selftest， +并运行真实回环 daemon 与 OpenSSH/Git。逐 issue 的当前入口和现场材料见 +[发布验收与交接](RELEASE_ACCEPTANCE.md)，不能从隔离演练推导生产已切流。 ## 部署验收 diff --git a/pyproject.toml b/pyproject.toml index 6eae7358..e538fbb2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -41,6 +41,9 @@ dev = [ "tiktoken>=0.12", ] +[tool.uv] +required-version = "==0.12.20" + [tool.uv.build-backend] module-name = "msg" source-include = [ @@ -74,5 +77,8 @@ select = [ ] explicit-preview-rules = true +[tool.ruff.lint.isort] +combine-as-imports = true + [tool.ruff.format] quote-style = "single" diff --git a/scripts/check_client_install.py b/scripts/check_client_install.py index f12ed662..308f9177 100644 --- a/scripts/check_client_install.py +++ b/scripts/check_client_install.py @@ -3,6 +3,7 @@ Run with --minimal-install in a fresh wheel-only virtualenv, outside the source checkout. Network traffic uses httpx.MockTransport, not a remote service. """ + from __future__ import annotations import argparse @@ -11,15 +12,24 @@ import importlib.abc import importlib.util import json -from pathlib import Path import stat import sys import tempfile +from pathlib import Path SERVER_DEPENDENCIES = ('starlette', 'uvicorn', 'psycopg', 'valkey', 'aiohttp', 'dns', 'graphql') -FORBIDDEN = ('msg.application', 'msg.core.executor', 'msg.storage', 'msg.admin', - 'msg.workers', 'msg.security.vault', 'msg.security.custodial_migration', - 'msg.transports.http', 'msg.transports.graphql', 'msg.transports.mcp') +FORBIDDEN = ( + 'msg.application', + 'msg.core.executor', + 'msg.storage', + 'msg.admin', + 'msg.workers', + 'msg.security.vault', + 'msg.security.custodial_migration', + 'msg.transports.http', + 'msg.transports.graphql', + 'msg.transports.mcp', +) class ClientBoundary(importlib.abc.MetaPathFinder): @@ -31,6 +41,7 @@ def find_spec(self, fullname, path=None, target=None): async def check(directory): import httpx + from msg.client import ClientState from msg.core.codec import canonical, loads, wire from msg.core.requests import request_for, signing_bytes @@ -38,8 +49,18 @@ async def check(directory): from msg.transports.client import TRANSPORTS from msg.transports.packet import decode_packet, path_packet - for name in ('cli', 'tui', 'client_certificates', 'client_content', 'client_custodial', - 'client_market', 'client_recovery', 'client_secrets', 'client_tokens', 'client_upgrade'): + for name in ( + 'cli', + 'tui', + 'client_certificates', + 'client_content', + 'client_custodial', + 'client_market', + 'client_recovery', + 'client_secrets', + 'client_tokens', + 'client_upgrade', + ): importlib.import_module('msg.' + name) state = ClientState(directory, server='https://unit.invalid') signer = Ed25519Signer.from_bytes(bytes(range(32))) @@ -51,17 +72,33 @@ async def check(directory): for path in (state.path, state.key_path, state.age_key_path): assert stat.S_IMODE(path.stat().st_mode) == 0o600 assert stat.S_IMODE(directory.stat().st_mode) == 0o700 - request = request_for('content.read', {'id': 'r_test'}, state.server, - subject='u_test', signer=signer, request_id='client-boundary') + request = request_for( + 'content.read', + {'id': 'r_test'}, + state.server, + subject='u_test', + signer=signer, + request_id='client-boundary', + ) calls = [] def respond(incoming): path = incoming.url.path if path == '/_transports': - return httpx.Response(200, json={'version': 1, 'target_service': state.server, - 'operations': {'content.read': 'read'}, 'limits': { - 'max_request_bytes': 1048576, 'max_response_bytes': 1048576, - 'max_path_bytes': 8192, 'encodings': ['j', 'gz']}}) + return httpx.Response( + 200, + json={ + 'version': 1, + 'target_service': state.server, + 'operations': {'content.read': 'read'}, + 'limits': { + 'max_request_bytes': 1048576, + 'max_response_bytes': 1048576, + 'max_path_bytes': 8192, + 'encodings': ['j', 'gz'], + }, + }, + ) if path.startswith('/-/g/'): _, _, _, operation, encoding, encoded = path.split('/') packet = path_packet(encoded, encoding, 1048576) @@ -84,24 +121,38 @@ def respond(incoming): assert canonical(packet) == canonical(request) verify(signer.public_key, signing_bytes(packet), packet.proof.signature, purpose='request') calls.append(kind) - result = {'request_id': packet.request_id, 'operation': packet.operation, 'status': 'ok', - 'actor': 'u_test', 'subject': 'u_test', 'resources': [{'id': 'r_test'}], - 'data': {'content': 'signed client response'}} + result = { + 'request_id': packet.request_id, + 'operation': packet.operation, + 'status': 'ok', + 'actor': 'u_test', + 'subject': 'u_test', + 'resources': [{'id': 'r_test'}], + 'data': {'content': 'signed client response'}, + } if kind == 'graphql': result = {'data': {'call': result}} elif kind == 'mcp_http': - result = {'jsonrpc': '2.0', 'id': packet.request_id, - 'result': {'structuredContent': result}} + result = { + 'jsonrpc': '2.0', + 'id': packet.request_id, + 'result': {'structuredContent': result}, + } return httpx.Response(200, json=result) async with httpx.AsyncClient(transport=httpx.MockTransport(respond)) as http: for name, transport in TRANSPORTS.items(): result = await transport(state.server, http=http).call(request) - assert result.status == 'ok' and result.resources[0].id == 'r_test', (name, wire(result)) + assert result.status == 'ok' and result.resources[0].id == 'r_test', ( + name, + wire(result), + ) assert result.data['content'] == 'signed client response' assert sorted(calls) == sorted(TRANSPORTS) - assert not any(any(name == prefix or name.startswith(prefix + '.') for prefix in FORBIDDEN) - for name in sys.modules) + assert not any( + any(name == prefix or name.startswith(prefix + '.') for prefix in FORBIDDEN) + for name in sys.modules + ) return calls @@ -110,14 +161,23 @@ def main(): parser.add_argument('--minimal-install', action='store_true') args = parser.parse_args() if args.minimal_install: - present = [name for name in SERVER_DEPENDENCIES if importlib.util.find_spec(name) is not None] - assert not present, 'server-only Python dependencies in client environment: ' + ', '.join(present) + present = [ + name for name in SERVER_DEPENDENCIES if importlib.util.find_spec(name) is not None + ] + assert not present, 'server-only Python dependencies in client environment: ' + ', '.join( + present + ) sys.meta_path.insert(0, ClientBoundary()) with tempfile.TemporaryDirectory(prefix='msg-client-install-') as folder: calls = asyncio.run(check(Path(folder) / 'client')) - print(json.dumps({'status': 'ok', 'transports': sorted(calls), - 'server_implementation_imported': False, - 'minimal_install': args.minimal_install})) + print( + json.dumps({ + 'status': 'ok', + 'transports': sorted(calls), + 'server_implementation_imported': False, + 'minimal_install': args.minimal_install, + }) + ) if __name__ == '__main__': diff --git a/scripts/check_installed_server.py b/scripts/check_installed_server.py new file mode 100644 index 00000000..ffbaef1b --- /dev/null +++ b/scripts/check_installed_server.py @@ -0,0 +1,197 @@ +"""Exercise a clean installed server on loopback with disposable data and Root. + +Run with -I outside the checkout as a non-root user with sudo -n. The only +privileged path is this invocation's newly generated temporary Root directory. +No production configuration, listener, trust or external recipient is used. +""" + +from __future__ import annotations + +import asyncio +import hashlib +import importlib.metadata +import json +import os +import socket +import subprocess +import sys +import tempfile +from pathlib import Path + +import httpx +import psycopg +from psycopg import sql + +import msg +from msg.admin.diagnostics import temporary_postgres +from msg.admin.root import _approve_csr, _provision +from msg.application import Application +from msg.client import ClientState, MsgClient +from msg.config import write_example +from msg.core.codec import canonical +from msg.transports.client import ( + GraphQLTransport, + HTTPTransport, + MCPHTTPTransport, + PathGETTransport, +) + + +def run(*command): + return subprocess.run(command, capture_output=True, text=True, timeout=60, check=True) + + +def database_digest(dsn): + """Read complete row/sequence values without advancing PostgreSQL sequences.""" + hasher = hashlib.sha256() + with psycopg.connect(dsn) as connection: + connection.execute('SET TRANSACTION ISOLATION LEVEL REPEATABLE READ, READ ONLY') + tables = connection.execute( + "SELECT tablename FROM pg_tables WHERE schemaname='public' ORDER BY tablename" + ).fetchall() + for (table,) in tables: + hasher.update(canonical(table)) + rows = connection.execute(sql.SQL('SELECT * FROM {}').format(sql.Identifier(table))) + for row in sorted(canonical(row) for row in rows): + hasher.update(len(row).to_bytes(8, 'big')) + hasher.update(row) + sequences = connection.execute( + "SELECT sequencename FROM pg_sequences WHERE schemaname='public' ORDER BY sequencename" + ).fetchall() + for (sequence,) in sequences: + value = connection.execute( + sql.SQL('SELECT last_value,is_called FROM {}').format(sql.Identifier(sequence)) + ).fetchone() + hasher.update(canonical((sequence, value))) + return hasher.hexdigest() + + +async def acceptance(): + assert os.geteuid() != 0 + assert 'site-packages' in Path(msg.__file__).parts, 'Use the installed wheel with -I' + assert msg.__version__ == importlib.metadata.version('msgctl') + checks = ['installed_wheel_and_version'] + with ( + tempfile.TemporaryDirectory(prefix='msg-installed-rehearsal-') as temporary, + temporary_postgres() as dsn, + ): + folder = Path(temporary) + with socket.socket() as probe: + probe.bind(('127.0.0.1', 0)) + port = probe.getsockname()[1] + origin = f'http://127.0.0.1:{port}' + settings = write_example(folder / 'etc', folder / 'data', origin, postgres_dsn=dsn) + assert settings.root_private_dir == folder / 'etc-root' + assert settings.server.mail is None and not settings.server.valkey_url + config = folder / 'etc/msgd.toml' + config.write_text(config.read_text().replace('port = 8042', f'port = {port}')) + app = Application(settings) + process = None + protected = False + executable = Path(sys.executable).with_name('msgd') + log = None + try: + try: + csr, root = await _provision(app, 'test-only-' + os.urandom(24).hex()) + await _approve_csr( + app, csr, root, expected_digest=None, operator='isolated-rehearsal' + ) + finally: + await app.close() + run('sudo', '-n', 'chown', '-R', '0:0', str(settings.root_private_dir)) + protected = True + assert not os.access(settings.root_private_dir, os.R_OK | os.X_OK) + checks.append('root_inaccessible_to_network_user') + log = (folder / 'server.log').open('w') + + async def start(): + child = subprocess.Popen( + [str(executable), '--config-dir', str(settings.config_dir), 'serve'], + stdout=log, + stderr=log, + ) + try: + async with httpx.AsyncClient(timeout=1, trust_env=False) as http: + for _ in range(100): + assert child.poll() is None, 'Installed daemon exited during startup' + try: + response = await http.get(origin + '/healthz') + if response.status_code == 200: + assert response.json() == {'status': 'ok'} + return child + except httpx.NetworkError: + pass + await asyncio.sleep(0.05) + raise AssertionError('Installed daemon never became ready') + except BaseException: + child.terminate() + child.wait(timeout=15) + raise + + process = await start() + checks.append('real_daemon_startup') + state = ClientState(folder / 'client', server=origin) + async with httpx.AsyncClient(timeout=10, trust_env=False) as http: + client = MsgClient(state, HTTPTransport(origin, http=http)) + registered = await client.register('rehearsal-client') + assert registered.status == 'ok' + packet = client.prepare( + 'content.post_create', + {'parent': '/main', 'body': 'installed release\r\n'}, + request_id='restart-stable-write', + ) + posted = await client.send(packet) + assert posted.status == 'ok' + checks.append('real_signed_registration_and_write') + process.terminate() + process.wait(timeout=15) + process = await start() + repeated = await client.send(packet) + assert repeated.status == 'ok' and repeated.replayed + assert repeated.resources == posted.resources + checks.append('restart_preserves_identity_content_and_idempotency') + before = database_digest(dsn) + for transport in ( + HTTPTransport, + PathGETTransport, + GraphQLTransport, + MCPHTTPTransport, + ): + reader = MsgClient(state, transport(origin, http=http)) + read = await reader.call('discovery.get', {'id': posted.resources[0].id}) + assert read.status == 'ok' and read.data['content'] == 'installed release\r\n' + assert database_digest(dsn) == before + checks.append('four_real_transports_preserve_all_database_facts') + before = database_digest(dsn) + diagnosed = json.loads( + run(str(executable), '--config-dir', str(settings.config_dir), 'doctor').stdout + ) + assert diagnosed['ok'] is True + assert database_digest(dsn) == before + checks.append('installed_doctor_is_read_only') + run(str(executable), '--config-dir', str(settings.config_dir), 'worker', '--once') + checks.append('installed_worker_without_mail_or_valkey') + process.terminate() + process.wait(timeout=15) + process = None + checks.append('clean_daemon_shutdown') + finally: + if process is not None and process.poll() is None: + process.terminate() + process.wait(timeout=15) + if log is not None: + log.close() + if protected: + run('sudo', '-n', 'rm', '-rf', '--', str(settings.root_private_dir)) + assert not folder.exists() + return { + 'checks': checks, + 'python': sys.version, + 'version': msg.__version__, + 'temporary_installation_removed': True, + 'external_recipients': 0, + } + + +if __name__ == '__main__': + print(json.dumps(asyncio.run(acceptance()), indent=2)) diff --git a/scripts/check_isolated_sshd.py b/scripts/check_isolated_sshd.py index a46e0eb3..f5cd0d83 100644 --- a/scripts/check_isolated_sshd.py +++ b/scripts/check_isolated_sshd.py @@ -4,11 +4,10 @@ Run with the installed project's Python as an ordinary user with sudo -n. Requires PostgreSQL tools, OpenSSH and Git. No production configuration is read. """ + import asyncio -from datetime import timedelta import json import os -from pathlib import Path import pwd import shlex import shutil @@ -16,19 +15,21 @@ import subprocess import sys import tempfile +from datetime import timedelta +from pathlib import Path from cryptography.hazmat.primitives import serialization from msg.admin.diagnostics import temporary_postgres -from msg.admin.root import _provision, _approve_csr, _revoke +from msg.admin.root import _approve_csr, _provision, _revoke from msg.application import Application from msg.config import write_example +from msg.constants import ROOT_SUBJECT from msg.core.codec import b64, canonical, wire -from msg.core.requests import request_for from msg.core.models import CertificateRequest, Signature -from msg.constants import ROOT_SUBJECT -from msg.security.certificates import csr_body +from msg.core.requests import request_for from msg.security.age_keys import generate_age_key +from msg.security.certificates import csr_body from msg.security.crypto import Ed25519Signer, subject_id @@ -42,9 +43,16 @@ async def acceptance(): sshd = shutil.which('sshd') assert sshd and shutil.which('ssh') and shutil.which('ssh-keygen') checks = [] - with tempfile.TemporaryDirectory(prefix='msg-sshd-check-') as temporary, temporary_postgres() as dsn: + with ( + tempfile.TemporaryDirectory(prefix='msg-sshd-check-') as temporary, + temporary_postgres() as dsn, + ): folder = Path(temporary) - app = Application(write_example(folder/'etc', folder/'data', 'https://sshd-check.invalid', postgres_dsn=dsn)) + app = Application( + write_example( + folder / 'etc', folder / 'data', 'https://sshd-check.invalid', postgres_dsn=dsn + ) + ) assert app.settings.root_private_dir.is_relative_to(folder) server = None master = None @@ -53,65 +61,157 @@ async def acceptance(): csr, root = await _provision(app, 'isolated-sshd-' + os.urandom(24).hex()) await _approve_csr(app, csr, root, expected_digest=None, operator='isolated-sshd-check') # The service account must actually be unable to read root material. - assert run('sudo', '-n', 'chown', '-R', '0:0', str(app.settings.root_private_dir)).returncode == 0 + assert ( + run( + 'sudo', '-n', 'chown', '-R', '0:0', str(app.settings.root_private_dir) + ).returncode + == 0 + ) key = Ed25519Signer.generate() uid = subject_id(key.public_key) async def call(name, args): - packet = request_for(name, args, app.settings.service_url, signer=key, subject=uid, - expires_at=app.clock()+timedelta(seconds=120)) + packet = request_for( + name, + args, + app.settings.service_url, + signer=key, + subject=uid, + expires_at=app.clock() + timedelta(seconds=120), + ) return await app.executor.execute(packet) _, recipient = generate_age_key() - packet = request_for('identity.register', {'handle': 'ssh-check', 'public_key': b64(key.public_key), - 'encryption_recipient': recipient}, app.settings.service_url, signer=key, subject=uid, - contract_version=2, expires_at=app.clock()+timedelta(seconds=120)) + packet = request_for( + 'identity.register', + { + 'handle': 'ssh-check', + 'public_key': b64(key.public_key), + 'encryption_recipient': recipient, + }, + app.settings.service_url, + signer=key, + subject=uid, + contract_version=2, + expires_at=app.clock() + timedelta(seconds=120), + ) assert (await app.executor.execute(packet)).status == 'ok' repo = await call('git.create', {'parent': uid, 'name': 'code.git'}) assert repo.status == 'ok' - client_key = folder/'client-key' - assert run('ssh-keygen', '-q', '-t', 'ed25519', '-N', '', '-f', str(client_key)).returncode == 0 + client_key = folder / 'client-key' + assert ( + run('ssh-keygen', '-q', '-t', 'ed25519', '-N', '', '-f', str(client_key)).returncode + == 0 + ) private = serialization.load_ssh_private_key(client_key.read_bytes(), password=None) - signer = Ed25519Signer.from_bytes(private.private_bytes( - serialization.Encoding.Raw, serialization.PrivateFormat.Raw, serialization.NoEncryption())) + signer = Ed25519Signer.from_bytes( + private.private_bytes( + serialization.Encoding.Raw, + serialization.PrivateFormat.Raw, + serialization.NoEncryption(), + ) + ) line = ' '.join(client_key.with_suffix('.pub').read_text().split()[:2]) - proof = signer.sign(canonical({'subject_id': uid, 'public_key': line}), purpose='ssh-key-add') - grants = [g for g in app.base_grants() if g.capability in {'git.basic', 'discovery.basic'}] - added = await call('identity.ssh_key_add', {'public_key': line, 'proof': wire(proof), - 'ceiling': wire(grants)}) + proof = signer.sign( + canonical({'subject_id': uid, 'public_key': line}), purpose='ssh-key-add' + ) + grants = [ + g for g in app.base_grants() if g.capability in {'git.basic', 'discovery.basic'} + ] + added = await call( + 'identity.ssh_key_add', + {'public_key': line, 'proof': wire(proof), 'ceiling': wire(grants)}, + ) assert added.status == 'ok' - csr = CertificateRequest(resource_id='pending', applicant=uid, subject_id=uid, - requested_issuer=ROOT_SUBJECT, public_key=signer.public_key, kind='capability', - grants=tuple(grants), issuance=None, requested_ttl_seconds=600, - target_service=app.settings.service_url, delegation_depth=0, authority_sources=(), - request_digest='', possession_proof=Signature(key_id=signer.key_id, - algorithm='ed25519', value=b'')) + csr = CertificateRequest( + resource_id='pending', + applicant=uid, + subject_id=uid, + requested_issuer=ROOT_SUBJECT, + public_key=signer.public_key, + kind='capability', + grants=tuple(grants), + issuance=None, + requested_ttl_seconds=600, + target_service=app.settings.service_url, + delegation_depth=0, + authority_sources=(), + request_digest='', + possession_proof=Signature(key_id=signer.key_id, algorithm='ed25519', value=b''), + ) possession = signer.sign(canonical(csr_body(csr)), purpose='csr') - requested = await call('cert.request', {'requested_issuer': ROOT_SUBJECT, - 'public_key': b64(signer.public_key), 'kind': 'capability', 'grants': wire(grants), - 'requested_ttl_seconds': 600, 'possession_proof': wire(possession)}) + requested = await call( + 'cert.request', + { + 'requested_issuer': ROOT_SUBJECT, + 'public_key': b64(signer.public_key), + 'kind': 'capability', + 'grants': wire(grants), + 'requested_ttl_seconds': 600, + 'possession_proof': wire(possession), + }, + ) assert requested.status == 'ok' - certificate = await _approve_csr(app, requested.data['csr_id'], root, - expected_digest=requested.data['request_digest'], operator='isolated-sshd-check') - attached = await call('identity.ssh_certificates', - {'key_id': signer.key_id, 'certificates': [certificate.resource_id]}) + certificate = await _approve_csr( + app, + requested.data['csr_id'], + root, + expected_digest=requested.data['request_digest'], + operator='isolated-sshd-check', + ) + attached = await call( + 'identity.ssh_certificates', + {'key_id': signer.key_id, 'certificates': [certificate.resource_id]}, + ) assert attached.status == 'ok' - host = folder/'host-key' - assert run('ssh-keygen', '-q', '-t', 'ed25519', '-N', '', '-f', str(host)).returncode == 0 - privileged = Path(run('sudo', '-n', 'mktemp', '-d', '/run/msg-sshd-check-XXXXXXXX', check=True).stdout.decode().strip()) - assert privileged.parent == Path('/run') and privileged.name.startswith('msg-sshd-check-') + host = folder / 'host-key' + assert ( + run('ssh-keygen', '-q', '-t', 'ed25519', '-N', '', '-f', str(host)).returncode == 0 + ) + privileged = Path( + run('sudo', '-n', 'mktemp', '-d', '/run/msg-sshd-check-XXXXXXXX', check=True) + .stdout.decode() + .strip() + ) + assert privileged.parent == Path('/run') and privileged.name.startswith( + 'msg-sshd-check-' + ) assert run('sudo', '-n', 'chmod', '755', str(privileged)).returncode == 0 - wrapper = folder/'authorized' - wrapper.write_text('#!/bin/sh\nexec ' + shlex.join([sys.executable, '-m', 'msg.daemon', - '--config-dir', str(folder/'etc'), 'ssh-authorized-key']) + ' "$@"\n') - assert run('sudo', '-n', 'install', '-m', '755', str(wrapper), str(privileged/'authorized')).returncode == 0 + wrapper = folder / 'authorized' + wrapper.write_text( + '#!/bin/sh\nexec ' + + shlex.join([ + sys.executable, + '-m', + 'msg.daemon', + '--config-dir', + str(folder / 'etc'), + 'ssh-authorized-key', + ]) + + ' "$@"\n' + ) + assert ( + run( + 'sudo', + '-n', + 'install', + '-m', + '755', + str(wrapper), + str(privileged / 'authorized'), + ).returncode + == 0 + ) + lookup = run(str(privileged / 'authorized'), *line.split()) + assert lookup.returncode == 0, lookup.stderr.decode() + assert lookup.stdout.startswith(b'restrict,command="') with socket.socket() as probe: probe.bind(('127.0.0.1', 0)) port = probe.getsockname()[1] username = pwd.getpwuid(os.geteuid()).pw_name - pidfile = folder/'sshd.pid' - config = folder/'sshd.conf' - config.write_text(f'''ListenAddress 127.0.0.1 + pidfile = folder / 'sshd.pid' + config = folder / 'sshd.conf' + config.write_text(f"""ListenAddress 127.0.0.1 Port {port} HostKey {host} PidFile {pidfile} @@ -121,7 +221,10 @@ async def call(name, args): KbdInteractiveAuthentication no PubkeyAuthentication yes AuthenticationMethods publickey -UsePAM no +# Match deploy/sshd_config: PAM performs account/session checks even when +# password authentication is disabled (including password-locked CI users). +UsePAM yes +StrictModes yes AuthorizedKeysFile none AuthorizedKeysCommand {privileged}/authorized %t %k AuthorizedKeysCommandUser {username} @@ -129,84 +232,146 @@ async def call(name, args): DisableForwarding yes PermitUserRC no PermitUserEnvironment no -LogLevel ERROR -''') +# Disposable test keys only; preserve enough detail to diagnose auth failures. +LogLevel VERBOSE +""") assert run('sudo', '-n', sshd, '-t', '-f', str(config)).returncode == 0 - server = subprocess.Popen(['sudo', '-n', sshd, '-D', '-e', '-f', str(config)], - stdout=subprocess.DEVNULL, stderr=(folder/'sshd.log').open('wb')) + server = subprocess.Popen( + ['sudo', '-n', sshd, '-D', '-e', '-f', str(config)], + stdout=subprocess.DEVNULL, + stderr=(folder / 'sshd.log').open('wb'), + ) for _ in range(100): if pidfile.exists(): break assert server.poll() is None, 'isolated sshd exited' - await asyncio.sleep(.05) + await asyncio.sleep(0.05) assert pidfile.exists() host_fields = host.with_suffix('.pub').read_text().split()[:2] - known = folder/'known-hosts' + known = folder / 'known-hosts' known.write_text(f'[127.0.0.1]:{port} ' + ' '.join(host_fields) + '\n') - ssh = ['ssh', '-F', '/dev/null', '-p', str(port), '-i', str(client_key), - '-o', 'IdentitiesOnly=yes', '-o', 'BatchMode=yes', '-o', 'StrictHostKeyChecking=yes', - '-o', f'UserKnownHostsFile={known}', '-o', 'ConnectTimeout=5'] + ssh = [ + 'ssh', + '-F', + '/dev/null', + '-p', + str(port), + '-i', + str(client_key), + '-o', + 'IdentitiesOnly=yes', + '-o', + 'BatchMode=yes', + '-o', + 'StrictHostKeyChecking=yes', + '-o', + f'UserKnownHostsFile={known}', + '-o', + 'ConnectTimeout=5', + ] target = f'{username}@127.0.0.1' - read = await asyncio.to_thread(run, *ssh, target, 'msg call discovery.get \'{"id":"/main"}\'') - assert read.returncode == 0, (read.stderr.decode(), (folder/'sshd.log').read_text()) + read = await asyncio.to_thread( + run, *ssh, target, 'msg call discovery.get \'{"id":"/main"}\'' + ) + assert read.returncode == 0, (read.stderr.decode(), (folder / 'sshd.log').read_text()) assert json.loads(read.stdout)['status'] == 'ok' checks.append('real_authorized_keys_and_forced_command') for command in ('id', 'sh', 'git-upload-pack /etc/passwd'): denied = await asyncio.to_thread(run, *ssh, target, command) assert denied.returncode != 0 checks.append('shell_and_path_escape_denied') - forwarding = await asyncio.to_thread(run, *ssh, '-o', 'ExitOnForwardFailure=yes', - '-R', '0:127.0.0.1:1', target, 'msg call discovery.get \'{"id":"/main"}\'') - assert forwarding.returncode != 0 and b'remote port forwarding failed' in forwarding.stderr + forwarding = await asyncio.to_thread( + run, + *ssh, + '-o', + 'ExitOnForwardFailure=yes', + '-R', + '0:127.0.0.1:1', + target, + 'msg call discovery.get \'{"id":"/main"}\'', + ) + assert ( + forwarding.returncode != 0 and b'remote port forwarding failed' in forwarding.stderr + ) checks.append('remote_forwarding_denied') - work = folder/'work'; work.mkdir() + work = folder / 'work' + work.mkdir() + def git(*args): return run('git', '-C', str(work), *args) + assert git('init', '--initial-branch=main').returncode == 0 assert git('config', 'user.name', 'Isolated').returncode == 0 assert git('config', 'user.email', 'isolated@example.invalid').returncode == 0 - (work/'README').write_text('real SSH Git acceptance\n') + (work / 'README').write_text('real SSH Git acceptance\n') assert git('add', '.').returncode == 0 assert git('commit', '-m', 'isolated').returncode == 0 assert git('branch', 'second').returncode == 0 remote = f'{target}:/@ssh-check/code.git' - command = ['git', '-C', str(work), '-c', 'core.sshCommand='+shlex.join(ssh)] - pushed = await asyncio.to_thread(run, *command, 'push', '--atomic', remote, 'main', 'second') + command = ['git', '-C', str(work), '-c', 'core.sshCommand=' + shlex.join(ssh)] + pushed = await asyncio.to_thread( + run, *command, 'push', '--atomic', remote, 'main', 'second' + ) assert pushed.returncode == 0, pushed.stderr.decode() fetched = await asyncio.to_thread(run, *command, 'ls-remote', remote) - assert fetched.returncode == 0 and b'refs/heads/main' in fetched.stdout and b'refs/heads/second' in fetched.stdout - clone = folder/'clone' + assert ( + fetched.returncode == 0 + and b'refs/heads/main' in fetched.stdout + and b'refs/heads/second' in fetched.stdout + ) + clone = folder / 'clone' cloned = await asyncio.to_thread(run, *command, 'clone', remote, str(clone)) - assert cloned.returncode == 0 and (clone/'README').read_bytes() == (work/'README').read_bytes() + assert ( + cloned.returncode == 0 + and (clone / 'README').read_bytes() == (work / 'README').read_bytes() + ) checks.append('real_ssh_atomic_multi_ref_push_and_fetch') - control = folder/'control' - master = subprocess.Popen([*ssh, '-M', '-S', str(control), '-N', target], - stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + control = folder / 'control' + master = subprocess.Popen( + [*ssh, '-M', '-S', str(control), '-N', target], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) for _ in range(200): if control.exists(): break assert master.poll() is None, 'SSH master connection failed' - await asyncio.sleep(.05) + await asyncio.sleep(0.05) assert control.exists() - await _revoke(app, certificate.resource_id, root, reason='isolated test', - operator='isolated-sshd-check') - cert_denied = await asyncio.to_thread(run, *ssh, '-S', str(control), target, - 'msg call discovery.get \'{"id":"/main"}\'') - assert cert_denied.returncode != 0 and b'certificate_revoked' in cert_denied.stdout + cert_denied.stderr - new_denied = await asyncio.to_thread(run, *ssh, target, - 'msg call discovery.get \'{"id":"/main"}\'') + await _revoke( + app, + certificate.resource_id, + root, + reason='isolated test', + operator='isolated-sshd-check', + ) + cert_denied = await asyncio.to_thread( + run, *ssh, '-S', str(control), target, 'msg call discovery.get \'{"id":"/main"}\'' + ) + assert ( + cert_denied.returncode != 0 + and b'certificate_revoked' in cert_denied.stdout + cert_denied.stderr + ) + new_denied = await asyncio.to_thread( + run, *ssh, target, 'msg call discovery.get \'{"id":"/main"}\'' + ) assert new_denied.returncode == 255 checks.append('revoked_certificate_denies_new_and_existing_authenticated_connections') # Explicitly detach the revoked optional certificate; never unrevoke it. - detached = await call('identity.ssh_certificates', {'key_id': signer.key_id, 'certificates': []}) + detached = await call( + 'identity.ssh_certificates', {'key_id': signer.key_id, 'certificates': []} + ) assert detached.status == 'ok' revoked = await call('identity.ssh_key_revoke', {'key_id': signer.key_id}) assert revoked.status == 'ok' - reused = await asyncio.to_thread(run, *ssh, '-S', str(control), target, - 'msg call discovery.get \'{"id":"/main"}\'') + reused = await asyncio.to_thread( + run, *ssh, '-S', str(control), target, 'msg call discovery.get \'{"id":"/main"}\'' + ) assert reused.returncode != 0 and b'credential_revoked' in reused.stdout + reused.stderr checks.append('revocation_rechecked_inside_already_authenticated_connection') - denied = await asyncio.to_thread(run, *ssh, target, 'msg call discovery.get \'{"id":"/main"}\'') + denied = await asyncio.to_thread( + run, *ssh, target, 'msg call discovery.get \'{"id":"/main"}\'' + ) assert denied.returncode == 255 and b'Permission denied' in denied.stderr denied_git = await asyncio.to_thread(run, *command, 'ls-remote', remote) assert denied_git.returncode != 0 @@ -225,7 +390,14 @@ def git(*args): run('sudo', '-n', 'rm', '-rf', '--', str(app.settings.root_private_dir)) if privileged is not None: run('sudo', '-n', 'rm', '-rf', '--', str(privileged)) - return {'checks': checks, 'temporary_installation_removed': not folder.exists()} + return { + 'checks': checks, + 'temporary_installation_removed': not folder.exists(), + 'openssh': run('ssh', '-V').stderr.decode().strip(), + 'python': sys.version, + 'use_pam': True, + 'strict_modes': True, + } if __name__ == '__main__': diff --git a/scripts/check_package_artifacts.py b/scripts/check_package_artifacts.py index 6f3d6f00..8d6ca726 100644 --- a/scripts/check_package_artifacts.py +++ b/scripts/check_package_artifacts.py @@ -1,46 +1,149 @@ -"""Check that the build contains the release-owned rules and installable entry points.""" +"""Reject stale code, changed release resources and mislabeled build artifacts.""" -import sys +import argparse +import configparser +import re import tomllib -from pathlib import Path +from email.parser import BytesParser +from pathlib import Path, PurePosixPath from tarfile import open as open_tar from zipfile import ZipFile -root = Path(__file__).resolve().parents[1] -dist = Path(sys.argv[1]) if len(sys.argv) > 1 else root / "dist" -version = tomllib.loads((root / "pyproject.toml").read_text())["project"]["version"] -wheels = sorted(dist.glob(f"msg_lmm_best-{version}-*.whl")) -sdists = sorted(dist.glob(f"msg_lmm_best-{version}.tar.gz")) -if len(wheels) != 1 or len(sdists) != 1: - raise SystemExit("expected exactly one wheel and one source distribution") - -with ZipFile(wheels[0]) as wheel: - names = set(wheel.namelist()) - for source in (root / "src/msg/data").rglob("*"): - if not source.is_file() or "__pycache__" in source.parts or source.suffix in {".pyc", ".pyo"}: + +def normalized_name(name): + return re.sub(r'[-_.]+', '_', name).lower() + + +def package_sources(root): + package = root / 'src' / 'msg' + result = {} + for source in package.rglob('*'): + if '__pycache__' in source.parts or source.suffix in {'.pyc', '.pyo'}: continue - target = "msg/data/" + source.relative_to(root / "src/msg/data").as_posix() - if target not in names or wheel.read(target) != source.read_bytes(): - raise SystemExit(f"missing or changed wheel resource: {target}") - entry_files = [name for name in names if name.endswith(".dist-info/entry_points.txt")] - if len(entry_files) != 1: - raise SystemExit("missing wheel entry points") - entries = wheel.read(entry_files[0]).decode() - if "msg = msg.cli:main" not in entries or "msgd = msg.daemon:main" not in entries: - raise SystemExit("missing msg or msgd entry point") - -with open_tar(sdists[0]) as sdist: - names = set(sdist.getnames()) - for relative in ( - "src/msg/data/system/AGENTS.md", - "docs/DEPLOYMENT.md", - "deploy/msgd.service", - "tests/test_system_rules.py", - "conformance/test_transports.py", - ): - if not any(name.endswith("/" + relative) for name in names): - raise SystemExit(f"missing sdist file: {relative}") - if any(name.endswith(("/setup.py", "/MANIFEST.in")) for name in names): - raise SystemExit("legacy build files present in sdist") - -print("package artifacts contain the expected code, rules, data, and entry points") + if source.is_symlink(): + raise ValueError(f'package source must not be a symlink: {source}') + if source.is_file(): + result['msg/' + source.relative_to(package).as_posix()] = source.read_bytes() + if 'msg/__init__.py' not in result: + raise ValueError('missing package source') + return result + + +def validate_names(names): + if len(names) != len(set(names)): + raise ValueError('duplicate archive member') + for name in names: + path = PurePosixPath(name) + if path.is_absolute() or '..' in path.parts or '\\' in name: + raise ValueError(f'unsafe archive member: {name}') + + +def check(root, dist): + project = tomllib.loads((root / 'pyproject.toml').read_text())['project'] + version = project['version'] + distribution = normalized_name(project['name']) + prefix = f'{distribution}-{version}' + wheels = sorted(dist.glob(f'{prefix}-*.whl')) + sdists = sorted(dist.glob(f'{prefix}.tar.gz')) + if len(wheels) != 1 or len(sdists) != 1: + raise ValueError('expected exactly one wheel and one source distribution') + expected = package_sources(root) + + with ZipFile(wheels[0]) as wheel: + names = wheel.namelist() + validate_names(names) + packaged = {name for name in names if name.startswith('msg/') and not name.endswith('/')} + if packaged != set(expected): + difference = sorted(packaged ^ set(expected)) + raise ValueError(f'wheel package file set differs from source: {difference}') + for name, content in expected.items(): + if wheel.read(name) != content: + raise ValueError(f'missing or changed wheel source: {name}') + metadata_path = prefix + '.dist-info/METADATA' + metadata_files = [name for name in names if name.endswith('.dist-info/METADATA')] + if metadata_files != [metadata_path]: + raise ValueError('missing or ambiguous wheel metadata') + metadata = BytesParser().parsebytes(wheel.read(metadata_path)) + for field, value in ( + ('Name', project['name']), + ('Version', version), + ('Requires-Python', project['requires-python']), + ): + if metadata.get_all(field) != [value]: + raise ValueError(f'wheel metadata differs from project: {field}') + entry_path = prefix + '.dist-info/entry_points.txt' + entry_files = [name for name in names if name.endswith('.dist-info/entry_points.txt')] + if entry_files != [entry_path]: + raise ValueError('missing or ambiguous wheel entry points') + entries = configparser.ConfigParser(interpolation=None) + entries.read_string(wheel.read(entry_path).decode()) + if not entries.has_section('console_scripts'): + raise ValueError('missing console scripts') + if dict(entries['console_scripts']) != project['scripts']: + raise ValueError('wheel console targets differ from project scripts') + + with open_tar(sdists[0]) as sdist: + names = sdist.getnames() + validate_names(names) + if any(name != prefix and not name.startswith(prefix + '/') for name in names): + raise ValueError('unexpected source distribution root') + if any(name.endswith(('/setup.py', '/MANIFEST.in')) for name in names): + raise ValueError('legacy build files present in sdist') + source_names = { + name.removeprefix(prefix + '/src/') + for name in names + if name.startswith(prefix + '/src/msg/') and sdist.getmember(name).isfile() + } + if source_names != set(expected): + raise ValueError('sdist package file set differs from source') + required = { + **{'src/' + name: data for name, data in expected.items()}, + **{ + name: (root / name).read_bytes() + for name in ( + 'uv.lock', + '.python-version', + 'docs/DEPLOYMENT.md', + 'deploy/msgd.service', + 'tests/test_system_rules.py', + 'conformance/test_transports.py', + ) + }, + } + for relative, content in required.items(): + name = prefix + '/' + relative + if name not in names or not sdist.getmember(name).isfile(): + raise ValueError(f'missing or non-regular sdist file: {relative}') + stream = sdist.extractfile(name) + if stream is None: + raise ValueError(f'unreadable sdist file: {relative}') + with stream: + if stream.read() != content: + raise ValueError(f'changed sdist source: {relative}') + project_path = prefix + '/pyproject.toml' + if project_path not in names or not sdist.getmember(project_path).isfile(): + raise ValueError('missing sdist project metadata') + stream = sdist.extractfile(project_path) + if stream is None: + raise ValueError('unreadable sdist project metadata') + with stream: + packaged_project = tomllib.loads(stream.read().decode())['project'] + for field in ('name', 'version', 'requires-python', 'scripts'): + if packaged_project.get(field) != project[field]: + raise ValueError(f'sdist project metadata differs from source: {field}') + + +def main(): + root = Path(__file__).resolve().parents[1] + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('dist', nargs='?', type=Path, default=root / 'dist') + args = parser.parse_args() + try: + check(root, args.dist) + except ValueError as exc: + parser.exit(1, str(exc) + '\n') + print('package artifacts match current code, rules, metadata, lockfile, and entry points') + + +if __name__ == '__main__': + main() diff --git a/scripts/check_tool_sandbox.py b/scripts/check_tool_sandbox.py index 4ee1e9d6..840345d0 100644 --- a/scripts/check_tool_sandbox.py +++ b/scripts/check_tool_sandbox.py @@ -3,6 +3,7 @@ Only this operator-invoked probe captures subprocess stderr. The packet contains no user input or credentials, and production errors keep their stable safe code. """ + from __future__ import annotations import asyncio @@ -22,14 +23,30 @@ async def check(): print(f'Python prefix: {sys.prefix}; base prefix: {sys.base_prefix}', flush=True) setting = Path('/proc/sys/kernel/apparmor_restrict_unprivileged_userns') if setting.exists(): - print(f'AppArmor unprivileged userns restriction: {setting.read_text().strip()}', flush=True) - policy = NetworkPolicy(schemes=frozenset({'https'}), hosts=(), ports=frozenset({443}), - methods=frozenset({'GET'}), allow_private=False, - timeout_ms=1000, max_response_bytes=1024, max_redirects=0) - app = SimpleNamespace(settings=SimpleNamespace(server=SimpleNamespace( - limits=SimpleNamespace(max_request_bytes=1024)))) + print( + f'AppArmor unprivileged userns restriction: {setting.read_text().strip()}', flush=True + ) + policy = NetworkPolicy( + schemes=frozenset({'https'}), + hosts=(), + ports=frozenset({443}), + methods=frozenset({'GET'}), + allow_private=False, + timeout_ms=1000, + max_response_bytes=1024, + max_redirects=0, + ) + app = SimpleNamespace( + settings=SimpleNamespace( + server=SimpleNamespace(limits=SimpleNamespace(max_request_bytes=1024)) + ) + ) spawn = asyncio.create_subprocess_exec - with TemporaryDirectory(prefix='msg-sandbox-preflight-') as directory, TemporaryFile() as diagnostics: + with ( + TemporaryDirectory(prefix='msg-sandbox-preflight-') as directory, + TemporaryFile() as diagnostics, + ): + async def capture(*args, **kwargs): kwargs['stderr'] = diagnostics return await spawn(*args, **kwargs) @@ -37,8 +54,12 @@ async def capture(*args, **kwargs): try: with patch('asyncio.create_subprocess_exec', capture): try: - await BubblewrapRunner(app)(SimpleNamespace(executor_key='curl'), - {'url': 'file:///not-a-network-target'}, (policy,), Path(directory)) + await BubblewrapRunner(app)( + SimpleNamespace(executor_key='curl'), + {'url': 'file:///not-a-network-target'}, + (policy,), + Path(directory), + ) except Failure as exc: if exc.code != 'network_policy_denied': raise diff --git a/scripts/ci_shards.py b/scripts/ci_shards.py index 8b45b623..1d010f15 100644 --- a/scripts/ci_shards.py +++ b/scripts/ci_shards.py @@ -3,6 +3,7 @@ Adapted from PR #104. Every runner collects the complete suite; the gate rejects skips, duplicate IDs, count-only substitutions and missing or failed evidence. """ + from __future__ import annotations import argparse @@ -18,14 +19,22 @@ def partition(nodeids, index, count): raise ValueError('invalid shard coordinates') if len(nodeids) != len(set(nodeids)): raise ValueError('duplicate test node ID') - return [node for node in nodeids - if int.from_bytes(hashlib.sha256(node.encode()).digest()[:8], 'big') % count == index] + return [ + node + for node in nodeids + if int.from_bytes(hashlib.sha256(node.encode()).digest()[:8], 'big') % count == index + ] def evidence(nodeids, index, count): all_nodes = sorted(nodeids) - return {'version': 2, 'index': index, 'count': count, 'all': all_nodes, - 'selected': partition(all_nodes, index, count)} + return { + 'version': 2, + 'index': index, + 'count': count, + 'all': all_nodes, + 'selected': partition(all_nodes, index, count), + } def report_nodes(path): @@ -34,8 +43,11 @@ def report_nodes(path): raise ValueError('JUnit result contains failed or skipped tests') nodes = [] for case in root.findall('.//testcase'): - ids = [p.get('value') for p in case.findall('./properties/property') - if p.get('name') == 'msg.nodeid'] + ids = [ + p.get('value') + for p in case.findall('./properties/property') + if p.get('name') == 'msg.nodeid' + ] if len(ids) != 1 or not ids[0]: raise ValueError('JUnit result is missing an unambiguous test node ID') nodes.append(ids[0]) @@ -85,10 +97,12 @@ def pytest_collection_modifyitems(self, config, items): item.user_properties.append(('msg.nodeid', item.nodeid)) config.hook.pytest_deselected(items=rejected) (directory / f'shard-{index}.json').write_text( - json.dumps(manifest, ensure_ascii=False, indent=2) + '\n') + json.dumps(manifest, ensure_ascii=False, indent=2) + '\n' + ) - return pytest.main(['tests', f'--junitxml={directory / f"tests-{index}.xml"}'], - plugins=[Collector()]) + return pytest.main( + ['tests', f'--junitxml={directory / f"tests-{index}.xml"}'], plugins=[Collector()] + ) def main(): diff --git a/scripts/verify_local.py b/scripts/verify_local.py new file mode 100644 index 00000000..8ad759bc --- /dev/null +++ b/scripts/verify_local.py @@ -0,0 +1,350 @@ +"""Verify an uncommitted checkout without GitHub writes or production deployment. + +Use the locked Python 3.15 development environment. Every stage records its +actual result; missing tools, incomplete tests and source drift fail closed. +""" + +from __future__ import annotations + +import argparse +import hashlib +import importlib.metadata +import json +import os +import platform +import shutil +import subprocess +import sys +import tempfile +import time +import tomllib +from datetime import UTC, datetime +from pathlib import Path +from urllib.parse import urlsplit + +PINNED_TOOLS = {'ruff': '0.16.9', 'uv': '0.12.20'} +REQUIRED_PACKAGES = ( + 'pytest', + 'pytest-asyncio', + 'build', + 'cryptography', + 'httpx', + 'jsonschema', + 'referencing', + 'starlette', + 'uvicorn', + 'aiohttp', + 'dnspython', + 'graphql-core', + 'psycopg', + 'valkey', + 'tiktoken', +) +REQUIRED_COMMANDS = ('git', 'git-lfs', 'age', 'openssl', 'nginx', 'bwrap', 'pg_dump', 'pg_restore') + + +def content_fingerprint(root, names): + """Hash the actual files, not the last commit; never follow source symlinks.""" + digest = hashlib.sha256() + records = [] + for name in sorted(set(names)): + path = root / name + if Path(name).is_absolute() or '..' in Path(name).parts: + raise ValueError('invalid source path') + if path.is_symlink(): + kind = 'symlink' + data = os.fsencode(os.readlink(path)) + elif not path.exists(): + continue + elif path.is_file(): + kind = 'executable' if path.stat().st_mode & 0o111 else 'file' + data = path.read_bytes() + else: + raise ValueError(f'source entry is not a file: {name}') + item = {'path': name, 'kind': kind, 'sha256': hashlib.sha256(data).hexdigest()} + digest.update(json.dumps(item, sort_keys=True, ensure_ascii=True).encode() + b'\n') + records.append(item) + if not records: + raise ValueError('empty source inventory') + return {'sha256': digest.hexdigest(), 'files': records} + + +def source_snapshot(root): + def git(*args): + return subprocess.check_output(['git', *args], cwd=root) + + tracked = git('ls-files', '-z', '--cached', '--others', '--exclude-standard') + names = [os.fsdecode(name) for name in tracked.split(b'\0') if name] + result = content_fingerprint(root, names) + result['base_commit'] = git('rev-parse', 'HEAD').decode().strip() + result['dirty'] = bool(git('status', '--porcelain', '--untracked-files=normal').strip()) + return result + + +def assert_same_source(before, after): + if before['sha256'] != after['sha256']: + raise ValueError('source changed during verification; results belong to different code') + + +def preflight(root): + blockers = [] + tools = {'python': platform.python_version()} + if sys.version_info[:2] != (3, 15): + blockers.append('Python 3.15 is required; do not substitute an older interpreter') + project = tomllib.loads((root / 'pyproject.toml').read_text()) + if project['project']['requires-python'] != '>=3.15': + blockers.append('project requires-python is not >=3.15') + if project['build-system']['requires'] != ['uv_build==0.12.20']: + blockers.append('uv_build is not pinned to 0.12.20') + for name in ('ruff', *REQUIRED_PACKAGES): + try: + version = importlib.metadata.version(name) + except importlib.metadata.PackageNotFoundError: + blockers.append(f'missing Python package: {name}') + continue + tools[name] = version + if name in PINNED_TOOLS and version != PINNED_TOOLS[name]: + blockers.append(f'{name} must be {PINNED_TOOLS[name]}; found {version}') + for name in REQUIRED_COMMANDS: + executable = shutil.which(name) + if executable is None: + blockers.append(f'missing executable: {name}') + else: + tools[name] = executable + if shutil.which('uv') is None: + blockers.append('missing uv executable') + else: + try: + version = subprocess.check_output(['uv', '--version'], text=True, timeout=10).strip() + except (OSError, subprocess.SubprocessError) as exc: + blockers.append('uv version check failed: ' + type(exc).__name__) + else: + tools['uv'] = version + if version.split()[:2] != ['uv', PINNED_TOOLS['uv']]: + blockers.append(f'uv must be {PINNED_TOOLS["uv"]}; found {version}') + for name in ('MSG_TEST_POSTGRES_URL_TEMPLATE', 'MSG_TEST_VALKEY_URL'): + configured = os.environ.get(name) + if configured: + parsed = urlsplit(configured) + if parsed.hostname not in {'localhost', '127.0.0.1', '::1'}: + blockers.append(f'{name} must point to a disposable local test service') + if not os.environ.get('MSG_TEST_POSTGRES_URL_TEMPLATE'): + for name in ('initdb', 'pg_ctl'): + if shutil.which(name) is None: + blockers.append(f'missing local PostgreSQL executable: {name}') + if hasattr(os, 'geteuid') and os.geteuid() == 0: + blockers.append( + 'initdb requires a non-root user, or configure disposable local PostgreSQL' + ) + if not os.environ.get('MSG_TEST_VALKEY_URL'): + blockers.append('MSG_TEST_VALKEY_URL must identify disposable local Valkey') + legacy = os.environ.get('MSG_TEST_LEGACY_SOURCE', str(root / '.legacy-ledger')) + if not (Path(legacy) / 'src').is_dir(): + blockers.append('missing pinned legacy source; set MSG_TEST_LEGACY_SOURCE') + else: + manifest = json.loads((root / 'tests/fixtures/legacy_market_source.json').read_text()) + for name, expected in manifest['sha256'].items(): + path = Path(legacy) / name + if not path.is_file() or hashlib.sha256(path.read_bytes()).hexdigest() != expected: + blockers.append('pinned legacy source mismatch: ' + name) + return {'tools': tools, 'blockers': blockers} + + +def run_step(name, command, directory, *, cwd, env=None, clean_stderr=False, timeout=1200): + started = time.monotonic() + stdout = directory / (name + '.stdout') + stderr = directory / (name + '.stderr') + result = {'name': name, 'command': command, 'status': 'failed'} + try: + with stdout.open('wb') as output, stderr.open('wb') as errors: + completed = subprocess.run( + command, + cwd=cwd, + env=env, + stdout=output, + stderr=errors, + check=False, + timeout=timeout, + ) + result['returncode'] = completed.returncode + result['stderr_bytes'] = stderr.stat().st_size + if completed.returncode == 0 and (not clean_stderr or not result['stderr_bytes']): + result['status'] = 'passed' + except OSError as exc: + result['error'] = type(exc).__name__ + except subprocess.TimeoutExpired: + result['error'] = 'timeout' + result['duration_seconds'] = round(time.monotonic() - started, 3) + return result + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--shards', type=int, default=8) + parser.add_argument('--directory', type=Path) + parser.add_argument('--preflight-only', action='store_true') + args = parser.parse_args(argv) + if not 1 <= args.shards <= 64: + parser.error('--shards must be between 1 and 64') + root = Path(__file__).resolve().parents[1] + stamp = datetime.now(UTC).strftime('%Y%m%dT%H%M%S.%fZ') + directory = (args.directory or root / 'artifacts' / ('local-' + stamp)).resolve() + directory.mkdir(parents=True, exist_ok=False) + report = { + 'status': 'blocked', + 'stages': [], + 'scope': 'local source verification; not deployment', + } + report_path = directory / 'result.json' + + def save(): + report_path.write_text(json.dumps(report, indent=2) + '\n') + + def execute(name, command, *, cwd=root, env=None, clean_stderr=False): + stage = run_step(name, command, directory, cwd=cwd, env=env, clean_stderr=clean_stderr) + report['stages'].append(stage) + save() + if stage['status'] != 'passed': + raise ValueError(f'{name} failed; see {directory}') + + try: + report['source_before'] = source_snapshot(root) + report['preflight'] = preflight(root) + save() + if report['preflight']['blockers']: + print('\n'.join(report['preflight']['blockers'])) + return 1 + if args.preflight_only: + report['status'] = 'preflight_passed' + save() + return 0 + python = sys.executable + execute('lock', ['uv', 'lock', '--check', '--python', python]) + execute('dependencies', ['uv', 'pip', 'check', '--python', python]) + execute( + 'ruff', + [python, '-m', 'ruff', 'check', '.', '--output-format', 'json'], + clean_stderr=True, + ) + execute('format', [python, '-m', 'ruff', 'format', '--check', '.'], clean_stderr=True) + execute( + 'compile', + [ + python, + '-W', + 'error', + '-m', + 'compileall', + '-q', + 'src', + 'tests', + 'scripts', + 'conformance', + 'conftest.py', + ], + clean_stderr=True, + ) + execute('tool-sandbox', [python, 'scripts/check_tool_sandbox.py']) + # Sequential shards keep one local Valkey service from cross-shard races. + for index in range(args.shards): + execute( + f'shard-{index}', + [ + python, + 'scripts/ci_shards.py', + 'run', + str(index), + str(args.shards), + '--directory', + str(directory), + ], + ) + execute( + 'test-union', + [ + python, + 'scripts/ci_shards.py', + 'check', + str(args.shards), + '--directory', + str(directory), + ], + ) + environment = {**os.environ, 'MSG_BENCHMARK_PATH': str(directory / 'token-budget.json')} + execute( + 'conformance', + [python, '-m', 'pytest', 'conformance', f'--junitxml={directory / "conformance.xml"}'], + env=environment, + ) + # Pytest exits zero for skips too; enforce the same no-skips acceptance here. + import xml.etree.ElementTree as ET + + conformance = ET.parse(directory / 'conformance.xml').getroot() + if not conformance.findall('.//testcase') or any( + conformance.findall('.//' + tag) for tag in ('failure', 'error', 'skipped') + ): + raise ValueError('conformance evidence is incomplete, failed or skipped') + dist = directory / 'dist' + execute('build', [python, '-m', 'build', '--outdir', str(dist)]) + execute('artifacts', [python, 'scripts/check_package_artifacts.py', str(dist)]) + requirements = directory / 'client-requirements.txt' + execute( + 'client-lock', + [ + 'uv', + 'export', + '--locked', + '--no-dev', + '--no-emit-project', + '--format', + 'requirements-txt', + '--output-file', + str(requirements), + ], + ) + with tempfile.TemporaryDirectory(prefix='msg-verify-client-') as temporary: + folder = Path(temporary) + venv = folder / 'venv' + execute('client-env', ['uv', 'venv', '--python', python, str(venv)]) + client_python = str(venv / ('Scripts/python.exe' if os.name == 'nt' else 'bin/python')) + wheels = list(dist.glob('*.whl')) + if len(wheels) != 1: + raise ValueError('ambiguous client wheel') + execute( + 'client-install', + [ + 'uv', + 'pip', + 'install', + '--python', + client_python, + '--constraint', + str(requirements), + str(wheels[0]), + ], + ) + execute('client-dependencies', ['uv', 'pip', 'check', '--python', client_python]) + environment = dict(os.environ) + environment.pop('PYTHONPATH', None) + execute( + 'client-transports', + [client_python, str(root / 'scripts/check_client_install.py'), '--minimal-install'], + cwd=folder, + env=environment, + ) + report['source_after'] = source_snapshot(root) + assert_same_source(report['source_before'], report['source_after']) + report['status'] = 'passed' + save() + print(f'Local verification passed: {report_path}') + return 0 + except (OSError, ValueError, subprocess.SubprocessError) as exc: + report['status'] = 'failed' + report['error'] = str(exc) + save() + print(str(exc), file=sys.stderr) + return 1 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/src/msg/__init__.py b/src/msg/__init__.py index d72073ba..89b5deec 100644 --- a/src/msg/__init__.py +++ b/src/msg/__init__.py @@ -1,2 +1,3 @@ """Atomic communication for sandboxed agents.""" -__version__ = "1.0.0a1" + +__version__ = '0.1.0a1' diff --git a/src/msg/admin/backup_retirement.py b/src/msg/admin/backup_retirement.py index f91e764c..5724e1db 100644 --- a/src/msg/admin/backup_retirement.py +++ b/src/msg/admin/backup_retirement.py @@ -5,22 +5,24 @@ databases. The record is verified again on every read, so this import is not a trusted cache of a success flag. """ + from __future__ import annotations -from msg.constants import ROOT_SUBJECT import os -from pathlib import Path import tempfile +from pathlib import Path +from msg.constants import ROOT_SUBJECT from msg.core.codec import canonical, digest, loads from msg.core.errors import Failure, require from msg.core.models import AuditEvent, Event, ResourceRef from msg.plugins.common import new_id -from msg.security.backup_retirement import FIELDS, DOMAIN, check, root_verifier, setting_key +from msg.security.backup_retirement import DOMAIN, FIELDS, check, root_verifier, setting_key def read_statement(path): from msg.security.root_files import read_private + return loads(read_private(path, limit=1024 * 1024)) @@ -49,41 +51,70 @@ def write_record(path, record): def unsigned_statement(value): - require(isinstance(value, dict) and set(value) == FIELDS and value.get('domain') == DOMAIN, - 'backup_retirement_invalid') + require( + isinstance(value, dict) and set(value) == FIELDS and value.get('domain') == DOMAIN, + 'backup_retirement_invalid', + ) return value def completed_migration(tx, subject_id, old_identity_key_id): """The one completed migration of this subject that retired exactly this old key.""" - matches = [loads(body) for (body,) in tx.rows( - "SELECT body FROM custodial_upgrades WHERE subject=? AND status='completed' ORDER BY id", - (subject_id,)) if loads(body).get('old_identity_key_id') == old_identity_key_id] + matches = [ + loads(body) + for (body,) in tx.rows( + "SELECT body FROM custodial_upgrades WHERE subject=? AND status='completed' ORDER BY id", + (subject_id,), + ) + if loads(body).get('old_identity_key_id') == old_identity_key_id + ] require(len(matches) == 1, 'backup_retirement_migration_not_found') return matches[0] async def import_record(tx, record, *, now, operator): body = record.get('statement') if isinstance(record, dict) else None - require(isinstance(body, dict) and isinstance(body.get('subject_id'), str) and - isinstance(body.get('old_identity_key_id'), str), 'backup_retirement_invalid') + require( + isinstance(body, dict) + and isinstance(body.get('subject_id'), str) + and isinstance(body.get('old_identity_key_id'), str), + 'backup_retirement_invalid', + ) subject = body['subject_id'] details = completed_migration(tx, subject, body['old_identity_key_id']) attestation = check(record, await root_verifier(tx, now=now), subject, details, now=now) key = setting_key(subject, attestation.old_identity_key_id) previous = tx.setting(key) tx.set_setting(key, record) - data = {'subject_id': subject, 'old_identity_key_id': attestation.old_identity_key_id, - 'old_encryption_key_id': attestation.old_encryption_key_id, - 'backup_sets': list(attestation.backup_sets), 'record_digest': attestation.record_digest, - 'signer_key_id': attestation.signer_key_id, 'operator': operator, - 'scope': 'listed_backup_sets_only'} - event = Event(id=new_id('audit'), type='root.custodial.backup_retirement', time=now, - request_id=new_id('local'), actor=ROOT_SUBJECT, subject=subject, - resources=(ResourceRef(id=subject),), data=data) - await tx.append_audit(AuditEvent(event=event, - authority=(ResourceRef(id=tx.setting('active_root_certificate')),), - before_digest=digest(previous) if previous is not None else None, - after_digest=attestation.record_digest, previous_digest=None, entry_digest='', - result='imported')) + data = { + 'subject_id': subject, + 'old_identity_key_id': attestation.old_identity_key_id, + 'old_encryption_key_id': attestation.old_encryption_key_id, + 'backup_sets': list(attestation.backup_sets), + 'record_digest': attestation.record_digest, + 'signer_key_id': attestation.signer_key_id, + 'operator': operator, + 'scope': 'listed_backup_sets_only', + } + event = Event( + id=new_id('audit'), + type='root.custodial.backup_retirement', + time=now, + request_id=new_id('local'), + actor=ROOT_SUBJECT, + subject=subject, + resources=(ResourceRef(id=subject),), + data=data, + ) + await tx.append_audit( + AuditEvent( + event=event, + authority=(ResourceRef(id=tx.setting('active_root_certificate')),), + before_digest=digest(previous) if previous is not None else None, + after_digest=attestation.record_digest, + previous_digest=None, + entry_digest='', + result='imported', + ) + ) return attestation diff --git a/src/msg/admin/backups.py b/src/msg/admin/backups.py index c3437724..dc09abdf 100644 --- a/src/msg/admin/backups.py +++ b/src/msg/admin/backups.py @@ -1,20 +1,24 @@ """Explicit local data backup; encrypted root material is a separate operation.""" + from __future__ import annotations + import asyncio import hashlib import os -from pathlib import Path import shutil +import subprocess import tempfile import tomllib import zipfile +from pathlib import Path + import psycopg -from psycopg.conninfo import conninfo_to_dict,make_conninfo -import subprocess -from msg.core.codec import canonical,loads -from msg.core.errors import Failure,require -from msg.config import load_settings,write_example +from psycopg.conninfo import conninfo_to_dict, make_conninfo + from msg.admin.restore_database import restore_dump +from msg.config import load_settings, write_example +from msg.core.codec import canonical, loads +from msg.core.errors import require from msg.storage.git import durable_write _FORMAT = 'msg-data-backup-v4' @@ -26,320 +30,438 @@ def _regular_tree(path): require(path.is_dir() and not path.is_symlink(), 'backup_source_missing') for root, directories, files in os.walk(path, followlinks=False): for name in (*directories, *files): - entry=Path(root)/name + entry = Path(root) / name require(not entry.is_symlink(), 'backup_symlink_forbidden') require(entry.is_dir() or entry.is_file(), 'backup_special_file_forbidden') def _db_refs(connection): """Revision and transfer chunk SQL pointers into Git/CAS.""" - contents={} - revisions=[] + contents = {} + revisions = [] for revision_id, raw in connection.execute('SELECT id,body FROM revisions ORDER BY id'): - body=loads(raw) - content=body['content'] + body = loads(raw) + content = body['content'] require(content['digest'].startswith('sha256:'), 'backup_reference_invalid') - contents[content['digest'][7:]]=content['size'] + contents[content['digest'][7:]] = content['size'] revisions.append(revision_id) for (raw,) in connection.execute('SELECT body FROM chunks ORDER BY transfer_id,"offset"'): - content=loads(raw)['content'] + content = loads(raw)['content'] require(content['digest'].startswith('sha256:'), 'backup_reference_invalid') - contents[content['digest'][7:]]=content['size'] + contents[content['digest'][7:]] = content['size'] from msg.market.references import content_references + for content in content_references(connection.execute): require(content['digest'].startswith('sha256:'), 'backup_reference_invalid') - contents[content['digest'][7:]]=content['size'] - live={row[0] for row in connection.execute('SELECT revision FROM resources WHERE revision IS NOT NULL')} + contents[content['digest'][7:]] = content['size'] + live = { + row[0] + for row in connection.execute('SELECT revision FROM resources WHERE revision IS NOT NULL') + } require(live.issubset(set(revisions)), 'backup_revision_missing') - return {'contents':contents, 'revisions':revisions} + return {'contents': contents, 'revisions': revisions} def _verify_storage(directory, refs, *, settings=None, repair_git_layout=True): """Verify archived bytes, Git objects and the SQL-referenced content index.""" - content=(settings.server.content_dir if settings is not None else directory/'content') - blobs=(settings.server.blob_dir if settings is not None else directory/'blobs') - repos=(settings.server.repositories_dir if settings is not None else directory/'repositories') - require(content.is_dir() and (content/'private.git').is_dir() and - (content/'index').is_dir() and blobs.is_dir(), - 'backup_storage_missing') - index_files={path.name:path for path in (content/'index').iterdir() if path.is_file()} + content = settings.server.content_dir if settings is not None else directory / 'content' + blobs = settings.server.blob_dir if settings is not None else directory / 'blobs' + repos = settings.server.repositories_dir if settings is not None else directory / 'repositories' + require( + content.is_dir() + and (content / 'private.git').is_dir() + and (content / 'index').is_dir() + and blobs.is_dir(), + 'backup_storage_missing', + ) + index_files = {path.name: path for path in (content / 'index').iterdir() if path.is_file()} require(set(refs['contents']).issubset(index_files), 'backup_content_missing') - for key,index in index_files.items(): - entry=loads(index.read_bytes()) - size=refs['contents'].get(key,entry.get('size')) - require(len(key)==64 and all(c in '0123456789abcdef' for c in key) and - type(size) is int and size>=0, 'backup_reference_invalid') - require(entry.get('size')==size, 'backup_content_size_mismatch') - if entry.get('kind')=='binary': - target=blobs/key - require(target.is_file() and target.stat().st_size==size and - _hash(target)==key, 'backup_content_digest_mismatch') + for key, index in index_files.items(): + entry = loads(index.read_bytes()) + size = refs['contents'].get(key, entry.get('size')) + require( + len(key) == 64 + and all(c in '0123456789abcdef' for c in key) + and type(size) is int + and size >= 0, + 'backup_reference_invalid', + ) + require(entry.get('size') == size, 'backup_content_size_mismatch') + if entry.get('kind') == 'binary': + target = blobs / key + require( + target.is_file() and target.stat().st_size == size and _hash(target) == key, + 'backup_content_digest_mismatch', + ) else: - require(entry.get('kind')=='git', 'backup_content_kind_invalid') - oid=entry.get('oid') - require(type(oid) is str and len(oid) in (40,64) and - all(c in '0123456789abcdef' for c in oid), 'backup_git_oid_invalid') - status,actual_size,actual_hash=_git_blob_digest(content/'private.git',oid) - require(status==0 and actual_size==size and actual_hash==key, - 'backup_git_content_mismatch') + require(entry.get('kind') == 'git', 'backup_content_kind_invalid') + oid = entry.get('oid') + require( + type(oid) is str + and len(oid) in (40, 64) + and all(c in '0123456789abcdef' for c in oid), + 'backup_git_oid_invalid', + ) + status, actual_size, actual_hash = _git_blob_digest(content / 'private.git', oid) + require( + status == 0 and actual_size == size and actual_hash == key, + 'backup_git_content_mismatch', + ) for revision in refs['revisions']: - require(type(revision) is str and '/' not in revision and revision not in ('.','..'), - 'backup_revision_invalid') - pointer=content/'revisions'/revision + require( + type(revision) is str and '/' not in revision and revision not in ('.', '..'), + 'backup_revision_invalid', + ) + pointer = content / 'revisions' / revision require(pointer.is_file(), 'backup_revision_content_missing') - oid=pointer.read_text().strip() - require(len(oid) in (40,64) and all(c in '0123456789abcdef' for c in oid), - 'backup_revision_oid_invalid') - process=subprocess.run(['git','--git-dir',str(content/'private.git'), - 'cat-file','-e',oid+'^{commit}'],capture_output=True,check=False) - require(process.returncode==0, 'backup_revision_content_missing') - for repository in (repos.iterdir() if repos.exists() else ()): - require(repository.is_dir() and repository.name.endswith('.git'), - 'backup_repository_invalid') + oid = pointer.read_text().strip() + require( + len(oid) in (40, 64) and all(c in '0123456789abcdef' for c in oid), + 'backup_revision_oid_invalid', + ) + process = subprocess.run( + ['git', '--git-dir', str(content / 'private.git'), 'cat-file', '-e', oid + '^{commit}'], + capture_output=True, + check=False, + ) + require(process.returncode == 0, 'backup_revision_content_missing') + for repository in repos.iterdir() if repos.exists() else (): + require( + repository.is_dir() and repository.name.endswith('.git'), 'backup_repository_invalid' + ) # ZIP stores files only. An empty bare repository still needs these # structural directories; recreating them changes no Git object/ref. - for name in ('objects/info','objects/pack','refs/heads','refs/tags'): + for name in ('objects/info', 'objects/pack', 'refs/heads', 'refs/tags'): if repair_git_layout: - (repository/name).mkdir(parents=True,exist_ok=True) + (repository / name).mkdir(parents=True, exist_ok=True) else: - require((repository/name).is_dir(), 'backup_git_layout_missing') - process=subprocess.run(['git','--git-dir',str(repository),'fsck','--full','--no-reflogs'], - capture_output=True,check=False) - require(process.returncode==0, 'backup_git_repository_invalid') - objects=repository/'lfs'/'objects' + require((repository / name).is_dir(), 'backup_git_layout_missing') + process = subprocess.run( + ['git', '--git-dir', str(repository), 'fsck', '--full', '--no-reflogs'], + capture_output=True, + check=False, + ) + require(process.returncode == 0, 'backup_git_repository_invalid') + objects = repository / 'lfs' / 'objects' if not objects.exists(): continue for path in objects.rglob('*'): if not path.is_file(): continue - oid=path.name - require(len(oid)==64 and all(c in '0123456789abcdef' for c in oid) and - path.relative_to(objects).parts==(oid[:2],oid[2:4],oid), - 'backup_lfs_path_invalid') - canonical_blob=blobs/oid - require(canonical_blob.is_file() and _hash(path)==oid and - path.stat().st_size==canonical_blob.stat().st_size and - _hash(canonical_blob)==oid, 'backup_lfs_digest_mismatch') + oid = path.name + require( + len(oid) == 64 + and all(c in '0123456789abcdef' for c in oid) + and path.relative_to(objects).parts == (oid[:2], oid[2:4], oid), + 'backup_lfs_path_invalid', + ) + canonical_blob = blobs / oid + require( + canonical_blob.is_file() + and _hash(path) == oid + and path.stat().st_size == canonical_blob.stat().st_size + and _hash(canonical_blob) == oid, + 'backup_lfs_digest_mismatch', + ) def _relink_lfs(data_dir, settings): """ZIP cannot retain hardlinks; restore canonical shared-CAS link counts.""" - repositories=settings.server.repositories_dir + repositories = settings.server.repositories_dir if not repositories.exists(): return for repository in repositories.iterdir(): - objects=repository/'lfs'/'objects' + objects = repository / 'lfs' / 'objects' if not objects.exists(): continue for path in objects.rglob('*'): if not path.is_file(): continue - shared=settings.server.blob_dir/path.name - require(path.stat().st_dev==shared.stat().st_dev, - 'restore_lfs_shared_volume_required') - temporary=path.with_name('.relink-'+path.name) + shared = settings.server.blob_dir / path.name + require( + path.stat().st_dev == shared.stat().st_dev, 'restore_lfs_shared_volume_required' + ) + temporary = path.with_name('.relink-' + path.name) try: - os.link(shared,temporary) - os.replace(temporary,path) + os.link(shared, temporary) + os.replace(temporary, path) finally: temporary.unlink(missing_ok=True) def _write_restored_config(source, settings, postgres_dsn): """Retain policy, but bind only to the new isolated storage and loopback.""" - raw=tomllib.loads(source.read_text()) - server={'service_url':settings.service_url,'listen':'127.0.0.1','port':8042} - for name in ('public_web_origin','temporary_ttl','transfer_ttl'): - if name in raw.get('server',{}): - server[name]=raw['server'][name] - storage={'postgres_dsn':postgres_dsn, - 'content':str(settings.server.content_dir), - 'repositories':str(settings.server.repositories_dir), - 'blobs':str(settings.server.blob_dir), - 'staging':str(settings.server.staging_dir), - 'service_keys':str(settings.service_keys)} - sections={'server':server,'storage':storage} - for name in ('identity','money','limits','plugins','tools'): + raw = tomllib.loads(source.read_text()) + server = {'service_url': settings.service_url, 'listen': '127.0.0.1', 'port': 8042} + for name in ('public_web_origin', 'temporary_ttl', 'transfer_ttl'): + if name in raw.get('server', {}): + server[name] = raw['server'][name] + storage = { + 'postgres_dsn': postgres_dsn, + 'content': str(settings.server.content_dir), + 'repositories': str(settings.server.repositories_dir), + 'blobs': str(settings.server.blob_dir), + 'staging': str(settings.server.staging_dir), + 'service_keys': str(settings.service_keys), + } + sections = {'server': server, 'storage': storage} + for name in ('identity', 'money', 'limits', 'plugins', 'tools'): if name in raw: - sections[name]=raw[name] - lines=[] - for section,values in sections.items(): + sections[name] = raw[name] + lines = [] + for section, values in sections.items(): lines.append(f'[{section}]') - for name,value in values.items(): + for name, value in values.items(): lines.append(f'{name} = {canonical(value).decode()}') lines.append('') - recovery=raw.get('recovery',{}) + recovery = raw.get('recovery', {}) if recovery: lines.append('[recovery]') - for name,value in recovery.items(): - if name!='custodians': + for name, value in recovery.items(): + if name != 'custodians': lines.append(f'{name} = {canonical(value).decode()}') lines.append('') - for custodian in recovery.get('custodians',()): + for custodian in recovery.get('custodians', ()): lines.append('[[recovery.custodians]]') - for name,value in custodian.items(): + for name, value in custodian.items(): lines.append(f'{name} = {canonical(value).decode()}') lines.append('') - path=settings.config_dir/'msgd.toml' + path = settings.config_dir / 'msgd.toml' path.write_text('\n'.join(lines)) - os.chmod(path,0o600) + os.chmod(path, 0o600) def _hash(path): - value=hashlib.sha256() + value = hashlib.sha256() with path.open('rb') as source: - while chunk:=source.read(65536):value.update(chunk) + while chunk := source.read(65536): + value.update(chunk) return value.hexdigest() def _git_blob_digest(repository, oid): - value=hashlib.sha256() - size=0 - with subprocess.Popen(['git','--git-dir',str(repository),'cat-file','blob',oid], - stdout=subprocess.PIPE,stderr=subprocess.DEVNULL) as process: - while chunk:=process.stdout.read(1024*1024): - value.update(chunk);size+=len(chunk) - status=process.wait() - return status,size,value.hexdigest() + value = hashlib.sha256() + size = 0 + with subprocess.Popen( + ['git', '--git-dir', str(repository), 'cat-file', 'blob', oid], + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, + ) as process: + while chunk := process.stdout.read(1024 * 1024): + value.update(chunk) + size += len(chunk) + status = process.wait() + return status, size, value.hexdigest() -async def backup(app,destination): - destination=Path(destination) - require(not destination.exists() and not destination.is_symlink(),'backup_destination_exists') - destination.parent.mkdir(parents=True,exist_ok=True) - with tempfile.TemporaryDirectory(prefix='msg-backup-',dir=destination.parent) as tmp: - directory=Path(tmp) - async with app.metadata.transaction(write=True) as tx: +async def backup(app, destination): + destination = Path(destination) + require(not destination.exists() and not destination.is_symlink(), 'backup_destination_exists') + destination.parent.mkdir(parents=True, exist_ok=True) + with tempfile.TemporaryDirectory(prefix='msg-backup-', dir=destination.parent) as tmp: + directory = Path(tmp) + async with app.metadata.transaction(write=True): # PostgreSQL's transaction-scoped deployment advisory lock excludes # other msgd writers while pg_dump uses its own MVCC connection. # Storage objects are published before their SQL pointers commit. - await asyncio.to_thread(app.metadata.backup,directory/'metadata.dump') + await asyncio.to_thread(app.metadata.backup, directory / 'metadata.dump') with psycopg.connect(app.settings.server.postgres_dsn) as connection: - refs=_db_refs(connection) - for source,name in ((app.settings.server.content_dir,'content'), - (app.settings.server.repositories_dir,'repositories'), - (app.settings.server.blob_dir,'blobs'), - (app.settings.server.staging_dir,'staging')): + refs = _db_refs(connection) + for source, name in ( + (app.settings.server.content_dir, 'content'), + (app.settings.server.repositories_dir, 'repositories'), + (app.settings.server.blob_dir, 'blobs'), + (app.settings.server.staging_dir, 'staging'), + ): if source.exists(): _regular_tree(source) - await asyncio.to_thread(shutil.copytree,source,directory/name) + await asyncio.to_thread(shutil.copytree, source, directory / name) else: - (directory/name).mkdir() + (directory / name).mkdir() _regular_tree(app.settings.service_keys) - await asyncio.to_thread(shutil.copytree,app.settings.service_keys,directory/'service') - require(all((directory/'service'/name).is_file() for name in _KEYS), - 'service_keys_missing') - require(app.settings.trust_file.is_file() and not app.settings.trust_file.is_symlink(), - 'backup_trust_missing') - shutil.copy2(app.settings.trust_file,directory/'root-public.json') + await asyncio.to_thread( + shutil.copytree, app.settings.service_keys, directory / 'service' + ) + require( + all((directory / 'service' / name).is_file() for name in _KEYS), + 'service_keys_missing', + ) + require( + app.settings.trust_file.is_file() and not app.settings.trust_file.is_symlink(), + 'backup_trust_missing', + ) + shutil.copy2(app.settings.trust_file, directory / 'root-public.json') # Keep policy/configuration for review. Restore rewrites storage paths, # disables Valkey and does not copy mail credentials. - config_source=(app.settings.config_dir/'msgd.toml' - if (app.settings.config_dir/'msgd.toml').exists() - else app.settings.config_dir/'server.toml') - require(config_source.is_file() and not config_source.is_symlink(), - 'backup_configuration_missing') - shutil.copy2(config_source,directory/'source-config.toml') - _verify_storage(directory,refs) - files={str(path.relative_to(directory)):_hash(path) for path in directory.rglob('*') if path.is_file()} - manifest={'format':_FORMAT,'service_url':app.settings.service_url,'files':files, - 'references':refs,'root_private_key_included':False, - 'mail_credentials_included':False} - (directory/'manifest.json').write_bytes(canonical(manifest)) - partial=directory/'archive.zip' - with zipfile.ZipFile(partial,'x',compression=zipfile.ZIP_DEFLATED) as archive: - for name in ('manifest.json',*sorted(files)): - archive.write(directory/name,name) - os.chmod(partial,0o600) - with partial.open('rb') as stream:os.fsync(stream.fileno()) - os.link(partial,destination) - return {'status':'backup_created','path':str(destination),'sha256':_hash(destination), - 'root_private_key_included':False} + config_source = ( + app.settings.config_dir / 'msgd.toml' + if (app.settings.config_dir / 'msgd.toml').exists() + else app.settings.config_dir / 'server.toml' + ) + require( + config_source.is_file() and not config_source.is_symlink(), + 'backup_configuration_missing', + ) + shutil.copy2(config_source, directory / 'source-config.toml') + _verify_storage(directory, refs) + files = { + str(path.relative_to(directory)): _hash(path) + for path in directory.rglob('*') + if path.is_file() + } + manifest = { + 'format': _FORMAT, + 'service_url': app.settings.service_url, + 'files': files, + 'references': refs, + 'root_private_key_included': False, + 'mail_credentials_included': False, + } + (directory / 'manifest.json').write_bytes(canonical(manifest)) + partial = directory / 'archive.zip' + with zipfile.ZipFile(partial, 'x', compression=zipfile.ZIP_DEFLATED) as archive: + for name in ('manifest.json', *sorted(files)): + archive.write(directory / name, name) + os.chmod(partial, 0o600) + with partial.open('rb') as stream: + os.fsync(stream.fileno()) + os.link(partial, destination) + return { + 'status': 'backup_created', + 'path': str(destination), + 'sha256': _hash(destination), + 'root_private_key_included': False, + } -def restore(source,config_dir,data_dir,*,postgres_dsn='service=msgd'): +def restore(source, config_dir, data_dir, *, postgres_dsn='service=msgd'): """Restore only into new locations and an empty PostgreSQL database.""" - config_dir,data_dir=Path(config_dir),Path(data_dir) - require(not config_dir.exists() and not data_dir.exists(),'restore_target_exists') - config_dir.parent.mkdir(parents=True,exist_ok=True) - data_dir.parent.mkdir(parents=True,exist_ok=True) - with tempfile.TemporaryDirectory(prefix='msg-restore-',dir=data_dir.parent) as temp: - directory=Path(temp) + config_dir, data_dir = Path(config_dir), Path(data_dir) + require(not config_dir.exists() and not data_dir.exists(), 'restore_target_exists') + config_dir.parent.mkdir(parents=True, exist_ok=True) + data_dir.parent.mkdir(parents=True, exist_ok=True) + with tempfile.TemporaryDirectory(prefix='msg-restore-', dir=data_dir.parent) as temp: + directory = Path(temp) with zipfile.ZipFile(source) as archive: - names=archive.namelist() - require(len(names)==len(set(names)) and 'manifest.json' in names,'invalid_backup') - manifest=loads(archive.read('manifest.json')) - require(manifest.get('format')==_FORMAT and manifest.get('root_private_key_included') is False, - 'invalid_backup') - require(set(names)==set(manifest['files'])|{'manifest.json'},'backup_file_mismatch') + names = archive.namelist() + require(len(names) == len(set(names)) and 'manifest.json' in names, 'invalid_backup') + manifest = loads(archive.read('manifest.json')) + require( + manifest.get('format') == _FORMAT + and manifest.get('root_private_key_included') is False, + 'invalid_backup', + ) + require( + set(names) == set(manifest['files']) | {'manifest.json'}, 'backup_file_mismatch' + ) for info in archive.infolist(): - name=info.filename - require(not name.startswith('/') and '\\' not in name and - all(part not in {'','..','.'} for part in name.split('/')),'unsafe_backup_path') - require((info.external_attr>>16)&0o170000 not in {0o120000,0o060000,0o020000},'unsafe_backup_entry') - target=directory/name;target.parent.mkdir(mode=0o700,parents=True,exist_ok=True) - with archive.open(info) as incoming,target.open('xb') as output: - shutil.copyfileobj(incoming,output,length=65536) - os.chmod(target,0o600) - if name!='manifest.json':require(_hash(target)==manifest['files'][name],'backup_digest_mismatch') + name = info.filename + require( + not name.startswith('/') + and '\\' not in name + and all(part not in {'', '..', '.'} for part in name.split('/')), + 'unsafe_backup_path', + ) + require( + (info.external_attr >> 16) & 0o170000 not in {0o120000, 0o060000, 0o020000}, + 'unsafe_backup_entry', + ) + target = directory / name + target.parent.mkdir(mode=0o700, parents=True, exist_ok=True) + with archive.open(info) as incoming, target.open('xb') as output: + shutil.copyfileobj(incoming, output, length=65536) + os.chmod(target, 0o600) + if name != 'manifest.json': + require(_hash(target) == manifest['files'][name], 'backup_digest_mismatch') # Empty storage directories have no ZIP file entries. - for name in ('content','repositories','blobs','staging'): - (directory/name).mkdir(exist_ok=True) - require((directory/'metadata.dump').is_file() and - (directory/'root-public.json').is_file() and - (directory/'source-config.toml').is_file() and - all((directory/'service'/name).is_file() for name in _KEYS), - 'backup_required_file_missing') - _verify_storage(directory,manifest['references']) - fields=conninfo_to_dict(postgres_dsn) - password=fields.pop('password',None) - env=os.environ.copy() - if password is not None:env['PGPASSWORD']=password - safe_dsn=make_conninfo(**fields) + for name in ('content', 'repositories', 'blobs', 'staging'): + (directory / name).mkdir(exist_ok=True) + require( + (directory / 'metadata.dump').is_file() + and (directory / 'root-public.json').is_file() + and (directory / 'source-config.toml').is_file() + and all((directory / 'service' / name).is_file() for name in _KEYS), + 'backup_required_file_missing', + ) + _verify_storage(directory, manifest['references']) + fields = conninfo_to_dict(postgres_dsn) + password = fields.pop('password', None) + env = os.environ.copy() + if password is not None: + env['PGPASSWORD'] = password + safe_dsn = make_conninfo(**fields) with psycopg.connect(postgres_dsn) as connection: - require(connection.execute( - "SELECT 1 FROM information_schema.tables WHERE table_schema=current_schema() LIMIT 1" - ).fetchone() is None,'restore_database_not_empty') - quarantine={'format':'msg-recovery-quarantine-v1','outbound_enabled':False, - 'source_backup_sha256':_hash(Path(source)), - 'revocation_replay':'required','authority':'health_only'} - restore_dump(directory/'metadata.dump',safe_dsn=safe_dsn,env=env, - quarantine=quarantine) + require( + connection.execute( + 'SELECT 1 FROM information_schema.tables WHERE table_schema=current_schema() LIMIT 1' + ).fetchone() + is None, + 'restore_database_not_empty', + ) + quarantine = { + 'format': 'msg-recovery-quarantine-v1', + 'outbound_enabled': False, + 'source_backup_sha256': _hash(Path(source)), + 'revocation_replay': 'required', + 'authority': 'health_only', + } + restore_dump(directory / 'metadata.dump', safe_dsn=safe_dsn, env=env, quarantine=quarantine) with psycopg.connect(postgres_dsn) as connection: - require(_db_refs(connection)==manifest['references'], - 'backup_database_reference_mismatch') - row=connection.execute("SELECT value FROM settings WHERE key='runtime_config'").fetchone() - runtime=loads(row[0]) if row is not None else {} - require(type(runtime) is dict,'backup_runtime_config_invalid') - runtime['accept_writes']=False - runtime['cleanup_enabled']=False - connection.execute('''INSERT INTO settings(key,value) VALUES('runtime_config',%s) - ON CONFLICT(key) DO UPDATE SET value=EXCLUDED.value''', - (canonical(runtime).decode(),)) + require( + _db_refs(connection) == manifest['references'], 'backup_database_reference_mismatch' + ) + row = connection.execute( + "SELECT value FROM settings WHERE key='runtime_config'" + ).fetchone() + runtime = loads(row[0]) if row is not None else {} + require(type(runtime) is dict, 'backup_runtime_config_invalid') + runtime['accept_writes'] = False + runtime['cleanup_enabled'] = False + connection.execute( + """INSERT INTO settings(key,value) VALUES('runtime_config',%s) + ON CONFLICT(key) DO UPDATE SET value=EXCLUDED.value""", + (canonical(runtime).decode(),), + ) data_dir.mkdir(mode=0o700) - settings=write_example(config_dir,data_dir,manifest['service_url'],postgres_dsn=postgres_dsn) + settings = write_example( + config_dir, data_dir, manifest['service_url'], postgres_dsn=postgres_dsn + ) # Install the fail-closed marker as soon as a runnable config exists, # including if a later filesystem verification fails. - durable_write(config_dir/'recovery-drill.json',canonical({ - **quarantine,'format':'msg-recovery-drill-v1'}),mode=0o600) - _write_restored_config(directory/'source-config.toml',settings,postgres_dsn) - settings=load_settings(config_dir) - for name,target in (('content',settings.server.content_dir), - ('repositories',settings.server.repositories_dir), - ('blobs',settings.server.blob_dir), - ('staging',settings.server.staging_dir)): - if (directory/name).exists(): - target.parent.mkdir(parents=True,exist_ok=True) - shutil.move(str(directory/name),target) - settings.service_keys.parent.mkdir(parents=True,exist_ok=True) - shutil.move(str(directory/'service'),settings.service_keys) - os.chmod(settings.service_keys,0o700) - settings.trust_file.parent.mkdir(parents=True,exist_ok=True) + durable_write( + config_dir / 'recovery-drill.json', + canonical({**quarantine, 'format': 'msg-recovery-drill-v1'}), + mode=0o600, + ) + _write_restored_config(directory / 'source-config.toml', settings, postgres_dsn) + settings = load_settings(config_dir) + for name, target in ( + ('content', settings.server.content_dir), + ('repositories', settings.server.repositories_dir), + ('blobs', settings.server.blob_dir), + ('staging', settings.server.staging_dir), + ): + if (directory / name).exists(): + target.parent.mkdir(parents=True, exist_ok=True) + shutil.move(str(directory / name), target) + settings.service_keys.parent.mkdir(parents=True, exist_ok=True) + shutil.move(str(directory / 'service'), settings.service_keys) + os.chmod(settings.service_keys, 0o700) + settings.trust_file.parent.mkdir(parents=True, exist_ok=True) from msg.security.trust_files import write_trust - write_trust(settings.config_dir,loads((directory/'root-public.json').read_bytes()), - writer=durable_write) - _relink_lfs(data_dir,settings) - _verify_storage(data_dir,manifest['references'],settings=settings) - return {'status':'restored','root_admin_material':'restore_separate_encrypted_root_backup_locally', - 'mail':'disabled_until_configured','outbound':'disabled_recovery_drill', - 'revocation_replay':'required','promotion':'blocked'} + + write_trust( + settings.config_dir, + loads((directory / 'root-public.json').read_bytes()), + writer=durable_write, + ) + _relink_lfs(data_dir, settings) + _verify_storage(data_dir, manifest['references'], settings=settings) + return { + 'status': 'restored', + 'root_admin_material': 'restore_separate_encrypted_root_backup_locally', + 'mail': 'disabled_until_configured', + 'outbound': 'disabled_recovery_drill', + 'revocation_replay': 'required', + 'promotion': 'blocked', + } diff --git a/src/msg/admin/config_check.py b/src/msg/admin/config_check.py index fe9940b9..7d3fc385 100644 --- a/src/msg/admin/config_check.py +++ b/src/msg/admin/config_check.py @@ -4,15 +4,15 @@ Selftest writes temporary TOML files and calls the real loader for each option. It does not connect to SMTP, bind listeners, read a credential, or provision keys. """ -from copy import deepcopy + import json +from copy import deepcopy from pathlib import Path from tempfile import TemporaryDirectory from msg.config_contracts import CONFIGURATION_FIELDS from msg.core.errors import Failure, require - _REQUIRED = object() _DISABLED = object() @@ -20,95 +20,188 @@ def _vectors(): from msg.plugins import BUILTINS from msg.security.age_keys import recipient_from_public - recipient = recipient_from_public(bytes.fromhex('09'+'00'*31)) - custodian = {'id':'test-custodian','name':'Isolated custodian','recipient':recipient} + + recipient = recipient_from_public(bytes.fromhex('09' + '00' * 31)) + custodian = {'id': 'test-custodian', 'name': 'Isolated custodian', 'recipient': recipient} # These are independent acceptance expectations, not configuration defaults # used by the server. Each case asserts the loader's effective Settings value. rows = { - 'server.service_url': ('https://msg.lmm.best','https://config.invalid','file:///tmp/x','invalid_service_url'), - 'server.listen': ('127.0.0.1','::1','','invalid_listen_address'), - 'server.port': (8042,65535,0,'invalid_listen_port'), - 'server.public_web_origin': (None,'https://msg.lmm.best','https://elsewhere.invalid','hosting_origin_must_match_service'), - 'server.temporary_ttl': (3600,1,0,'invalid_ttl'), - 'server.transfer_ttl': (86400,1,0,'invalid_ttl'), - 'storage.postgres_dsn': (_REQUIRED,'service=isolated','sqlite:///tmp/x','invalid_postgres_dsn'), - 'storage.valkey_url': (None,'redis://localhost:6379/0','http://localhost:6379','invalid_valkey_url'), - 'storage.content': ('/var/lib/msgd/git/content','/tmp/config-content','relative','storage_paths_must_be_absolute'), - 'storage.repositories': ('/var/lib/msgd/git/repos','/tmp/config-repos','relative','storage_paths_must_be_absolute'), - 'storage.blobs': ('/var/lib/msgd/blobs/sha256','/tmp/config-blobs','relative','storage_paths_must_be_absolute'), - 'storage.staging': ('/var/lib/msgd/transfers/staging','/tmp/config-staging','relative','storage_paths_must_be_absolute'), - 'storage.service_keys': ('/var/lib/msgd/service','/tmp/config-service','relative','storage_paths_must_be_absolute'), - 'limits.request_bytes': (1048576,256,255,'invalid_limit'), - 'limits.response_bytes': (1048576,256,255,'invalid_limit'), - 'limits.path_bytes': (8192,256,255,'invalid_limit'), - 'limits.part_bytes': (65536,256,255,'invalid_limit'), - 'plugins.enabled': (list(BUILTINS),['identity'],['arbitrary'],'unknown_plugin'), - 'tools.isolation': ('bubblewrap','bubblewrap','none','unsafe_tool_worker'), - 'tools.timeout_ms': (10000,1,0,'invalid_tool_limit'), - 'tools.max_response_bytes': (4194304,1,0,'invalid_tool_limit'), - 'tools.methods': (['GET','HEAD'],['GET'],['EXEC'],'invalid_tool_methods'), - 'tools.ports': ([80,443],[65535],[0],'invalid_tool_ports'), - 'identity.credential_delivery_recovery_window': (900,'60m','61m','invalid_credential_delivery_recovery_window'), - 'money.enabled': (True,True,False,'invalid_money_configuration'), - 'money.currency_id': ('primary','primary','other','invalid_money_configuration'), - 'money.display_name': ('MSG','Community credits','','invalid_money_display_name'), - 'money.code': ('MSG','CREDIT','msg','invalid_money_code'), - 'money.scale': (6,6,7,'invalid_money_configuration'), - 'money.transfer_fee': (0,0,1,'invalid_money_configuration'), - 'money.allow_overdraft': (False,False,True,'invalid_money_configuration'), - 'hosting.base_capacity_bytes': (10485760,1024**4,0,'invalid_hosting_capacity'), - 'hosting.recovery_marker': (None,'/tmp/config-marker','relative','invalid_hosting_recovery_marker'), - 'hosting.content_group_read': (False,True,'true','invalid_content_group_read'), - 'recovery.custodians': ([],[custodian],[custodian]*17,'invalid_recovery_custodians'), - 'mail.enabled': (_DISABLED,False,'false','invalid_mail_enabled'), - 'mail.host': (_DISABLED,'smtp.example.test','bad host','invalid_mail_host'), - 'mail.port': (_DISABLED,465,65536,'invalid_mail_port'), - 'mail.tls': (_DISABLED,'tls','plaintext','mail_tls_required'), - 'mail.sender': (_DISABLED,'Message service ','sender\r\ninjected','invalid_mail_sender'), - 'mail.credential_file': (_DISABLED,'/tmp/unread-smtp-credentials','relative','invalid_mail_credential_file'), - 'recovery.custodians.id': (_DISABLED,'other-custodian','INVALID','invalid_recovery_custodian'), - 'recovery.custodians.name': (_DISABLED,'Other custodian','','invalid_recovery_custodian'), - 'recovery.custodians.recipient': (_DISABLED,recipient,'invalid','invalid_age_recipient'), - 'recovery.custodians.description': (_DISABLED,'Description','x'*501,'invalid_recovery_custodian'), - 'recovery.custodians.policy_ref': (_DISABLED,'/wiki/recovery-policy','x'*501,'invalid_recovery_custodian'), + 'server.service_url': ( + 'https://msg.lmm.best', + 'https://config.invalid', + 'file:///tmp/x', + 'invalid_service_url', + ), + 'server.listen': ('127.0.0.1', '::1', '', 'invalid_listen_address'), + 'server.port': (8042, 65535, 0, 'invalid_listen_port'), + 'server.public_web_origin': ( + None, + 'https://msg.lmm.best', + 'https://elsewhere.invalid', + 'hosting_origin_must_match_service', + ), + 'server.temporary_ttl': (3600, 1, 0, 'invalid_ttl'), + 'server.transfer_ttl': (86400, 1, 0, 'invalid_ttl'), + 'storage.postgres_dsn': ( + _REQUIRED, + 'service=isolated', + 'sqlite:///tmp/x', + 'invalid_postgres_dsn', + ), + 'storage.valkey_url': ( + None, + 'redis://localhost:6379/0', + 'http://localhost:6379', + 'invalid_valkey_url', + ), + 'storage.content': ( + '/var/lib/msgd/git/content', + '/tmp/config-content', + 'relative', + 'storage_paths_must_be_absolute', + ), + 'storage.repositories': ( + '/var/lib/msgd/git/repos', + '/tmp/config-repos', + 'relative', + 'storage_paths_must_be_absolute', + ), + 'storage.blobs': ( + '/var/lib/msgd/blobs/sha256', + '/tmp/config-blobs', + 'relative', + 'storage_paths_must_be_absolute', + ), + 'storage.staging': ( + '/var/lib/msgd/transfers/staging', + '/tmp/config-staging', + 'relative', + 'storage_paths_must_be_absolute', + ), + 'storage.service_keys': ( + '/var/lib/msgd/service', + '/tmp/config-service', + 'relative', + 'storage_paths_must_be_absolute', + ), + 'limits.request_bytes': (1048576, 256, 255, 'invalid_limit'), + 'limits.response_bytes': (1048576, 256, 255, 'invalid_limit'), + 'limits.path_bytes': (8192, 256, 255, 'invalid_limit'), + 'limits.part_bytes': (65536, 256, 255, 'invalid_limit'), + 'plugins.enabled': (list(BUILTINS), ['identity'], ['arbitrary'], 'unknown_plugin'), + 'tools.isolation': ('bubblewrap', 'bubblewrap', 'none', 'unsafe_tool_worker'), + 'tools.timeout_ms': (10000, 1, 0, 'invalid_tool_limit'), + 'tools.max_response_bytes': (4194304, 1, 0, 'invalid_tool_limit'), + 'tools.methods': (['GET', 'HEAD'], ['GET'], ['EXEC'], 'invalid_tool_methods'), + 'tools.ports': ([80, 443], [65535], [0], 'invalid_tool_ports'), + 'identity.credential_delivery_recovery_window': ( + 900, + '60m', + '61m', + 'invalid_credential_delivery_recovery_window', + ), + 'money.enabled': (True, True, False, 'invalid_money_configuration'), + 'money.currency_id': ('primary', 'primary', 'other', 'invalid_money_configuration'), + 'money.display_name': ('MSG', 'Community credits', '', 'invalid_money_display_name'), + 'money.code': ('MSG', 'CREDIT', 'msg', 'invalid_money_code'), + 'money.scale': (6, 6, 7, 'invalid_money_configuration'), + 'money.transfer_fee': (0, 0, 1, 'invalid_money_configuration'), + 'money.allow_overdraft': (False, False, True, 'invalid_money_configuration'), + 'hosting.base_capacity_bytes': (10485760, 1024**4, 0, 'invalid_hosting_capacity'), + 'hosting.recovery_marker': ( + None, + '/tmp/config-marker', + 'relative', + 'invalid_hosting_recovery_marker', + ), + 'hosting.content_group_read': (False, True, 'true', 'invalid_content_group_read'), + 'recovery.custodians': ([], [custodian], [custodian] * 17, 'invalid_recovery_custodians'), + 'mail.enabled': (_DISABLED, False, 'false', 'invalid_mail_enabled'), + 'mail.host': (_DISABLED, 'smtp.example.test', 'bad host', 'invalid_mail_host'), + 'mail.port': (_DISABLED, 465, 65536, 'invalid_mail_port'), + 'mail.tls': (_DISABLED, 'tls', 'plaintext', 'mail_tls_required'), + 'mail.sender': ( + _DISABLED, + 'Message service ', + 'sender\r\ninjected', + 'invalid_mail_sender', + ), + 'mail.credential_file': ( + _DISABLED, + '/tmp/unread-smtp-credentials', + 'relative', + 'invalid_mail_credential_file', + ), + 'recovery.custodians.id': ( + _DISABLED, + 'other-custodian', + 'INVALID', + 'invalid_recovery_custodian', + ), + 'recovery.custodians.name': ( + _DISABLED, + 'Other custodian', + '', + 'invalid_recovery_custodian', + ), + 'recovery.custodians.recipient': (_DISABLED, recipient, 'invalid', 'invalid_age_recipient'), + 'recovery.custodians.description': ( + _DISABLED, + 'Description', + 'x' * 501, + 'invalid_recovery_custodian', + ), + 'recovery.custodians.policy_ref': ( + _DISABLED, + '/wiki/recovery-policy', + 'x' * 501, + 'invalid_recovery_custodian', + ), } return rows, custodian def _toml(value): - if isinstance(value,dict): - return '{'+', '.join(key+' = '+_toml(item) for key,item in value.items())+'}' - if isinstance(value,(list,tuple)): - return '['+', '.join(_toml(item) for item in value)+']' - return json.dumps(value,ensure_ascii=False) + if isinstance(value, dict): + return '{' + ', '.join(key + ' = ' + _toml(item) for key, item in value.items()) + '}' + if isinstance(value, (list, tuple)): + return '[' + ', '.join(_toml(item) for item in value) + ']' + return json.dumps(value, ensure_ascii=False) def _write(directory, data, mail): - (directory/'msgd.toml').write_text('\n'.join( - '['+section+']\n'+'\n'.join(key+' = '+_toml(value) for key,value in values.items()) - for section,values in data.items())) - path=directory/'mail.toml' + (directory / 'msgd.toml').write_text( + '\n'.join( + '[' + + section + + ']\n' + + '\n'.join(key + ' = ' + _toml(value) for key, value in values.items()) + for section, values in data.items() + ) + ) + path = directory / 'mail.toml' if mail is None: path.unlink(missing_ok=True) else: - path.write_text('\n'.join(key+' = '+_toml(value) for key,value in mail.items())) + path.write_text('\n'.join(key + ' = ' + _toml(value) for key, value in mail.items())) def _effective(settings, field): - value=settings + value = settings for part in CONFIGURATION_FIELDS[field]['setting'].split('.'): if value is None: return None if part.isdigit(): - value=value[int(part)] if value else None + value = value[int(part)] if value else None else: - value=getattr(value,part) - if isinstance(value,Path): + value = getattr(value, part) + if isinstance(value, Path): return str(value) - if isinstance(value,tuple): - if field=='recovery.custodians': - return [{key:getattr(item,key) for key in ('id','name','recipient')} - for item in value] + if isinstance(value, tuple): + if field == 'recovery.custodians': + return [ + {key: getattr(item, key) for key in ('id', 'name', 'recipient')} for item in value + ] return list(value) return value @@ -116,74 +209,107 @@ def _effective(settings, field): def configuration_selftest(): """Return only status/code per field; never return loaded configuration values.""" from msg.config import load_settings - vectors,custodian=_vectors() - require(set(vectors)==set(CONFIGURATION_FIELDS),'configuration_diagnostic_coverage_missing') - results={} + + vectors, custodian = _vectors() + require(set(vectors) == set(CONFIGURATION_FIELDS), 'configuration_diagnostic_coverage_missing') + results = {} with TemporaryDirectory(prefix='msg-config-selftest-') as temporary: - directory=Path(temporary) - baseline={'storage':{'postgres_dsn':'service=msgd'}} - for field,(default,good,bad,error_code) in vectors.items(): + directory = Path(temporary) + baseline = {'storage': {'postgres_dsn': 'service=msgd'}} + for field, (default, good, bad, error_code) in vectors.items(): try: - _write(directory,baseline,None) - defaults=load_settings(directory) + _write(directory, baseline, None) + defaults = load_settings(directory) if default is _REQUIRED: - _write(directory,{},None) + _write(directory, {}, None) try: load_settings(directory) except Failure as exc: - require(exc.code==error_code,'configuration_default_mismatch') + require(exc.code == error_code, 'configuration_default_mismatch') else: raise Failure('configuration_default_mismatch') - default_status='required' + default_status = 'required' elif default is _DISABLED: - require(_effective(defaults,field) is None,'configuration_default_mismatch') - default_status='disabled' + require(_effective(defaults, field) is None, 'configuration_default_mismatch') + default_status = 'disabled' else: - require(_effective(defaults,field)==default,'configuration_default_mismatch') - default_status='pass' - for accepted,value in ((True,good),(False,bad)): - data=deepcopy(baseline) - mail=None + require( + _effective(defaults, field) == default, 'configuration_default_mismatch' + ) + default_status = 'pass' + for accepted, value in ((True, good), (False, bad)): + data = deepcopy(baseline) + mail = None if field.startswith('mail.'): - mail={'enabled':True,'host':'smtp.example.test','tls':'starttls', - 'sender':'sender@example.test'} - mail[field.split('.')[1]]=value + mail = { + 'enabled': True, + 'host': 'smtp.example.test', + 'tls': 'starttls', + 'sender': 'sender@example.test', + } + mail[field.split('.')[1]] = value elif field.startswith('recovery.custodians.'): - item=dict(custodian) - item[field.rsplit('.',1)[1]]=value - data['recovery']={'custodians':[item]} + item = dict(custodian) + item[field.rsplit('.', 1)[1]] = value + data['recovery'] = {'custodians': [item]} else: - section,key=field.split('.') - data.setdefault(section,{})[key]=value - _write(directory,data,mail) + section, key = field.split('.') + data.setdefault(section, {})[key] = value + _write(directory, data, mail) try: - loaded=load_settings(directory) + loaded = load_settings(directory) except Failure as exc: - require(not accepted and exc.code==error_code, - 'configuration_rejection_mismatch') + require( + not accepted and exc.code == error_code, + 'configuration_rejection_mismatch', + ) else: - require(accepted,'configuration_rejection_missing') - expected=(int(good[:-1])*60 if field=='identity.credential_delivery_recovery_window' - else None if field=='mail.enabled' else good) - require(_effective(loaded,field)==expected,'configuration_effective_value_mismatch') - results[field]={'status':'pass','default':default_status, - 'accepted':'pass','rejected':'pass'} - except (Failure,ValueError,TypeError,KeyError,OSError,AttributeError) as exc: - results[field]={'status':'fail','code':getattr(exc,'code','configuration_check_failed')} + require(accepted, 'configuration_rejection_missing') + expected = ( + int(good[:-1]) * 60 + if field == 'identity.credential_delivery_recovery_window' + else None + if field == 'mail.enabled' + else good + ) + require( + _effective(loaded, field) == expected, + 'configuration_effective_value_mismatch', + ) + results[field] = { + 'status': 'pass', + 'default': default_status, + 'accepted': 'pass', + 'rejected': 'pass', + } + except (Failure, ValueError, TypeError, KeyError, OSError, AttributeError) as exc: + results[field] = { + 'status': 'fail', + 'code': getattr(exc, 'code', 'configuration_check_failed'), + } return results def configuration_doctor(observations): """Parsing success does not prove listener, SMTP, OS permissions or live limits.""" - result={} - for field,spec in CONFIGURATION_FIELDS.items(): - row={'status':'pass' if observations.get('configuration',{}).get('ok') is True else 'fail', - 'scope':'configuration_loading','selftest_case':field} - check=spec['doctor_check'] - value=observations.get(check,{}) if check else {} - row['related_check']={'check':check,'status': - 'disabled' if value.get('status')=='disabled' else - 'pass' if value.get('ok') is True else - 'fail' if value.get('ok') is False else 'not_checked'} - result[field]=row + result = {} + for field, spec in CONFIGURATION_FIELDS.items(): + row = { + 'status': 'pass' if observations.get('configuration', {}).get('ok') is True else 'fail', + 'scope': 'configuration_loading', + 'selftest_case': field, + } + check = spec['doctor_check'] + value = observations.get(check, {}) if check else {} + row['related_check'] = { + 'check': check, + 'status': 'disabled' + if value.get('status') == 'disabled' + else 'pass' + if value.get('ok') is True + else 'fail' + if value.get('ok') is False + else 'not_checked', + } + result[field] = row return result diff --git a/src/msg/admin/content_edit_check.py b/src/msg/admin/content_edit_check.py index 5e08bd9a..f528f221 100644 --- a/src/msg/admin/content_edit_check.py +++ b/src/msg/admin/content_edit_check.py @@ -1,4 +1,5 @@ """Bounded, real-operation content editing vector for the isolated selftest.""" + from msg.core.codec import b64 from msg.core.errors import require from msg.core.models import ResourceRef @@ -9,20 +10,32 @@ async def check_content_editing(app, call, register, test_path): other_key, other = await register('content-edit-outsider') original = '# Selftest\n\nKeep this line.\nReplace this word.\n' expected_text = original.replace('word', 'value') - created = await call('file.create', { - 'parent': test_path + '/@content-edit-owner/files', 'name': 'edit.md', - 'data': b64(original.encode()), 'media_type': 'text/markdown', - }, owner_key, owner) + created = await call( + 'file.create', + { + 'parent': test_path + '/@content-edit-owner/files', + 'name': 'edit.md', + 'data': b64(original.encode()), + 'media_type': 'text/markdown', + }, + owner_key, + owner, + ) require(created.status == 'ok', 'selftest_content_create_failed') rid = created.resources[0].id old_ref = created.resources[0] async with app.metadata.transaction(write=False) as tx: original_revision = await tx.revision(old_ref) patch = {'kind': 'exact', 'exact': 'word', 'replacement': 'value'} - args = {'id': rid, 'base_revision': old_ref.revision, - 'base_generation': created.data['generation'], 'patch': patch} - patched = await call('file.patch', args, owner_key, owner, - expected=((rid, created.data['generation']),)) + args = { + 'id': rid, + 'base_revision': old_ref.revision, + 'base_generation': created.data['generation'], + 'patch': patch, + } + patched = await call( + 'file.patch', args, owner_key, owner, expected=((rid, created.data['generation']),) + ) require(patched.status == 'ok', 'selftest_content_patch_failed') generation = patched.data['generation'] tables = ('resources', 'revisions', 'events', 'results', 'audit', 'jobs') @@ -30,27 +43,43 @@ async def check_content_editing(app, call, register, test_path): before = tuple(tx.one('SELECT COUNT(*) FROM ' + table)[0] for table in tables) current_before = await tx.resource(rid) # Current generation lets this request reach the stale base_revision check. - stale = await call('file.patch', args, owner_key, owner, - expected=((rid, generation),)) - denied = await call('file.patch', { - **args, 'base_revision': patched.resources[0].revision, - 'base_generation': generation, - 'patch': {'kind': 'exact', 'exact': 'value', 'replacement': 'unauthorized'}, - }, other_key, other, expected=((rid, generation),)) + stale = await call('file.patch', args, owner_key, owner, expected=((rid, generation),)) + denied = await call( + 'file.patch', + { + **args, + 'base_revision': patched.resources[0].revision, + 'base_generation': generation, + 'patch': {'kind': 'exact', 'exact': 'value', 'replacement': 'unauthorized'}, + }, + other_key, + other, + expected=((rid, generation),), + ) current = await call('file.read', {'id': rid}, owner_key, owner) previous = await call('file.read', {'id': rid, 'revision': old_ref.revision}, owner_key, owner) history = await call('file.read', {'id': rid, 'view': 'history', 'limit': 10}, owner_key, owner) async with app.metadata.transaction(write=False) as tx: after = tuple(tx.one('SELECT COUNT(*) FROM ' + table)[0] for table in tables) - unchanged = (await tx.resource(rid) == current_before and - await tx.revision(old_ref) == original_revision) + unchanged = ( + await tx.resource(rid) == current_before + and await tx.revision(old_ref) == original_revision + ) revision = await tx.revision(ResourceRef(id=rid)) - return (stale.error is not None and stale.error.code == 'revision_conflict' and - denied.error is not None and denied.error.code == 'permission_denied' and - all(result.status == 'ok' for result in (current, previous, history)) and - current.data['content'] == expected_text and previous.data['content'] == original and - {row['id'] for row in history.data['revisions']} == - {old_ref.revision, patched.resources[0].revision} and - len(history.data['revisions']) == 2 and - revision.parents == (old_ref.revision,) and revision.author == owner and - generation == created.data['generation'] + 1 and before == after and unchanged) + return ( + stale.error is not None + and stale.error.code == 'revision_conflict' + and denied.error is not None + and denied.error.code == 'permission_denied' + and all(result.status == 'ok' for result in (current, previous, history)) + and current.data['content'] == expected_text + and previous.data['content'] == original + and {row['id'] for row in history.data['revisions']} + == {old_ref.revision, patched.resources[0].revision} + and len(history.data['revisions']) == 2 + and revision.parents == (old_ref.revision,) + and revision.author == owner + and generation == created.data['generation'] + 1 + and before == after + and unchanged + ) diff --git a/src/msg/admin/custodial_check.py b/src/msg/admin/custodial_check.py index 04873034..773072e7 100644 --- a/src/msg/admin/custodial_check.py +++ b/src/msg/admin/custodial_check.py @@ -1,4 +1,5 @@ """Read-only lifecycle inspection and an isolated real-age history selftest.""" + from dataclasses import replace from pathlib import Path from tempfile import TemporaryDirectory @@ -12,20 +13,33 @@ async def inspect_custodial_history(tx): from msg.security.custodial_migration import snapshot + constraint = tx.one("""SELECT pg_get_constraintdef(oid) FROM pg_constraint WHERE conrelid='custodial_vault'::regclass AND conname='custodial_vault_lifecycle_check'""") - require(constraint is not None and 'decrypt_only' in constraint[0], 'custodial_lifecycle_schema_missing') + require( + constraint is not None and 'decrypt_only' in constraint[0], + 'custodial_lifecycle_schema_missing', + ) states = [] for subject, status, challenge, body in tx.rows( - 'SELECT subject,status,challenge,body FROM custodial_upgrades ORDER BY id'): + 'SELECT subject,status,challenge,body FROM custodial_upgrades ORDER BY id' + ): if status == 'failed': continue view = (await snapshot(tx, subject, status, loads(challenge), loads(body))).data - states.append({'subject': subject, 'challenge_id': view['challenge_id'], 'phase': view['phase'], - 'unresolved_count': len(view['unresolved_revisions']), 'inventory_changed': view['inventory_changed'], - 'retirement': view['retirement']}) - return {'migrations': states, 'backup_retirement_verified': False, - 'verification_scope': 'online_state_only'} + states.append({ + 'subject': subject, + 'challenge_id': view['challenge_id'], + 'phase': view['phase'], + 'unresolved_count': len(view['unresolved_revisions']), + 'inventory_changed': view['inventory_changed'], + 'retirement': view['retirement'], + }) + return { + 'migrations': states, + 'backup_retirement_verified': False, + 'verification_scope': 'online_state_only', + } async def check_custodial_history(app, now): @@ -36,42 +50,90 @@ async def check_custodial_history(app, now): from msg.core.models import Resource, Revision from msg.transports.client import HTTPTransport from msg.transports.http import create_app + require(app.selftest_run_id is not None, 'isolated_selftest_required') with TemporaryDirectory(prefix='msg-history-selftest-') as directory: async with httpx.AsyncClient(transport=httpx.ASGITransport(app=create_app(app))) as http: - client = MsgClient(ClientState(Path(directory), server=app.settings.service_url), - HTTPTransport(app.settings.service_url, http=http), clock=lambda: now) + client = MsgClient( + ClientState(Path(directory), server=app.settings.service_url), + HTTPTransport(app.settings.service_url, http=http), + clock=lambda: now, + ) created = client.checked(await client.custodial('history-' + uuid4().hex[:12])) subject = client.state.subject - ciphertext = _age('--encrypt', '--recipient', created.data['encryption_recipient'], - input_data=b'isolated historical plaintext') + ciphertext = _age( + '--encrypt', + '--recipient', + created.data['encryption_recipient'], + input_data=b'isolated historical plaintext', + ) # Explicit fixture fact, not a new privileged network operation. The # normal custodial credential ceiling does not grant keystore writes. async with app.metadata.transaction(write=True) as tx: - folder = tx.one("SELECT id FROM resources WHERE parent=? AND name='keystore'", (subject,))[0] - resource = Resource(id='ks_' + uuid4().hex, type='keystore', type_version=1, - name='history.age', parent=folder, owner=subject, group='g_public', mode=0o600, - generation=0, revision=None, state='active', created_at=now, created_by=subject, - modified_at=now, modified_by=subject) + folder = tx.one( + "SELECT id FROM resources WHERE parent=? AND name='keystore'", (subject,) + )[0] + resource = Resource( + id='ks_' + uuid4().hex, + type='keystore', + type_version=1, + name='history.age', + parent=folder, + owner=subject, + group='g_public', + mode=0o600, + generation=0, + revision=None, + state='active', + created_at=now, + created_by=subject, + modified_at=now, + modified_by=subject, + ) await tx.insert(resource) blob = await app.contents.put_bytes(ciphertext, 'application/octet-stream') - revision = Revision(format_version=1, id='v_' + uuid4().hex, resource_id=resource.id, - parents=(), content=blob, relations=(), actor=subject, subject=subject, author=subject, - created_at=now, manifest_digest='') - revision = replace(revision, manifest_digest=digest({k: v for k, v in wire(revision).items() - if k not in {'manifest_digest', 'signature'}})) + revision = Revision( + format_version=1, + id='v_' + uuid4().hex, + resource_id=resource.id, + parents=(), + content=blob, + relations=(), + actor=subject, + subject=subject, + author=subject, + created_at=now, + manifest_digest='', + ) + revision = replace( + revision, + manifest_digest=digest({ + k: v + for k, v in wire(revision).items() + if k not in {'manifest_digest', 'signature'} + }), + ) await app.contents.pin(blob, revision.id) await app.contents.commit_revision(folder, revision) await tx.append_revision(revision) await tx.replace(replace(resource, generation=1, revision=revision.id), 0) tx.set_setting('keystore_format:' + revision.id, 'age') - pending = client.checked(await client.upgrade_custodial('history-self-' + uuid4().hex[:12], - external_ciphertexts_migrated=True)) + pending = client.checked( + await client.upgrade_custodial( + 'history-self-' + uuid4().hex[:12], external_ciphertexts_migrated=True + ) + ) if pending.data['status'] != 'pending_rewrap': return False result = await migrate(client) if not result['inventory']['history_recoverable']: return False - done = client.checked(await transition(client, 'finalize', external_ciphertexts_migrated=True)) - return (done.data['history_recoverable'] and done.data['retirement']['online_encryption_key_deleted'] - and not done.data['retirement']['backup_retired'] and client.state.token is None) + done = client.checked( + await transition(client, 'finalize', external_ciphertexts_migrated=True) + ) + return ( + done.data['history_recoverable'] + and done.data['retirement']['online_encryption_key_deleted'] + and not done.data['retirement']['backup_retired'] + and client.state.token is None + ) diff --git a/src/msg/admin/diagnostics.py b/src/msg/admin/diagnostics.py index f9d4abac..1e2a4e96 100644 --- a/src/msg/admin/diagnostics.py +++ b/src/msg/admin/diagnostics.py @@ -1,66 +1,95 @@ """Read-only deployment inspection and disposable, real-operation self-tests.""" + from __future__ import annotations + import asyncio -from contextlib import contextmanager -from datetime import UTC, datetime, timedelta -from importlib.util import find_spec import os -from pathlib import Path import shutil -import psycopg import stat import subprocess import sys import tempfile import uuid +from contextlib import contextmanager +from datetime import UTC, datetime, timedelta +from importlib.util import find_spec +from pathlib import Path from urllib.parse import quote + +import psycopg from psycopg import sql -from msg.core.codec import canonical, decode, digest, loads, unb64, wire, b64 -from msg.core.errors import Failure, require -from msg.core.models import Certificate, Resource, ResourceRef, Subject, AuditEvent from msg.constants import ROOT_SUBJECT +from msg.core.codec import b64, canonical, decode, digest, loads, unb64, wire +from msg.core.errors import Failure, require +from msg.core.models import Certificate, ResourceRef @contextmanager def temporary_postgres(): """Run selftest against a new local cluster, never an installed database.""" - configured=os.environ.get('MSG_TEST_POSTGRES_URL_TEMPLATE') + configured = os.environ.get('MSG_TEST_POSTGRES_URL_TEMPLATE') if configured: - name='msg_selftest_'+uuid.uuid4().hex - admin=configured.format(database='postgres') - with psycopg.connect(admin,autocommit=True) as connection: + name = 'msg_selftest_' + uuid.uuid4().hex + admin = configured.format(database='postgres') + with psycopg.connect(admin, autocommit=True) as connection: connection.execute(sql.SQL('CREATE DATABASE {}').format(sql.Identifier(name))) try: yield configured.format(database=name) finally: - with psycopg.connect(admin,autocommit=True) as connection: - connection.execute(sql.SQL('DROP DATABASE {} WITH (FORCE)').format(sql.Identifier(name))) + with psycopg.connect(admin, autocommit=True) as connection: + connection.execute( + sql.SQL('DROP DATABASE {} WITH (FORCE)').format(sql.Identifier(name)) + ) return - with tempfile.TemporaryDirectory(prefix='msg-selftest-pg-',dir='/tmp') as temporary: - root=Path(temporary) - data=root/'data' - socket=root/'socket';socket.mkdir() - subprocess.run(['initdb','-D',str(data),'-A','trust','--no-instructions'], - check=True,capture_output=True,text=True) + with tempfile.TemporaryDirectory(prefix='msg-selftest-pg-', dir='/tmp') as temporary: + root = Path(temporary) + data = root / 'data' + socket = root / 'socket' + socket.mkdir() + subprocess.run( + ['initdb', '-D', str(data), '-A', 'trust', '--no-instructions'], + check=True, + capture_output=True, + text=True, + ) try: - subprocess.run(['pg_ctl','-D',str(data),'-l',str(root/'postgres.log'), - '-o',f"-k {socket} -h '' -p 5432",'-w','start'], - check=True,capture_output=True,text=True) - yield f'postgresql://localhost:5432/postgres?host={quote(str(socket),safe="")}' + subprocess.run( + [ + 'pg_ctl', + '-D', + str(data), + '-l', + str(root / 'postgres.log'), + '-o', + f"-k {socket} -h '' -p 5432", + '-w', + 'start', + ], + check=True, + capture_output=True, + text=True, + ) + yield f'postgresql://localhost:5432/postgres?host={quote(str(socket), safe="")}' finally: - subprocess.run(['pg_ctl','-D',str(data),'-m','immediate','-w','stop'], - check=False,capture_output=True,text=True) + subprocess.run( + ['pg_ctl', '-D', str(data), '-m', 'immediate', '-w', 'stop'], + check=False, + capture_output=True, + text=True, + ) class ReadOnlyStore: """Inspect PostgreSQL without schema creation or repair.""" - def __init__(self,dsn): - self.dsn=dsn + + def __init__(self, dsn): + self.dsn = dsn async def inspect(self, callback): from msg.storage.postgres import PostgresMetadataStore - store=PostgresMetadataStore(self.dsn,initialize=False) + + store = PostgresMetadataStore(self.dsn, initialize=False) async with store.transaction(write=False) as session: return await callback(session) @@ -68,26 +97,32 @@ async def inspect(self, callback): def feature_results(features, observations, field, *, enabled_plugins=None): """Link manifest entries to checks actually run; never synthesize a pass.""" from msg.plugins.features import FEATURE_SOURCES - result={} + + result = {} for feature in features: - feature_id=feature['feature_id'] - name=feature[field] - if (not feature['enabled_by_default'] or - (enabled_plugins is not None and - FEATURE_SOURCES[feature_id][0] not in enabled_plugins)): - result[feature_id]={'status':'disabled','check':name} + feature_id = feature['feature_id'] + name = feature[field] + if not feature['enabled_by_default'] or ( + enabled_plugins is not None and FEATURE_SOURCES[feature_id][0] not in enabled_plugins + ): + result[feature_id] = {'status': 'disabled', 'check': name} elif name is None: - result[feature_id]={'status':'skip','check':None} + result[feature_id] = {'status': 'skip', 'check': None} elif name not in observations: - result[feature_id]={'status':'fail','check':name,'reason':'check_missing'} + result[feature_id] = {'status': 'fail', 'check': name, 'reason': 'check_missing'} else: - value=observations[name] - if isinstance(value,dict): - status=('skip' if value.get('status')=='disabled' else - 'pass' if value.get('ok') is True else 'fail') + value = observations[name] + if isinstance(value, dict): + status = ( + 'skip' + if value.get('status') == 'disabled' + else 'pass' + if value.get('ok') is True + else 'fail' + ) else: - status='pass' if value is True else 'fail' - result[feature_id]={'status':status,'check':name} + status = 'pass' if value is True else 'fail' + result[feature_id] = {'status': status, 'check': name} return result @@ -96,70 +131,117 @@ def inspect_recovery_checkpoint(): from importlib.resources import files from msg.admin.recovery_replay import FORMAT, SUPPORTED_FACTS - schema=loads(files('msg.data').joinpath('recovery-checkpoint.schema.json').read_bytes()) - checkpoint=schema['properties']['checkpoint']['properties'] - kinds=checkpoint['entries']['items']['properties']['kind']['enum'] - require(checkpoint['format'].get('const')==FORMAT and len(kinds)==len(set(kinds)) and - set(kinds)==SUPPORTED_FACTS,'recovery_checkpoint_schema_drift') - return {'fact_kinds':len(kinds),'promotion':'blocked'} + + schema = loads(files('msg.data').joinpath('recovery-checkpoint.schema.json').read_bytes()) + checkpoint = schema['properties']['checkpoint']['properties'] + kinds = checkpoint['entries']['items']['properties']['kind']['enum'] + require( + checkpoint['format'].get('const') == FORMAT + and len(kinds) == len(set(kinds)) + and set(kinds) == SUPPORTED_FACTS, + 'recovery_checkpoint_schema_drift', + ) + return {'fact_kinds': len(kinds), 'promotion': 'blocked'} def inspect_lexical_search(app): - specs=[spec for spec in app.registry.operations() if spec.name=='discovery.lexical_search'] - require(bool(specs) and all(spec.effect=='read' for spec in specs),'lexical_search_contract_invalid') - return {'versions':sorted(spec.version for spec in specs),'read_only':True} + specs = [spec for spec in app.registry.operations() if spec.name == 'discovery.lexical_search'] + require( + bool(specs) and all(spec.effect == 'read' for spec in specs), + 'lexical_search_contract_invalid', + ) + return {'versions': sorted(spec.version for spec in specs), 'read_only': True} def inspect_hosting(app): - for name in ('hosting.create','hosting.deploy'): - require(app.registry.operation(name).require_signature,'hosting_contract_invalid') + for name in ('hosting.create', 'hosting.deploy'): + require(app.registry.operation(name).require_signature, 'hosting_contract_invalid') app.registry.resource_type('website') - return {'operations':['hosting.create@1','hosting.deploy@1']} + return {'operations': ['hosting.create@1', 'hosting.deploy@1']} -async def _selftest_search_hosting(app,call,register,test_path): +async def _selftest_search_hosting(app, call, register, test_path): """Exercise search and publishing only inside the disposable test namespace.""" - handle='test-features' - key,uid=await register(handle) - marker='selftestsearch'+uuid.uuid4().hex[:12] - search=False + handle = 'test-features' + key, uid = await register(handle) + marker = 'selftestsearch' + uuid.uuid4().hex[:12] + search = False try: - post=await call('content.post_create',{'parent':test_path+'/tmp','body':'isolated '+marker},key,uid) - require(post.status=='ok','selftest_search_post_failed') - found=await call('discovery.lexical_search',{'scope':test_path+'/tmp','terms':marker}, - key,uid,contract_version=4) - denied=await call('discovery.lexical_search',{'scope':test_path+'/private','terms':marker}, - key,uid,contract_version=4) - search=(found.status=='ok' and - [item['id'] for item in found.data['items']]==[post.resources[0].id] and - denied.error is not None and denied.error.code=='permission_denied') + post = await call( + 'content.post_create', + {'parent': test_path + '/tmp', 'body': 'isolated ' + marker}, + key, + uid, + ) + require(post.status == 'ok', 'selftest_search_post_failed') + found = await call( + 'discovery.lexical_search', + {'scope': test_path + '/tmp', 'terms': marker}, + key, + uid, + contract_version=4, + ) + denied = await call( + 'discovery.lexical_search', + {'scope': test_path + '/private', 'terms': marker}, + key, + uid, + contract_version=4, + ) + search = ( + found.status == 'ok' + and [item['id'] for item in found.data['items']] == [post.resources[0].id] + and denied.error is not None + and denied.error.code == 'permission_denied' + ) except Failure: - search=False - hosting=False + search = False + hosting = False try: - page=b'

isolated selftest

' - source=await call('content.file_put',{'parent':test_path+'/@'+handle+'/files','name':'index.html', - 'data':b64(page),'media_type':'text/html'},key,uid) - require(source.status=='ok','selftest_hosting_source_failed') - site=await call('hosting.create',{'parent':test_path+'/@'+handle,'name':'site'},key,uid) - require(site.status=='ok','selftest_hosting_create_failed') - website_id=site.resources[0].id - deployed=await call('hosting.deploy',{'id':website_id, - 'entries':[{'path':'index.html','source':wire(source.resources[0])}]},key,uid, - expected=((website_id,site.data['generation']),)) - require(deployed.status=='ok','selftest_hosting_deploy_failed') + page = b'

isolated selftest

' + source = await call( + 'content.file_put', + { + 'parent': test_path + '/@' + handle + '/files', + 'name': 'index.html', + 'data': b64(page), + 'media_type': 'text/html', + }, + key, + uid, + ) + require(source.status == 'ok', 'selftest_hosting_source_failed') + site = await call( + 'hosting.create', {'parent': test_path + '/@' + handle, 'name': 'site'}, key, uid + ) + require(site.status == 'ok', 'selftest_hosting_create_failed') + website_id = site.resources[0].id + deployed = await call( + 'hosting.deploy', + { + 'id': website_id, + 'entries': [{'path': 'index.html', 'source': wire(source.resources[0])}], + }, + key, + uid, + expected=((website_id, site.data['generation']),), + ) + require(deployed.status == 'ok', 'selftest_hosting_deploy_failed') async with app.metadata.transaction(write=False) as tx: - website=await tx.resource(website_id) - revision=await tx.revision(ResourceRef(id=website_id,revision=website.revision)) - manifest=loads(await app.contents.read_bytes(revision.content)) - entry=decode(ResourceRef,manifest['entries']['index.html']) - published=await app.contents.read_bytes((await tx.revision(entry)).content) - site_path=await tx.path(website_id) - hosting=(site_path.startswith(test_path+'/') and set(manifest['entries'])=={'index.html'} and - published==page) - except (Failure,KeyError,TypeError,ValueError): - hosting=False - return {'lexical_search_access':search,'hosting_deploy':hosting} + website = await tx.resource(website_id) + revision = await tx.revision(ResourceRef(id=website_id, revision=website.revision)) + manifest = loads(await app.contents.read_bytes(revision.content)) + entry = decode(ResourceRef, manifest['entries']['index.html']) + published = await app.contents.read_bytes((await tx.revision(entry)).content) + site_path = await tx.path(website_id) + hosting = ( + site_path.startswith(test_path + '/') + and set(manifest['entries']) == {'index.html'} + and published == page + ) + except Failure, KeyError, TypeError, ValueError: + hosting = False + return {'lexical_search_access': search, 'hosting_deploy': hosting} async def authority_snapshot_drift(app, root, online, tx): @@ -171,28 +253,38 @@ async def authority_snapshot_drift(app, root, online, tx): well; a narrower old grant must not be described as covering a broad one. """ from msg.security.policy import constraints_subset, scope_subset - require(root.issuance is not None and online.issuance is not None,'issuer_not_ca') + + require(root.issuance is not None and online.issuance is not None, 'issuer_not_ca') async def missing(expected, actual): - gaps=[] + gaps = [] for desired in expected: - covered=set() + covered = set() for grant in actual: - if (grant.capability==desired.capability and grant.version==desired.version and - await scope_subset(desired.scope,grant.scope,tx) and - constraints_subset(desired.constraints,grant.constraints)): + if ( + grant.capability == desired.capability + and grant.version == desired.version + and await scope_subset(desired.scope, grant.scope, tx) + and constraints_subset(desired.constraints, grant.constraints) + ): covered.update(grant.operations) - operations=sorted(desired.operations-covered) + operations = sorted(desired.operations - covered) if operations: - gaps.append({'capability':desired.capability,'version':desired.version, - 'scope':wire(desired.scope),'operations':operations}) + gaps.append({ + 'capability': desired.capability, + 'version': desired.version, + 'scope': wire(desired.scope), + 'operations': operations, + }) return gaps - primary=app.primary_ceiling() - base=app.base_grants() - return {'root_use':await missing(primary,root.grants), - 'root_issue':await missing(primary,root.issuance.issue_grants), - 'online_issue':await missing(base,online.issuance.issue_grants)} + primary = app.primary_ceiling() + base = app.base_grants() + return { + 'root_use': await missing(primary, root.grants), + 'root_issue': await missing(primary, root.issuance.issue_grants), + 'online_issue': await missing(base, online.issuance.issue_grants), + } def doctor(config_dir=Path('/etc/msgd'), *, clock=None): @@ -200,650 +292,1242 @@ def doctor(config_dir=Path('/etc/msgd'), *, clock=None): # Running the read-only coroutine in a short-lived thread avoids nesting an # asyncio.run() inside adapters/tests with an already running event loop. from concurrent.futures import ThreadPoolExecutor + with ThreadPoolExecutor(max_workers=1) as pool: - return pool.submit(lambda: asyncio.run(_doctor(config_dir,clock))).result() + return pool.submit(lambda: asyncio.run(_doctor(config_dir, clock))).result() -async def _doctor(config_dir,clock): +async def _doctor(config_dir, clock): from msg.application import Application + from msg.bootstrap import feature_manifest, manifest from msg.config import load_settings - from msg.bootstrap import manifest,feature_manifest from msg.security.certificates import CertificateValidator - checks={} - warnings=[] - def success(name,**data):checks[name]={'ok':True,**data} - def failed(name,code,**data):checks[name]={'ok':False,'code':code,**data} - path=Path(config_dir) + + checks = {} + warnings = [] + + def success(name, **data): + checks[name] = {'ok': True, **data} + + def failed(name, code, **data): + checks[name] = {'ok': False, 'code': code, **data} + + path = Path(config_dir) try: - settings=load_settings(path) - app=Application(settings,clock=clock) + settings = load_settings(path) + app = Application(settings, clock=clock) success('configuration') - success('money_config',enabled=settings.money.enabled, - currency_id=settings.money.currency_id, - display_name=settings.money.display_name,code=settings.money.code, - scale=settings.money.scale,transfer_fee=settings.money.transfer_fee, - allow_overdraft=settings.money.allow_overdraft) - except (Failure,ValueError,KeyError,OSError) as exc: - failed('configuration',getattr(exc,'code','invalid_configuration')) + success( + 'money_config', + enabled=settings.money.enabled, + currency_id=settings.money.currency_id, + display_name=settings.money.display_name, + code=settings.money.code, + scale=settings.money.scale, + transfer_fee=settings.money.transfer_fee, + allow_overdraft=settings.money.allow_overdraft, + ) + except (Failure, ValueError, KeyError, OSError) as exc: + failed('configuration', getattr(exc, 'code', 'invalid_configuration')) from msg.admin.config_check import configuration_doctor - return {'ok':False,'root_id':ROOT_SUBJECT,'checks':checks, - 'configuration_fields':configuration_doctor(checks)} + + return { + 'ok': False, + 'root_id': ROOT_SUBJECT, + 'checks': checks, + 'configuration_fields': configuration_doctor(checks), + } # This read-only phase checks installed contracts; real behavior belongs to # the separate isolated selftest vectors below. - for feature,owner,operations in ( - ('content_editing','file',('file.create','file.patch','file.read')), - ('private_dm','communication',('communication.dm_request','communication.dm_accept', - 'communication.dm_send','communication.dm_list')), - ('transfer','transfer',('transfer.open','transfer.part_put','transfer.part_get', - 'transfer.status','transfer.seal'))): + for feature, owner, operations in ( + ('content_editing', 'file', ('file.create', 'file.patch', 'file.read')), + ( + 'private_dm', + 'communication', + ( + 'communication.dm_request', + 'communication.dm_accept', + 'communication.dm_send', + 'communication.dm_list', + ), + ), + ( + 'transfer', + 'transfer', + ( + 'transfer.open', + 'transfer.part_put', + 'transfer.part_get', + 'transfer.status', + 'transfer.seal', + ), + ), + ): if owner not in app.registry._plugins: - checks[feature]={'ok':True,'status':'disabled'} + checks[feature] = {'ok': True, 'status': 'disabled'} continue try: for name in operations: - operation=app.registry.operation(name) + operation = app.registry.operation(name) app.registry.schema(operation.input_schema) app.registry.schema(operation.output_schema) - success(feature,scope='registered_contracts',operations=list(operations)) - except Failure as exc:failed(feature,exc.code) - if sys.version_info[:2]>=(3,15):success('python',version='.'.join(map(str,sys.version_info[:3]))) - else:failed('python','python_315_required',actual='.'.join(map(str,sys.version_info[:3]))) - missing=[name for name in ('cryptography','starlette','uvicorn','httpx','jsonschema','aiohttp','dns','graphql','psycopg','valkey') if find_spec(name) is None] - if not missing:success('dependencies') - else:failed('dependencies','dependency_unavailable',missing=missing) - if shutil.which('git'):success('git') - else:failed('git','git_missing') + success(feature, scope='registered_contracts', operations=list(operations)) + except Failure as exc: + failed(feature, exc.code) + # Doctor must diagnose an unsupported interpreter rather than report success. + minimum_python = (3, 15) + if sys.version_info[:2] >= minimum_python: + success('python', version='.'.join(map(str, sys.version_info[:3]))) + else: + failed('python', 'python_315_required', actual='.'.join(map(str, sys.version_info[:3]))) + missing = [ + name + for name in ( + 'cryptography', + 'starlette', + 'uvicorn', + 'httpx', + 'jsonschema', + 'aiohttp', + 'dns', + 'graphql', + 'psycopg', + 'valkey', + ) + if find_spec(name) is None + ] + if not missing: + success('dependencies') + else: + failed('dependencies', 'dependency_unavailable', missing=missing) + if shutil.which('git'): + success('git') + else: + failed('git', 'git_missing') if 'extensions' in settings.server.plugins: from msg.admin.tool_check import inspect_tool_sandbox - try:success('tool_sandbox',**await inspect_tool_sandbox(app)) - except (Failure,OSError) as exc: - failed('tool_sandbox',getattr(exc,'code','tool_isolation_failed')) + + try: + success('tool_sandbox', **await inspect_tool_sandbox(app)) + except (Failure, OSError) as exc: + failed('tool_sandbox', getattr(exc, 'code', 'tool_isolation_failed')) else: - checks['tool_sandbox']={'ok':True,'status':'disabled'} + checks['tool_sandbox'] = {'ok': True, 'status': 'disabled'} try: - trust=loads(settings.trust_file.read_bytes()) - root=decode(Certificate,trust['certificate']) - validator=CertificateValidator(app.registry,root,unb64(trust['public_key']),settings.service_url, - clock or (lambda:datetime.now(UTC))) + trust = loads(settings.trust_file.read_bytes()) + root = decode(Certificate, trust['certificate']) + validator = CertificateValidator( + app.registry, + root, + unb64(trust['public_key']), + settings.service_url, + clock or (lambda: datetime.now(UTC)), + ) + async def inspect(tx): - require(tx.one('SELECT version FROM schema_version')[0]==1,'schema_version_unknown') + require(tx.one('SELECT version FROM schema_version')[0] == 1, 'schema_version_unknown') success('storage') try: from msg.admin.custodial_check import inspect_custodial_history + success('custodial_history', **await inspect_custodial_history(tx)) - except Failure as exc:failed('custodial_history',exc.code) + except Failure as exc: + failed('custodial_history', exc.code) if 'orders' in settings.server.plugins: try: from msg.admin.market_check import inspect_market - success('market',**(await inspect_market(tx))) - except (Failure,KeyError,ValueError,psycopg.Error) as exc: - failed('market',getattr(exc,'code','market_inspection_failed')) + + success('market', **(await inspect_market(tx))) + except (Failure, KeyError, ValueError, psycopg.Error) as exc: + failed('market', getattr(exc, 'code', 'market_inspection_failed')) else: - checks['market']={'ok':True,'status':'disabled'} + checks['market'] = {'ok': True, 'status': 'disabled'} try: from msg.admin.market_check import inspect_bounty_contracts, inspect_clearing - clearing = inspect_clearing(app,tx) - await inspect_bounty_contracts(app,tx) - success('market_clearing',**clearing) - except (Failure,OSError,ValueError,KeyError,psycopg.Error) as exc: - failed('market_clearing',getattr(exc,'code','market_inspection_failed')) + + clearing = inspect_clearing(app, tx) + await inspect_bounty_contracts(app, tx) + success('market_clearing', **clearing) + except (Failure, OSError, ValueError, KeyError, psycopg.Error) as exc: + failed('market_clearing', getattr(exc, 'code', 'market_inspection_failed')) try: from msg.core.requests import SECRET_DELIVERY_MIN_VERSION - columns={row[0] for row in tx.rows("SELECT column_name FROM information_schema.columns " - "WHERE table_schema='public' AND table_name='token_deliveries'")} - require({'credential_id','subject','request_id','request_digest','recovery_verifier', - 'recovery_expires_at','claimed_at','consumed_at'} <= columns, - 'credential_delivery_schema_missing') - for operation,version in SECRET_DELIVERY_MIN_VERSION.items(): - require(app.registry.operation(operation,version).effect=='transaction', - 'credential_delivery_contract_invalid') - success('credential_delivery',recovery_window_seconds=settings.credential_delivery_recovery_window, - release='once',secret_url=False,versions=SECRET_DELIVERY_MIN_VERSION) - except Failure as exc:failed('credential_delivery',exc.code) + + columns = { + row[0] + for row in tx.rows( + 'SELECT column_name FROM information_schema.columns ' + "WHERE table_schema='public' AND table_name='token_deliveries'" + ) + } + require( + { + 'credential_id', + 'subject', + 'request_id', + 'request_digest', + 'recovery_verifier', + 'recovery_expires_at', + 'claimed_at', + 'consumed_at', + } + <= columns, + 'credential_delivery_schema_missing', + ) + for operation, version in SECRET_DELIVERY_MIN_VERSION.items(): + require( + app.registry.operation(operation, version).effect == 'transaction', + 'credential_delivery_contract_invalid', + ) + success( + 'credential_delivery', + recovery_window_seconds=settings.credential_delivery_recovery_window, + release='once', + secret_url=False, + versions=SECRET_DELIVERY_MIN_VERSION, + ) + except Failure as exc: + failed('credential_delivery', exc.code) if 'achievements' in settings.server.plugins: try: from msg.admin.honor_check import inspect_honors - success('honors', **await inspect_honors(app,tx)) - except (Failure,OSError,ValueError,KeyError,psycopg.Error) as exc: - failed('honors',getattr(exc,'code','honor_inspection_failed')) + + success('honors', **await inspect_honors(app, tx)) + except (Failure, OSError, ValueError, KeyError, psycopg.Error) as exc: + failed('honors', getattr(exc, 'code', 'honor_inspection_failed')) else: - checks['honors']={'ok':True,'status':'disabled'} + checks['honors'] = {'ok': True, 'status': 'disabled'} if 'communication' in settings.server.plugins: try: - spec=app.registry.operation('communication.following') - require(spec.effect=='read','following_contract_invalid') - columns={row[0] for row in tx.rows("SELECT column_name FROM information_schema.columns " - "WHERE table_schema='public' AND table_name='watches'")} - require({'subject','resource'}<=columns,'following_schema_missing') - success('following',default='empty',read_only=True,max_page_size=100) - except Failure as exc:failed('following',exc.code) + spec = app.registry.operation('communication.following') + require(spec.effect == 'read', 'following_contract_invalid') + columns = { + row[0] + for row in tx.rows( + 'SELECT column_name FROM information_schema.columns ' + "WHERE table_schema='public' AND table_name='watches'" + ) + } + require({'subject', 'resource'} <= columns, 'following_schema_missing') + success('following', default='empty', read_only=True, max_page_size=100) + except Failure as exc: + failed('following', exc.code) try: - for kind in ('request','offer','checkpoint','proposal'): - require(app.registry.operation('communication.'+kind+'_create').effect=='transaction', - 'collaboration_contract_invalid') - require(app.registry.operation('communication.'+kind+'_get').effect=='read' and - app.registry.operation('communication.'+kind+'_list').effect=='read', - 'collaboration_contract_invalid') - success('collaboration',default='empty',storage='resource_revision', - kinds=['request','offer','checkpoint','proposal']) - except Failure as exc:failed('collaboration',exc.code) + for kind in ('request', 'offer', 'checkpoint', 'proposal'): + require( + app.registry.operation('communication.' + kind + '_create').effect + == 'transaction', + 'collaboration_contract_invalid', + ) + require( + app.registry.operation('communication.' + kind + '_get').effect + == 'read' + and app.registry.operation('communication.' + kind + '_list').effect + == 'read', + 'collaboration_contract_invalid', + ) + success( + 'collaboration', + default='empty', + storage='resource_revision', + kinds=['request', 'offer', 'checkpoint', 'proposal'], + ) + except Failure as exc: + failed('collaboration', exc.code) else: - checks['following']={'ok':True,'status':'disabled'} - checks['collaboration']={'ok':True,'status':'disabled'} + checks['following'] = {'ok': True, 'status': 'disabled'} + checks['collaboration'] = {'ok': True, 'status': 'disabled'} try: - await validator.validate(root.resource_id,tx) - require((await tx.subject(ROOT_SUBJECT)).local_only,'root_policy_corrupt') - success('root_trust',fingerprint=digest(unb64(trust['public_key']))) - except Failure as exc:failed('root_trust',exc.code) + await validator.validate(root.resource_id, tx) + require((await tx.subject(ROOT_SUBJECT)).local_only, 'root_policy_corrupt') + success('root_trust', fingerprint=digest(unb64(trust['public_key']))) + except Failure as exc: + failed('root_trust', exc.code) try: - online=tx.setting('online_ca_certificate') - require(online is not None,'issuer_not_ready') - online_certificate=await validator.validate(online,tx) + online = tx.setting('online_ca_certificate') + require(online is not None, 'issuer_not_ready') + online_certificate = await validator.validate(online, tx) from msg.security.crypto import Ed25519Signer - online_key=Ed25519Signer.from_bytes((settings.service_keys/'online.key').read_bytes()) - require(online_certificate.key_id==online_key.key_id,'issuer_key_mismatch') + + online_key = Ed25519Signer.from_bytes( + (settings.service_keys / 'online.key').read_bytes() + ) + require(online_certificate.key_id == online_key.key_id, 'issuer_key_mismatch') success('online_ca') - except (Failure,OSError,ValueError) as exc:failed('online_ca',getattr(exc,'code','issuer_key_missing')) + except (Failure, OSError, ValueError) as exc: + failed('online_ca', getattr(exc, 'code', 'issuer_key_missing')) try: - online_id=tx.setting('online_ca_certificate') - require(online_id is not None,'issuer_not_ready') - online_certificate=await tx.certificate(online_id) - gaps=await authority_snapshot_drift(app,root,online_certificate,tx) + online_id = tx.setting('online_ca_certificate') + require(online_id is not None, 'issuer_not_ready') + online_certificate = await tx.certificate(online_id) + gaps = await authority_snapshot_drift(app, root, online_certificate, tx) if any(gaps.values()): - failed('authority_snapshot','signed_authority_outdated',missing=gaps, - action='No automatic authority expansion. Preserve this chain and keep the new ' - 'operation unavailable, or explicitly rotate the root and reissue ' - 'certificates; rotation invalidates old chains.') + failed( + 'authority_snapshot', + 'signed_authority_outdated', + missing=gaps, + action='No automatic authority expansion. Preserve this chain and keep the new ' + 'operation unavailable, or explicitly rotate the root and reissue ' + 'certificates; rotation invalidates old chains.', + ) else: success('authority_snapshot') except Failure as exc: - failed('authority_snapshot',exc.code) - drift=[] + failed('authority_snapshot', exc.code) + drift = [] for expected in manifest()['resources']: try: - resource=await tx.resource(expected['id']) - for field in ('type','parent','owner','group','mode','name'): - desired=int(expected[field],8) if field=='mode' else expected[field] - if getattr(resource,field)!=desired:drift.append({'id':resource.id,'field':field}) - except Failure:drift.append({'id':expected['id'],'field':'missing'}) - if drift:failed('bootstrap','bootstrap_drift',drift=drift) - else:success('bootstrap') - previous=None - for raw, in tx.execute('SELECT body FROM audit ORDER BY seq'): - audit=loads(raw) - entry_digest=audit.pop('entry_digest') - require(audit.get('previous_digest')==previous and digest(audit)==entry_digest,'audit_chain_mismatch') - previous=entry_digest - success('audit',head=previous) + resource = await tx.resource(expected['id']) + for field in ('type', 'parent', 'owner', 'group', 'mode', 'name'): + desired = int(expected[field], 8) if field == 'mode' else expected[field] + if getattr(resource, field) != desired: + drift.append({'id': resource.id, 'field': field}) + except Failure: + drift.append({'id': expected['id'], 'field': 'missing'}) + if drift: + failed('bootstrap', 'bootstrap_drift', drift=drift) + else: + success('bootstrap') + previous = None + for (raw,) in tx.execute('SELECT body FROM audit ORDER BY seq'): + audit = loads(raw) + entry_digest = audit.pop('entry_digest') + require( + audit.get('previous_digest') == previous and digest(audit) == entry_digest, + 'audit_chain_mismatch', + ) + previous = entry_digest + success('audit', head=previous) + await ReadOnlyStore(settings.server.postgres_dsn).inspect(inspect) - except (Failure,OSError,ValueError,KeyError,psycopg.Error) as exc: - failed('inspection',getattr(exc,'code','inspection_failed')) + except (Failure, OSError, ValueError, KeyError, psycopg.Error) as exc: + failed('inspection', getattr(exc, 'code', 'inspection_failed')) if settings.server.valkey_url: try: from msg.storage.valkey_bus import ValkeyOutboxSignal + await asyncio.to_thread(ValkeyOutboxSignal(settings.server.valkey_url).client.ping) success('valkey') except Exception: - failed('valkey','valkey_unavailable') + failed('valkey', 'valkey_unavailable') else: - checks['valkey']={'ok':True,'status':'disabled'} - content=settings.server.content_dir - if (content/'private.git'/'HEAD').is_file():success('content_layout') - else:failed('content_layout','content_missing') + checks['valkey'] = {'ok': True, 'status': 'disabled'} + content = settings.server.content_dir + if (content / 'private.git' / 'HEAD').is_file(): + success('content_layout') + else: + failed('content_layout', 'content_missing') if settings.server.staging_dir.exists(): - disk=shutil.disk_usage(settings.server.staging_dir) - success('capacity',free_bytes=disk.free) + disk = shutil.disk_usage(settings.server.staging_dir) + success('capacity', free_bytes=disk.free) from msg.admin.root import root_envelope - protected=root_envelope(path).parent - if protected==path/'root': - warnings.append({'code':'legacy_root_storage','effect':'migrate_to_var_lib_msgd_root'}) + + protected = root_envelope(path).parent + if protected == path / 'root': + warnings.append({'code': 'legacy_root_storage', 'effect': 'migrate_to_var_lib_msgd_root'}) try: - require(not ((path/'root'/'key.json').exists() and - (settings.root_private_dir/'key.json').exists()), - 'duplicate_root_material') - mode=stat.S_IMODE(protected.stat().st_mode) - require(mode==0o700 and protected.stat().st_uid==0,'unsafe_root_permissions') - secret=protected/'key.json' + require( + not ( + (path / 'root' / 'key.json').exists() + and (settings.root_private_dir / 'key.json').exists() + ), + 'duplicate_root_material', + ) + mode = stat.S_IMODE(protected.stat().st_mode) + require(mode == 0o700 and protected.stat().st_uid == 0, 'unsafe_root_permissions') + secret = protected / 'key.json' try: - require(stat.S_IMODE(secret.stat().st_mode)==0o600 and secret.stat().st_uid==0,'unsafe_root_permissions') + require( + stat.S_IMODE(secret.stat().st_mode) == 0o600 and secret.stat().st_uid == 0, + 'unsafe_root_permissions', + ) except PermissionError: # The network service being unable to inspect a root-owned 0700 # directory is the intended deployment boundary. pass success('root_private_boundary') - except (Failure,OSError) as exc:failed('root_private_boundary',getattr(exc,'code','root_material_missing')) - try:success('recovery_checkpoint',**inspect_recovery_checkpoint()) - except (Failure,OSError,ValueError,KeyError,TypeError,AttributeError) as exc: - failed('recovery_checkpoint',getattr(exc,'code','recovery_checkpoint_schema_invalid')) - for name,plugin,inspect_feature in (('lexical_search','discovery',inspect_lexical_search), - ('hosting','extensions',inspect_hosting)): + except (Failure, OSError) as exc: + failed('root_private_boundary', getattr(exc, 'code', 'root_material_missing')) + try: + success('recovery_checkpoint', **inspect_recovery_checkpoint()) + except (Failure, OSError, ValueError, KeyError, TypeError, AttributeError) as exc: + failed('recovery_checkpoint', getattr(exc, 'code', 'recovery_checkpoint_schema_invalid')) + for name, plugin, inspect_feature in ( + ('lexical_search', 'discovery', inspect_lexical_search), + ('hosting', 'extensions', inspect_hosting), + ): if plugin not in settings.server.plugins: - checks[name]={'ok':True,'status':'disabled'} + checks[name] = {'ok': True, 'status': 'disabled'} continue - try:success(name,**inspect_feature(app)) - except Failure as exc:failed(name,exc.code) + try: + success(name, **inspect_feature(app)) + except Failure as exc: + failed(name, exc.code) from msg.admin.config_check import configuration_doctor - features=feature_results(feature_manifest(),checks,'doctor_check', - enabled_plugins=tuple(app.registry._plugins)) - return {'ok':all(c['ok'] for c in checks.values()) and - all(row['status']!='fail' for row in features.values()), - 'root_id':ROOT_SUBJECT,'checks':checks,'features':features,'warnings':warnings, - 'configuration_fields':configuration_doctor(checks)} + + features = feature_results( + feature_manifest(), checks, 'doctor_check', enabled_plugins=tuple(app.registry._plugins) + ) + return { + 'ok': all(c['ok'] for c in checks.values()) + and all(row['status'] != 'fail' for row in features.values()), + 'root_id': ROOT_SUBJECT, + 'checks': checks, + 'features': features, + 'warnings': warnings, + 'configuration_fields': configuration_doctor(checks), + } -async def _selftest_ca_chain(app,root,call,register,now): +async def _selftest_ca_chain(app, root, call, register, now): """Publish a real, narrowly scoped CA chain in the disposable test instance.""" from dataclasses import replace + + from msg.admin.root import _approve_csr from msg.core.models import CertificateRequest, IssuancePolicy, Scope, Signature from msg.plugins.common import new_id from msg.security.capabilities import grant_for from msg.security.certificates import csr_body, sign_certificate - from msg.admin.root import _approve_csr - scope=Scope(resource_id=app.namespace_root,descendants=True) - issue=grant_for(app.registry.capability('cert.issue'),scope=scope, - operations=('cert.publish@1',)) - ca_issue=grant_for(app.registry.capability('cert.ca.issue'),scope=scope, - operations=('cert.publish@1',)) - read=grant_for(app.registry.capability('discovery.basic'),scope=scope, - operations=('discovery.get@1',)) - bounded_tool=replace(grant_for(app.registry.capability('tool.use'), - scope=Scope(resource_id='t_tools'),operations=('tool.run@1',)), - constraints={'hosts':['example.org']}) - ca_grants=(issue,ca_issue) - issuable=(issue,ca_issue,read,bounded_tool) - - async def request_certificate(key,uid,issuer,kind,grants,policy=None,ttl=600): - csr=CertificateRequest(resource_id='pending',applicant=uid,subject_id=uid, - requested_issuer=issuer,public_key=key.public_key,kind=kind,grants=tuple(grants), - issuance=policy,requested_ttl_seconds=ttl,target_service=app.settings.service_url, - delegation_depth=0,authority_sources=(),request_digest='', - possession_proof=Signature(key_id=key.key_id,algorithm='ed25519',value=b'')) - proof=key.sign(canonical(csr_body(csr)),purpose='csr') - result=await call('cert.request',{'requested_issuer':issuer,'public_key':b64(key.public_key), - 'kind':kind,'grants':wire(tuple(grants)),'issuance':wire(policy), - 'requested_ttl_seconds':ttl,'possession_proof':wire(proof)},key,uid) - require(result.status=='ok',result.error.code if result.error else 'selftest_csr_failed') + scope = Scope(resource_id=app.namespace_root, descendants=True) + issue = grant_for( + app.registry.capability('cert.issue'), scope=scope, operations=('cert.publish@1',) + ) + ca_issue = grant_for( + app.registry.capability('cert.ca.issue'), scope=scope, operations=('cert.publish@1',) + ) + read = grant_for( + app.registry.capability('discovery.basic'), scope=scope, operations=('discovery.get@1',) + ) + bounded_tool = replace( + grant_for( + app.registry.capability('tool.use'), + scope=Scope(resource_id='t_tools'), + operations=('tool.run@1',), + ), + constraints={'hosts': ['example.org']}, + ) + ca_grants = (issue, ca_issue) + issuable = (issue, ca_issue, read, bounded_tool) + + async def request_certificate(key, uid, issuer, kind, grants, policy=None, ttl=600): + csr = CertificateRequest( + resource_id='pending', + applicant=uid, + subject_id=uid, + requested_issuer=issuer, + public_key=key.public_key, + kind=kind, + grants=tuple(grants), + issuance=policy, + requested_ttl_seconds=ttl, + target_service=app.settings.service_url, + delegation_depth=0, + authority_sources=(), + request_digest='', + possession_proof=Signature(key_id=key.key_id, algorithm='ed25519', value=b''), + ) + proof = key.sign(canonical(csr_body(csr)), purpose='csr') + result = await call( + 'cert.request', + { + 'requested_issuer': issuer, + 'public_key': b64(key.public_key), + 'kind': kind, + 'grants': wire(tuple(grants)), + 'issuance': wire(policy), + 'requested_ttl_seconds': ttl, + 'possession_proof': wire(proof), + }, + key, + uid, + ) + require(result.status == 'ok', result.error.code if result.error else 'selftest_csr_failed') return result - async def publish(parent,parent_key,parent_uid,key,uid,kind,grants,policy=None, - *,ttl=600,target_service=None): - requested=await request_certificate(key,uid,parent_uid,kind,grants,policy,ttl) - cert=Certificate(resource_id=new_id('cert'),serial=new_id('serial'),subject_id=uid, - key_id=key.key_id,issuer_id=parent_uid,parent_certificate_id=parent.resource_id, - authority_sources=(),kind=kind,grants=tuple(grants),not_before=now, - expires_at=now+timedelta(seconds=ttl), + async def publish( + parent, + parent_key, + parent_uid, + key, + uid, + kind, + grants, + policy=None, + *, + ttl=600, + target_service=None, + ): + requested = await request_certificate(key, uid, parent_uid, kind, grants, policy, ttl) + cert = Certificate( + resource_id=new_id('cert'), + serial=new_id('serial'), + subject_id=uid, + key_id=key.key_id, + issuer_id=parent_uid, + parent_certificate_id=parent.resource_id, + authority_sources=(), + kind=kind, + grants=tuple(grants), + not_before=now, + expires_at=now + timedelta(seconds=ttl), target_service=target_service or app.settings.service_url, - delegation_depth=0,issuance=policy, - signature=Signature(key_id=parent_key.key_id,algorithm='ed25519',value=b'')) - cert=sign_certificate(cert,parent_key) - result=await call('cert.publish',{'csr_id':requested.data['csr_id'], - 'certificate':wire(cert)},parent_key,parent_uid,(parent.resource_id,)) - return result,cert,requested - - l1_key,l1_uid=await register('test-l1') - l1_policy=IssuancePolicy(issue_grants=issuable,max_cert_ttl_seconds=600, - max_child_ca_depth=2,max_delegation_depth=0) - l1_request=await request_certificate(l1_key,l1_uid,ROOT_SUBJECT,'ca',ca_grants,l1_policy) - l1=await _approve_csr(app,l1_request.data['csr_id'],root, - expected_digest=l1_request.data['request_digest'],operator='isolated-selftest') - l2_key,l2_uid=await register('test-l2') - l2_policy=IssuancePolicy(issue_grants=issuable,max_cert_ttl_seconds=600, - max_child_ca_depth=1,max_delegation_depth=0) - l2_result,l2,_=await publish(l1,l1_key,l1_uid,l2_key,l2_uid,'ca',ca_grants,l2_policy) - require(l2_result.status=='ok',l2_result.error.code if l2_result.error else 'selftest_l2_failed') - l3_key,l3_uid=await register('test-l3') - l3_policy=IssuancePolicy(issue_grants=issuable,max_cert_ttl_seconds=600, - max_child_ca_depth=0,max_delegation_depth=0) - l3_result,l3,_=await publish(l2,l2_key,l2_uid,l3_key,l3_uid,'ca',ca_grants,l3_policy) - require(l3_result.status=='ok',l3_result.error.code if l3_result.error else 'selftest_l3_failed') - leaf_key,leaf_uid=await register('test-leaf') - leaf_result,leaf,_=await publish(l3,l3_key,l3_uid,leaf_key,leaf_uid,'capability',(read,)) - require(leaf_result.status=='ok',leaf_result.error.code if leaf_result.error else 'selftest_leaf_failed') + delegation_depth=0, + issuance=policy, + signature=Signature(key_id=parent_key.key_id, algorithm='ed25519', value=b''), + ) + cert = sign_certificate(cert, parent_key) + result = await call( + 'cert.publish', + {'csr_id': requested.data['csr_id'], 'certificate': wire(cert)}, + parent_key, + parent_uid, + (parent.resource_id,), + ) + return result, cert, requested + + l1_key, l1_uid = await register('test-l1') + l1_policy = IssuancePolicy( + issue_grants=issuable, + max_cert_ttl_seconds=600, + max_child_ca_depth=2, + max_delegation_depth=0, + ) + l1_request = await request_certificate(l1_key, l1_uid, ROOT_SUBJECT, 'ca', ca_grants, l1_policy) + l1 = await _approve_csr( + app, + l1_request.data['csr_id'], + root, + expected_digest=l1_request.data['request_digest'], + operator='isolated-selftest', + ) + l2_key, l2_uid = await register('test-l2') + l2_policy = IssuancePolicy( + issue_grants=issuable, + max_cert_ttl_seconds=600, + max_child_ca_depth=1, + max_delegation_depth=0, + ) + l2_result, l2, _ = await publish(l1, l1_key, l1_uid, l2_key, l2_uid, 'ca', ca_grants, l2_policy) + require( + l2_result.status == 'ok', l2_result.error.code if l2_result.error else 'selftest_l2_failed' + ) + l3_key, l3_uid = await register('test-l3') + l3_policy = IssuancePolicy( + issue_grants=issuable, + max_cert_ttl_seconds=600, + max_child_ca_depth=0, + max_delegation_depth=0, + ) + l3_result, l3, _ = await publish(l2, l2_key, l2_uid, l3_key, l3_uid, 'ca', ca_grants, l3_policy) + require( + l3_result.status == 'ok', l3_result.error.code if l3_result.error else 'selftest_l3_failed' + ) + leaf_key, leaf_uid = await register('test-leaf') + leaf_result, leaf, _ = await publish( + l3, l3_key, l3_uid, leaf_key, leaf_uid, 'capability', (read,) + ) + require( + leaf_result.status == 'ok', + leaf_result.error.code if leaf_result.error else 'selftest_leaf_failed', + ) async with app.metadata.transaction(write=False) as tx: - for cert in (l1,l2,l3,leaf): - await app.certificates.validate(cert.resource_id,tx) - paths=[await tx.path(cert.resource_id) for cert in (l1,l2,l3,leaf)] - scope_ok=True - for cert in (l1,l2,l3,leaf): - for grant in (*cert.grants,*(cert.issuance.issue_grants if cert.issuance else ())): - if grant.scope.resource_id==app.namespace_root: + for cert in (l1, l2, l3, leaf): + await app.certificates.validate(cert.resource_id, tx) + paths = [await tx.path(cert.resource_id) for cert in (l1, l2, l3, leaf)] + scope_ok = True + for cert in (l1, l2, l3, leaf): + for grant in (*cert.grants, *(cert.issuance.issue_grants if cert.issuance else ())): + if grant.scope.resource_id == app.namespace_root: continue - if app.namespace_root not in {item.id for item in await tx.ancestors(grant.scope.resource_id)}: - scope_ok=False - test_path='/_test/'+app.selftest_run_id - valid=all(path.startswith(test_path+'/') for path in paths) and scope_ok + if app.namespace_root not in { + item.id for item in await tx.ancestors(grant.scope.resource_id) + }: + scope_ok = False + test_path = '/_test/' + app.selftest_run_id + valid = all(path.startswith(test_path + '/') for path in paths) and scope_ok - l4_key,l4_uid=await register('test-l4') - l4_result,l4,l4_request=await publish(l3,l3_key,l3_uid,l4_key,l4_uid,'ca',ca_grants,l3_policy) + l4_key, l4_uid = await register('test-l4') + l4_result, l4, l4_request = await publish( + l3, l3_key, l3_uid, l4_key, l4_uid, 'ca', ca_grants, l3_policy + ) async with app.metadata.transaction(write=False) as tx: - pending=(await tx.csr_state(l4_request.data['csr_id'])).status=='pending' - absent=tx.one('SELECT 1 FROM certificates WHERE id=?',(l4.resource_id,)) is None - denied=(l4_result.error is not None and l4_result.error.code=='ca_depth_exceeded' - and pending and absent) - - out_of_scope=grant_for(app.registry.capability('discovery.basic'), - scope=Scope(resource_id='r_root',descendants=True),operations=('discovery.get@1',)) - bad_scope_key,bad_scope_uid=await register('test-bad-scope') - bad_scope,bad_scope_cert,bad_scope_request=await publish( - l1,l1_key,l1_uid,bad_scope_key,bad_scope_uid,'capability',(out_of_scope,)) + pending = (await tx.csr_state(l4_request.data['csr_id'])).status == 'pending' + absent = tx.one('SELECT 1 FROM certificates WHERE id=?', (l4.resource_id,)) is None + denied = ( + l4_result.error is not None + and l4_result.error.code == 'ca_depth_exceeded' + and pending + and absent + ) + + out_of_scope = grant_for( + app.registry.capability('discovery.basic'), + scope=Scope(resource_id='r_root', descendants=True), + operations=('discovery.get@1',), + ) + bad_scope_key, bad_scope_uid = await register('test-bad-scope') + bad_scope, bad_scope_cert, bad_scope_request = await publish( + l1, l1_key, l1_uid, bad_scope_key, bad_scope_uid, 'capability', (out_of_scope,) + ) async with app.metadata.transaction(write=False) as tx: - scope_pending=(await tx.csr_state(bad_scope_request.data['csr_id'])).status=='pending' - scope_absent=tx.one('SELECT 1 FROM certificates WHERE id=?',(bad_scope_cert.resource_id,)) is None - denied_scope=(bad_scope.error is not None and bad_scope.error.code=='issuance_scope_exceeded' - and scope_pending and scope_absent) - - broad_read=grant_for(app.registry.capability('discovery.basic'),scope=scope, - operations=('discovery.get@1','discovery.list@1')) - bad_ops_key,bad_ops_uid=await register('test-bad-ops') - bad_ops,bad_ops_cert,bad_ops_request=await publish( - l1,l1_key,l1_uid,bad_ops_key,bad_ops_uid,'capability',(broad_read,)) + scope_pending = (await tx.csr_state(bad_scope_request.data['csr_id'])).status == 'pending' + scope_absent = ( + tx.one('SELECT 1 FROM certificates WHERE id=?', (bad_scope_cert.resource_id,)) is None + ) + denied_scope = ( + bad_scope.error is not None + and bad_scope.error.code == 'issuance_scope_exceeded' + and scope_pending + and scope_absent + ) + + broad_read = grant_for( + app.registry.capability('discovery.basic'), + scope=scope, + operations=('discovery.get@1', 'discovery.list@1'), + ) + bad_ops_key, bad_ops_uid = await register('test-bad-ops') + bad_ops, bad_ops_cert, bad_ops_request = await publish( + l1, l1_key, l1_uid, bad_ops_key, bad_ops_uid, 'capability', (broad_read,) + ) async with app.metadata.transaction(write=False) as tx: - ops_pending=(await tx.csr_state(bad_ops_request.data['csr_id'])).status=='pending' - ops_absent=tx.one('SELECT 1 FROM certificates WHERE id=?',(bad_ops_cert.resource_id,)) is None - denied_ops=(bad_ops.error is not None and bad_ops.error.code=='issuance_scope_exceeded' - and ops_pending and ops_absent) - - broad_tool=replace(bounded_tool,constraints={'hosts':['example.org','other.example']}) - bad_constraints_key,bad_constraints_uid=await register('test-bad-constraints') - bad_constraints,bad_constraints_cert,bad_constraints_request=await publish( - l1,l1_key,l1_uid,bad_constraints_key,bad_constraints_uid,'capability',(broad_tool,)) + ops_pending = (await tx.csr_state(bad_ops_request.data['csr_id'])).status == 'pending' + ops_absent = ( + tx.one('SELECT 1 FROM certificates WHERE id=?', (bad_ops_cert.resource_id,)) is None + ) + denied_ops = ( + bad_ops.error is not None + and bad_ops.error.code == 'issuance_scope_exceeded' + and ops_pending + and ops_absent + ) + + broad_tool = replace(bounded_tool, constraints={'hosts': ['example.org', 'other.example']}) + bad_constraints_key, bad_constraints_uid = await register('test-bad-constraints') + bad_constraints, bad_constraints_cert, bad_constraints_request = await publish( + l1, l1_key, l1_uid, bad_constraints_key, bad_constraints_uid, 'capability', (broad_tool,) + ) async with app.metadata.transaction(write=False) as tx: - constraints_pending=(await tx.csr_state(bad_constraints_request.data['csr_id'])).status=='pending' - constraints_absent=tx.one('SELECT 1 FROM certificates WHERE id=?', - (bad_constraints_cert.resource_id,)) is None - denied_constraints=(bad_constraints.error is not None and - bad_constraints.error.code=='issuance_scope_exceeded' and - constraints_pending and constraints_absent) - - bad_ttl_key,bad_ttl_uid=await register('test-bad-ttl') - bad_ttl,bad_ttl_cert,bad_ttl_request=await publish( - l1,l1_key,l1_uid,bad_ttl_key,bad_ttl_uid,'capability',(read,),ttl=601) + constraints_pending = ( + await tx.csr_state(bad_constraints_request.data['csr_id']) + ).status == 'pending' + constraints_absent = ( + tx.one('SELECT 1 FROM certificates WHERE id=?', (bad_constraints_cert.resource_id,)) + is None + ) + denied_constraints = ( + bad_constraints.error is not None + and bad_constraints.error.code == 'issuance_scope_exceeded' + and constraints_pending + and constraints_absent + ) + + bad_ttl_key, bad_ttl_uid = await register('test-bad-ttl') + bad_ttl, bad_ttl_cert, bad_ttl_request = await publish( + l1, l1_key, l1_uid, bad_ttl_key, bad_ttl_uid, 'capability', (read,), ttl=601 + ) async with app.metadata.transaction(write=False) as tx: - ttl_pending=(await tx.csr_state(bad_ttl_request.data['csr_id'])).status=='pending' - ttl_absent=tx.one('SELECT 1 FROM certificates WHERE id=?',(bad_ttl_cert.resource_id,)) is None - denied_ttl=(bad_ttl.error is not None and bad_ttl.error.code in - {'certificate_validity_escalation','certificate_ttl_escalation'} and - ttl_pending and ttl_absent) - - bad_service_key,bad_service_uid=await register('test-bad-service') - bad_service,bad_service_cert,bad_service_request=await publish( - l1,l1_key,l1_uid,bad_service_key,bad_service_uid,'capability',(read,), - target_service='https://wrong-service.invalid') + ttl_pending = (await tx.csr_state(bad_ttl_request.data['csr_id'])).status == 'pending' + ttl_absent = ( + tx.one('SELECT 1 FROM certificates WHERE id=?', (bad_ttl_cert.resource_id,)) is None + ) + denied_ttl = ( + bad_ttl.error is not None + and bad_ttl.error.code in {'certificate_validity_escalation', 'certificate_ttl_escalation'} + and ttl_pending + and ttl_absent + ) + + bad_service_key, bad_service_uid = await register('test-bad-service') + bad_service, bad_service_cert, bad_service_request = await publish( + l1, + l1_key, + l1_uid, + bad_service_key, + bad_service_uid, + 'capability', + (read,), + target_service='https://wrong-service.invalid', + ) async with app.metadata.transaction(write=False) as tx: - service_pending=(await tx.csr_state(bad_service_request.data['csr_id'])).status=='pending' - service_absent=tx.one('SELECT 1 FROM certificates WHERE id=?',(bad_service_cert.resource_id,)) is None - denied_service=(bad_service.error is not None and bad_service.error.code=='csr_certificate_mismatch' - and service_pending and service_absent) - - bad_policy=IssuancePolicy(issue_grants=(*issuable,broad_read),max_cert_ttl_seconds=600, - max_child_ca_depth=1,max_delegation_depth=0) - bad_policy_key,bad_policy_uid=await register('test-bad-issue-grants') - bad_issue_grants,bad_policy_cert,bad_policy_request=await publish( - l1,l1_key,l1_uid,bad_policy_key,bad_policy_uid,'ca',ca_grants,bad_policy) + service_pending = ( + await tx.csr_state(bad_service_request.data['csr_id']) + ).status == 'pending' + service_absent = ( + tx.one('SELECT 1 FROM certificates WHERE id=?', (bad_service_cert.resource_id,)) is None + ) + denied_service = ( + bad_service.error is not None + and bad_service.error.code == 'csr_certificate_mismatch' + and service_pending + and service_absent + ) + + bad_policy = IssuancePolicy( + issue_grants=(*issuable, broad_read), + max_cert_ttl_seconds=600, + max_child_ca_depth=1, + max_delegation_depth=0, + ) + bad_policy_key, bad_policy_uid = await register('test-bad-issue-grants') + bad_issue_grants, bad_policy_cert, bad_policy_request = await publish( + l1, l1_key, l1_uid, bad_policy_key, bad_policy_uid, 'ca', ca_grants, bad_policy + ) async with app.metadata.transaction(write=False) as tx: - policy_pending=(await tx.csr_state(bad_policy_request.data['csr_id'])).status=='pending' - policy_absent=tx.one('SELECT 1 FROM certificates WHERE id=?',(bad_policy_cert.resource_id,)) is None - denied_policy=(bad_issue_grants.error is not None and - bad_issue_grants.error.code=='issuance_scope_exceeded' and - policy_pending and policy_absent) - - child_key,child_uid=await register('test-leaf-child') - leaf_issue,leaf_child,leaf_request=await publish( - leaf,leaf_key,leaf_uid,child_key,child_uid,'capability',(read,)) + policy_pending = (await tx.csr_state(bad_policy_request.data['csr_id'])).status == 'pending' + policy_absent = ( + tx.one('SELECT 1 FROM certificates WHERE id=?', (bad_policy_cert.resource_id,)) is None + ) + denied_policy = ( + bad_issue_grants.error is not None + and bad_issue_grants.error.code == 'issuance_scope_exceeded' + and policy_pending + and policy_absent + ) + + child_key, child_uid = await register('test-leaf-child') + leaf_issue, leaf_child, leaf_request = await publish( + leaf, leaf_key, leaf_uid, child_key, child_uid, 'capability', (read,) + ) async with app.metadata.transaction(write=False) as tx: - leaf_pending=(await tx.csr_state(leaf_request.data['csr_id'])).status=='pending' - leaf_absent=tx.one('SELECT 1 FROM certificates WHERE id=?',(leaf_child.resource_id,)) is None - leaf_denied=(leaf_issue.error is not None and leaf_pending and leaf_absent) + leaf_pending = (await tx.csr_state(leaf_request.data['csr_id'])).status == 'pending' + leaf_absent = ( + tx.one('SELECT 1 FROM certificates WHERE id=?', (leaf_child.resource_id,)) is None + ) + leaf_denied = leaf_issue.error is not None and leaf_pending and leaf_absent - revoked=True - for ancestor in (l1,l2,l3): + revoked = True + for ancestor in (l1, l2, l3): async with app.metadata.transaction(write=True) as tx: - tx.execute('UPDATE certificates SET revoked=1 WHERE id=?',(ancestor.resource_id,),write=True) + tx.execute( + 'UPDATE certificates SET revoked=1 WHERE id=?', (ancestor.resource_id,), write=True + ) async with app.metadata.transaction(write=False) as tx: try: - await app.certificates.validate(leaf.resource_id,tx) + await app.certificates.validate(leaf.resource_id, tx) except Failure as exc: - revoked=revoked and exc.code=='certificate_revoked' + revoked = revoked and exc.code == 'certificate_revoked' else: - revoked=False + revoked = False async with app.metadata.transaction(write=True) as tx: - tx.execute('UPDATE certificates SET revoked=0 WHERE id=?',(ancestor.resource_id,),write=True) - return {'ca_l1_l2_l3_leaf':valid,'ca_l3_cannot_issue_ca':denied, - 'ca_ancestor_revocation':revoked,'ca_scope_expansion_denied':denied_scope, - 'ca_operations_expansion_denied':denied_ops, - 'ca_constraints_expansion_denied':denied_constraints, - 'ca_ttl_expansion_denied':denied_ttl, - 'ca_target_service_change_denied':denied_service, - 'ca_issue_grants_expansion_denied':denied_policy, - 'ca_leaf_cannot_issue':leaf_denied} + tx.execute( + 'UPDATE certificates SET revoked=0 WHERE id=?', (ancestor.resource_id,), write=True + ) + return { + 'ca_l1_l2_l3_leaf': valid, + 'ca_l3_cannot_issue_ca': denied, + 'ca_ancestor_revocation': revoked, + 'ca_scope_expansion_denied': denied_scope, + 'ca_operations_expansion_denied': denied_ops, + 'ca_constraints_expansion_denied': denied_constraints, + 'ca_ttl_expansion_denied': denied_ttl, + 'ca_target_service_change_denied': denied_service, + 'ca_issue_grants_expansion_denied': denied_policy, + 'ca_leaf_cannot_issue': leaf_denied, + } async def selftest(): """No production database, listener, root PIN or live trust state is touched.""" - from dataclasses import replace + from msg.admin import recovery_replay + from msg.admin.config_check import configuration_selftest + from msg.admin.root import _approve_csr, _provision from msg.application import Application + from msg.bootstrap import feature_manifest from msg.config import write_example - from msg.admin.root import _provision,_approve_csr - from msg.security.crypto import Ed25519Signer,subject_id - from msg.security.age_keys import generate_age_key + from msg.core.models import CertificateRequest, Scope, Signature from msg.core.requests import request_for - from msg.core.models import CertificateRequest,Signature,Scope - from msg.security.certificates import csr_body + from msg.security.age_keys import generate_age_key from msg.security.capabilities import grant_for - from msg.bootstrap import feature_manifest - from msg.admin import recovery_replay - from msg.admin.config_check import configuration_selftest - configuration_fields=configuration_selftest() - tool_sandbox_details={'code':'not_run'} - checks={'configuration_loading':all(row['status']=='pass' - for row in configuration_fields.values())} + from msg.security.certificates import csr_body + from msg.security.crypto import Ed25519Signer, subject_id + + configuration_fields = configuration_selftest() + tool_sandbox_details = {'code': 'not_run'} + checks = { + 'configuration_loading': all( + row['status'] == 'pass' for row in configuration_fields.values() + ) + } try: inspect_recovery_checkpoint() - checks['recovery_checkpoint_replay']=recovery_replay.selftest()['status']=='passed' - except (Failure,OSError,ValueError,KeyError,TypeError,AttributeError): - checks['recovery_checkpoint_replay']=False - with tempfile.TemporaryDirectory(prefix='msg-selftest-') as temporary, temporary_postgres() as dsn: - folder=Path(temporary) - now=datetime.now(UTC) - run_id=uuid.uuid4().hex - test_path='/_test/'+run_id - app=Application(write_example(folder/'etc',folder/'data','https://selftest.invalid', - postgres_dsn=dsn),clock=lambda:now,selftest_run_id=run_id) + checks['recovery_checkpoint_replay'] = recovery_replay.selftest()['status'] == 'passed' + except Failure, OSError, ValueError, KeyError, TypeError, AttributeError: + checks['recovery_checkpoint_replay'] = False + with ( + tempfile.TemporaryDirectory(prefix='msg-selftest-') as temporary, + temporary_postgres() as dsn, + ): + folder = Path(temporary) + now = datetime.now(UTC) + run_id = uuid.uuid4().hex + test_path = '/_test/' + run_id + app = Application( + write_example( + folder / 'etc', folder / 'data', 'https://selftest.invalid', postgres_dsn=dsn + ), + clock=lambda: now, + selftest_run_id=run_id, + ) try: - csr,root=await _provision(app,'selftest-'+os.urandom(24).hex()) - await _approve_csr(app,csr,root,expected_digest=None,operator='isolated-selftest') + csr, root = await _provision(app, 'selftest-' + os.urandom(24).hex()) + await _approve_csr(app, csr, root, expected_digest=None, operator='isolated-selftest') async with app.metadata.transaction(write=False) as tx: - namespace=app.namespace_root - require(await tx.path(namespace)==test_path,'selftest_namespace_missing') - root_scoped=True + namespace = app.namespace_root + require(await tx.path(namespace) == test_path, 'selftest_namespace_missing') + root_scoped = True for (resource_id,) in tx.execute('SELECT id FROM resources'): - if resource_id in {'r_root','t_selftest'}: + if resource_id in {'r_root', 't_selftest'}: continue - ancestors=await tx.ancestors(resource_id) - if resource_id!=namespace and namespace not in {item.id for item in ancestors}: - root_scoped=False + ancestors = await tx.ancestors(resource_id) + if resource_id != namespace and namespace not in { + item.id for item in ancestors + }: + root_scoped = False break - checks['isolated_namespace']=root_scoped - async def call(name,args,key=None,subject=None,certs=(),expected=(),request_id=None, - contract_version=1): - packet=request_for(name,args,app.settings.service_url,subject=subject,signer=key, - certificates=certs,expected=expected,expires_at=now+timedelta(seconds=120), - request_id=request_id,contract_version=contract_version) + checks['isolated_namespace'] = root_scoped + + async def call( + name, + args, + key=None, + subject=None, + certs=(), + expected=(), + request_id=None, + contract_version=1, + ): + packet = request_for( + name, + args, + app.settings.service_url, + subject=subject, + signer=key, + certificates=certs, + expected=expected, + expires_at=now + timedelta(seconds=120), + request_id=request_id, + contract_version=contract_version, + ) return await app.executor.execute(packet) + async def register(name): - key=Ed25519Signer.generate();uid=subject_id(key.public_key) - _,recipient=generate_age_key() - result=await call('identity.register',{'handle':name,'public_key':b64(key.public_key), - 'encryption_recipient':recipient},key,uid,contract_version=2) - require(result.status=='ok','selftest_registration_failed') - return key,uid - async def approve(key,uid,grants): - csr=CertificateRequest(resource_id='pending',applicant=uid,subject_id=uid,requested_issuer=ROOT_SUBJECT, - public_key=key.public_key,kind='capability',grants=tuple(grants),issuance=None, - requested_ttl_seconds=600,target_service=app.settings.service_url,delegation_depth=0,authority_sources=(), - request_digest='',possession_proof=Signature(key_id=key.key_id,algorithm='ed25519',value=b'')) - proof=key.sign(canonical(csr_body(csr)),purpose='csr') - result=await call('cert.request',{'requested_issuer':ROOT_SUBJECT,'public_key':b64(key.public_key), - 'kind':'capability','grants':wire(grants),'requested_ttl_seconds':600,'possession_proof':wire(proof)},key,uid) - require(result.status=='ok','selftest_csr_failed') - return await _approve_csr(app,result.data['csr_id'],root,expected_digest=result.data['request_digest'],operator='isolated-selftest') - alice,ua=await register('alice');bob,ub=await register('bob') + key = Ed25519Signer.generate() + uid = subject_id(key.public_key) + _, recipient = generate_age_key() + result = await call( + 'identity.register', + { + 'handle': name, + 'public_key': b64(key.public_key), + 'encryption_recipient': recipient, + }, + key, + uid, + contract_version=2, + ) + require(result.status == 'ok', 'selftest_registration_failed') + return key, uid + + async def approve(key, uid, grants): + csr = CertificateRequest( + resource_id='pending', + applicant=uid, + subject_id=uid, + requested_issuer=ROOT_SUBJECT, + public_key=key.public_key, + kind='capability', + grants=tuple(grants), + issuance=None, + requested_ttl_seconds=600, + target_service=app.settings.service_url, + delegation_depth=0, + authority_sources=(), + request_digest='', + possession_proof=Signature(key_id=key.key_id, algorithm='ed25519', value=b''), + ) + proof = key.sign(canonical(csr_body(csr)), purpose='csr') + result = await call( + 'cert.request', + { + 'requested_issuer': ROOT_SUBJECT, + 'public_key': b64(key.public_key), + 'kind': 'capability', + 'grants': wire(grants), + 'requested_ttl_seconds': 600, + 'possession_proof': wire(proof), + }, + key, + uid, + ) + require(result.status == 'ok', 'selftest_csr_failed') + return await _approve_csr( + app, + result.data['csr_id'], + root, + expected_digest=result.data['request_digest'], + operator='isolated-selftest', + ) + + alice, ua = await register('alice') + bob, ub = await register('bob') from msg.admin.tool_check import check_tool_sandbox + try: - tool_sandbox_details=await check_tool_sandbox(app,call,approve,alice,ua) - checks['tool_sandbox']=True - except (Failure,OSError) as exc: - tool_sandbox_details={'code':getattr(exc,'code','tool_isolation_failed')} - checks['tool_sandbox']=False - post=await call('content.post_create',{'parent':test_path+'/tmp','body':'retained source bytes\r\n'},alice,ua,request_id='same-write') - require(post.status=='ok',post.error.code if post.error else 'selftest_post_failed') + tool_sandbox_details = await check_tool_sandbox(app, call, approve, alice, ua) + checks['tool_sandbox'] = True + except (Failure, OSError) as exc: + tool_sandbox_details = {'code': getattr(exc, 'code', 'tool_isolation_failed')} + checks['tool_sandbox'] = False + post = await call( + 'content.post_create', + {'parent': test_path + '/tmp', 'body': 'retained source bytes\r\n'}, + alice, + ua, + request_id='same-write', + ) + require(post.status == 'ok', post.error.code if post.error else 'selftest_post_failed') # Following reads only existing watches; use a separate fixture so # its explicit permission change cannot alter other lifecycle tests. - empty_following=await call('communication.following',{},bob,ub) - anonymous_following=await call('communication.following',{}) - followed=await call('content.post_create', - {'parent':test_path+'/tmp','body':'isolated Following fixture'},alice,ua) - require(followed.status=='ok','selftest_following_post_failed') - followed_id=followed.resources[0].id - watched=await call('communication.watch',{'id':followed_id},bob,ub) - shown=await call('communication.following',{},bob,ub) - hidden=await call('content.chmod',{'id':followed_id,'mode':'0600'},alice,ua, - expected=((followed_id,followed.data['generation']),)) - redacted=await call('communication.following',{},bob,ub) - checks['following']=(empty_following.status=='ok' and not empty_following.data['items'] and - anonymous_following.error is not None and - anonymous_following.error.code=='authentication_required' and - watched.status=='ok' and shown.status=='ok' and - [item['id'] for item in shown.data['items']]==[followed_id] and - hidden.status=='ok' and redacted.status=='ok' and not redacted.data['items']) - work = await call('communication.request_create', - {'title':'Isolated task','description':'Validate work records','requirements':'Explicit claim'},alice,ua) - offer = await call('communication.offer_create', - {'description':'Review','scope':'Selftest','availability':'Now'},alice,ua) - checkpoint = await call('communication.checkpoint_create', - {'summary':'Selftest progress','resource_refs':[]},alice,ua) - proposal_target = await call('content.post_create', - {'parent':test_path+'/tmp','body':'before proposal'},alice,ua) - proposal_source = await call('content.post_create', - {'parent':test_path+'/tmp','body':'after proposal'},alice,ua) - require(all(item.status=='ok' for item in - (work,offer,checkpoint,proposal_target,proposal_source)), 'selftest_collaboration_create_failed') - proposal = await call('communication.proposal_create', - {'target':proposal_target.resources[0].id, - 'base_revision':proposal_target.resources[0].revision, - 'content_ref':wire(proposal_source.resources[0]),'message':'Apply selftest text'},alice,ua) - require(proposal.status=='ok','selftest_proposal_create_failed') - accepted = await call('communication.proposal_accept', - {'id':proposal.data['id'],'proposal_revision':proposal.resources[0].revision, - 'base_revision':proposal_target.resources[0].revision},alice,ua, - expected=((proposal.data['id'],proposal.data['generation']), - (proposal_target.resources[0].id,proposal_target.data['generation']))) - restored = await call('discovery.get',{'id':proposal_target.resources[0].id},alice,ua) - checks['collaboration']=(accepted.status=='ok' and restored.status=='ok' and - restored.data['content']=='after proposal') + empty_following = await call('communication.following', {}, bob, ub) + anonymous_following = await call('communication.following', {}) + followed = await call( + 'content.post_create', + {'parent': test_path + '/tmp', 'body': 'isolated Following fixture'}, + alice, + ua, + ) + require(followed.status == 'ok', 'selftest_following_post_failed') + followed_id = followed.resources[0].id + watched = await call('communication.watch', {'id': followed_id}, bob, ub) + shown = await call('communication.following', {}, bob, ub) + hidden = await call( + 'content.chmod', + {'id': followed_id, 'mode': '0600'}, + alice, + ua, + expected=((followed_id, followed.data['generation']),), + ) + redacted = await call('communication.following', {}, bob, ub) + checks['following'] = ( + empty_following.status == 'ok' + and not empty_following.data['items'] + and anonymous_following.error is not None + and anonymous_following.error.code == 'authentication_required' + and watched.status == 'ok' + and shown.status == 'ok' + and [item['id'] for item in shown.data['items']] == [followed_id] + and hidden.status == 'ok' + and redacted.status == 'ok' + and not redacted.data['items'] + ) + work = await call( + 'communication.request_create', + { + 'title': 'Isolated task', + 'description': 'Validate work records', + 'requirements': 'Explicit claim', + }, + alice, + ua, + ) + offer = await call( + 'communication.offer_create', + {'description': 'Review', 'scope': 'Selftest', 'availability': 'Now'}, + alice, + ua, + ) + checkpoint = await call( + 'communication.checkpoint_create', + {'summary': 'Selftest progress', 'resource_refs': []}, + alice, + ua, + ) + proposal_target = await call( + 'content.post_create', + {'parent': test_path + '/tmp', 'body': 'before proposal'}, + alice, + ua, + ) + proposal_source = await call( + 'content.post_create', + {'parent': test_path + '/tmp', 'body': 'after proposal'}, + alice, + ua, + ) + require( + all( + item.status == 'ok' + for item in (work, offer, checkpoint, proposal_target, proposal_source) + ), + 'selftest_collaboration_create_failed', + ) + proposal = await call( + 'communication.proposal_create', + { + 'target': proposal_target.resources[0].id, + 'base_revision': proposal_target.resources[0].revision, + 'content_ref': wire(proposal_source.resources[0]), + 'message': 'Apply selftest text', + }, + alice, + ua, + ) + require(proposal.status == 'ok', 'selftest_proposal_create_failed') + accepted = await call( + 'communication.proposal_accept', + { + 'id': proposal.data['id'], + 'proposal_revision': proposal.resources[0].revision, + 'base_revision': proposal_target.resources[0].revision, + }, + alice, + ua, + expected=( + (proposal.data['id'], proposal.data['generation']), + (proposal_target.resources[0].id, proposal_target.data['generation']), + ), + ) + restored = await call( + 'discovery.get', {'id': proposal_target.resources[0].id}, alice, ua + ) + checks['collaboration'] = ( + accepted.status == 'ok' + and restored.status == 'ok' + and restored.data['content'] == 'after proposal' + ) # OnlineIssuer exercises only the independent temporary Test Root # created above. No production trust material or signer is opened. async with app.metadata.transaction(write=False) as tx: - identity_cert=next(decode(Certificate,loads(raw)) for (raw,) in - tx.execute('SELECT body FROM certificates WHERE subject=?',(ua,)) - if decode(Certificate,loads(raw)).kind=='identity') - issued_audits=[loads(raw) for (raw,) in tx.execute('SELECT body FROM audit ORDER BY seq') - if loads(raw)['event']['type']=='cert.auto.issue'] - registration_audit=next((entry for entry in issued_audits if - entry['event']['data']['certificate_id']==identity_cert.resource_id),None) - checks['online_registration']=bool(registration_audit and - registration_audit['event']['data']['automatic'] is True and - registration_audit['event']['data']['authority_source']['key_id']==alice.key_id) - ordinary=grant_for(app.registry.capability('discovery.basic'), - scope=Scope(resource_id=post.resources[0].id),operations=('discovery.get@1',)) - delegated=await call('identity.delegate',{'grantee':ub,'key_id':bob.key_id, - 'grants':wire((ordinary,)),'ttl':600},alice,ua,request_id='online-selftest-delegation') - require(delegated.status=='ok','selftest_online_delegation_failed') + identity_cert = next( + decode(Certificate, loads(raw)) + for (raw,) in tx.execute('SELECT body FROM certificates WHERE subject=?', (ua,)) + if decode(Certificate, loads(raw)).kind == 'identity' + ) + issued_audits = [ + loads(raw) + for (raw,) in tx.execute('SELECT body FROM audit ORDER BY seq') + if loads(raw)['event']['type'] == 'cert.auto.issue' + ] + registration_audit = next( + ( + entry + for entry in issued_audits + if entry['event']['data']['certificate_id'] == identity_cert.resource_id + ), + None, + ) + checks['online_registration'] = bool( + registration_audit + and registration_audit['event']['data']['automatic'] is True + and registration_audit['event']['data']['authority_source']['key_id'] + == alice.key_id + ) + ordinary = grant_for( + app.registry.capability('discovery.basic'), + scope=Scope(resource_id=post.resources[0].id), + operations=('discovery.get@1',), + ) + delegated = await call( + 'identity.delegate', + {'grantee': ub, 'key_id': bob.key_id, 'grants': wire((ordinary,)), 'ttl': 600}, + alice, + ua, + request_id='online-selftest-delegation', + ) + require(delegated.status == 'ok', 'selftest_online_delegation_failed') async with app.metadata.transaction(write=False) as tx: - audit_rows=[loads(raw) for (raw,) in tx.execute('SELECT body FROM audit ORDER BY seq')] - delegation_audit=next((entry for entry in audit_rows if - entry['event']['type']=='cert.auto.issue' and delegated.status=='ok' and - entry['event']['data']['certificate_id']==delegated.data['certificate_id']),None) - checks['online_delegation']=bool(delegation_audit and - delegation_audit['event']['data']['authority_source']['id']==delegated.resources[0].id and - delegation_audit['event']['data']['grant_digest']==digest((ordinary,))) - renewed=await call('identity.certificate_renew',{},alice,ua) - overlong=await call('identity.certificate_renew',{'ttl':31536000},alice,ua) - checks['online_renewal']=(renewed.status=='ok' and overlong.error is not None and - overlong.error.code=='renewal_ttl_exceeded') - revoked_source=await call('identity.delegation_revoke',{'id':delegated.resources[0].id},alice,ua) + audit_rows = [ + loads(raw) for (raw,) in tx.execute('SELECT body FROM audit ORDER BY seq') + ] + delegation_audit = next( + ( + entry + for entry in audit_rows + if entry['event']['type'] == 'cert.auto.issue' + and delegated.status == 'ok' + and entry['event']['data']['certificate_id'] == delegated.data['certificate_id'] + ), + None, + ) + checks['online_delegation'] = bool( + delegation_audit + and delegation_audit['event']['data']['authority_source']['id'] + == delegated.resources[0].id + and delegation_audit['event']['data']['grant_digest'] == digest((ordinary,)) + ) + renewed = await call('identity.certificate_renew', {}, alice, ua) + overlong = await call('identity.certificate_renew', {'ttl': 31536000}, alice, ua) + checks['online_renewal'] = ( + renewed.status == 'ok' + and overlong.error is not None + and overlong.error.code == 'renewal_ttl_exceeded' + ) + revoked_source = await call( + 'identity.delegation_revoke', {'id': delegated.resources[0].id}, alice, ua + ) async with app.metadata.transaction(write=False) as tx: - derived=await app.certificates.validate(identity_cert.resource_id,tx) - delegation_invalid=False - try:await app.certificates.validate(delegated.data['certificate_id'],tx) - except Failure as exc:delegation_invalid=exc.code=='authority_source_inactive' - checks['online_source_revocation']=revoked_source.status=='ok' and derived.kind=='identity' and delegation_invalid - repeat=await call('content.post_create',{'parent':test_path+'/tmp','body':'retained source bytes\r\n'},alice,ua,request_id='same-write') - checks['idempotency']=repeat.replayed and repeat.resources==post.resources - rejected=await call('content.archive',{'id':post.resources[0].id},bob,ub, - expected=((post.resources[0].id,post.data['generation']),)) - checks['sticky']=rejected.error is not None and rejected.error.code=='sticky_denied' - denied=await call('discovery.get',{'id':test_path+'/private'},bob,ub) - checks['deny']=denied.error is not None and denied.error.code=='permission_denied' - denied=await call('content.post_create',{'parent':test_path+'/certified','body':'gate'},bob,ub) - grant=grant_for(app.registry.capability('resource.certified_write'),scope=Scope(resource_id='t_certified',descendants=True), - operations=('content.post_create@1',)) - cert=await approve(bob,ub,(grant,)) - accepted=await call('content.post_create',{'parent':test_path+'/certified','body':'gate'},bob,ub,(cert.resource_id,)) - checks['certgate']=denied.error is not None and denied.error.code=='certificate_gate' and accepted.status=='ok' - wrong=grant_for(app.registry.capability('resource.certified_write'),scope=Scope(resource_id='t_main',descendants=True), - operations=('content.post_create@1',)) - wrong_cert=await approve(alice,ua,(wrong,)) - rejected=await call('content.post_create',{'parent':test_path+'/certified','body':'wrong scope'},alice,ua,(wrong_cert.resource_id,)) - checks['certificate_scope']=rejected.error is not None and rejected.error.code=='certificate_gate' - manage=grant_for(app.registry.capability('group.manage_override'),scope=Scope(resource_id='g_admins'), - operations=('group.member.add@1','group.member.remove@1')) - manager=await approve(alice,ua,(manage,)) - await call('group.member.add',{'group':'g_admins','subject':ub},alice,ua,(manager.resource_id,)) - child=await call('content.topic_create',{'parent':test_path+'/admins','name':'selftest'},bob,ub) - require(child.status=='ok','selftest_group_failed') + derived = await app.certificates.validate(identity_cert.resource_id, tx) + delegation_invalid = False + try: + await app.certificates.validate(delegated.data['certificate_id'], tx) + except Failure as exc: + delegation_invalid = exc.code == 'authority_source_inactive' + checks['online_source_revocation'] = ( + revoked_source.status == 'ok' and derived.kind == 'identity' and delegation_invalid + ) + repeat = await call( + 'content.post_create', + {'parent': test_path + '/tmp', 'body': 'retained source bytes\r\n'}, + alice, + ua, + request_id='same-write', + ) + checks['idempotency'] = repeat.replayed and repeat.resources == post.resources + rejected = await call( + 'content.archive', + {'id': post.resources[0].id}, + bob, + ub, + expected=((post.resources[0].id, post.data['generation']),), + ) + checks['sticky'] = rejected.error is not None and rejected.error.code == 'sticky_denied' + denied = await call('discovery.get', {'id': test_path + '/private'}, bob, ub) + checks['deny'] = denied.error is not None and denied.error.code == 'permission_denied' + denied = await call( + 'content.post_create', {'parent': test_path + '/certified', 'body': 'gate'}, bob, ub + ) + grant = grant_for( + app.registry.capability('resource.certified_write'), + scope=Scope(resource_id='t_certified', descendants=True), + operations=('content.post_create@1',), + ) + cert = await approve(bob, ub, (grant,)) + accepted = await call( + 'content.post_create', + {'parent': test_path + '/certified', 'body': 'gate'}, + bob, + ub, + (cert.resource_id,), + ) + checks['certgate'] = ( + denied.error is not None + and denied.error.code == 'certificate_gate' + and accepted.status == 'ok' + ) + wrong = grant_for( + app.registry.capability('resource.certified_write'), + scope=Scope(resource_id='t_main', descendants=True), + operations=('content.post_create@1',), + ) + wrong_cert = await approve(alice, ua, (wrong,)) + rejected = await call( + 'content.post_create', + {'parent': test_path + '/certified', 'body': 'wrong scope'}, + alice, + ua, + (wrong_cert.resource_id,), + ) + checks['certificate_scope'] = ( + rejected.error is not None and rejected.error.code == 'certificate_gate' + ) + manage = grant_for( + app.registry.capability('group.manage_override'), + scope=Scope(resource_id='g_admins'), + operations=('group.member.add@1', 'group.member.remove@1'), + ) + manager = await approve(alice, ua, (manage,)) + await call( + 'group.member.add', + {'group': 'g_admins', 'subject': ub}, + alice, + ua, + (manager.resource_id,), + ) + child = await call( + 'content.topic_create', + {'parent': test_path + '/admins', 'name': 'selftest'}, + bob, + ub, + ) + require(child.status == 'ok', 'selftest_group_failed') async with app.metadata.transaction(write=False) as tx: - resource=await tx.resource(child.resources[0].id) - checks['setgid']=resource.group=='g_admins' and bool(resource.mode&0o2000) - await call('group.member.remove',{'group':'g_admins','subject':ub},alice,ua,(manager.resource_id,)) - rejected=await call('discovery.get',{'id':child.resources[0].id},bob,ub) - checks['membership_revocation']=rejected.error is not None + resource = await tx.resource(child.resources[0].id) + checks['setgid'] = resource.group == 'g_admins' and bool(resource.mode & 0o2000) + await call( + 'group.member.remove', + {'group': 'g_admins', 'subject': ub}, + alice, + ua, + (manager.resource_id,), + ) + rejected = await call('discovery.get', {'id': child.resources[0].id}, bob, ub) + checks['membership_revocation'] = rejected.error is not None async with app.metadata.transaction(write=True) as tx: - tx.execute('UPDATE certificates SET revoked=1 WHERE id=?',(cert.resource_id,),write=True) - rejected=await call('content.post_create',{'parent':test_path+'/certified','body':'revoked'},bob,ub,(cert.resource_id,)) - checks['certificate_revocation']=rejected.error is not None and rejected.error.code=='certificate_revoked' - rejected=await call('discovery.get',{'id':test_path+'/private'},root,ROOT_SUBJECT) - checks['root_network_rejected']=rejected.error is not None and rejected.error.code=='local_only' - all_versions=await call('discovery.get',{'id':post.resources[0].id,'view':'meta'}) - checks['stable_id_read']=all_versions.status=='ok' + tx.execute( + 'UPDATE certificates SET revoked=1 WHERE id=?', (cert.resource_id,), write=True + ) + rejected = await call( + 'content.post_create', + {'parent': test_path + '/certified', 'body': 'revoked'}, + bob, + ub, + (cert.resource_id,), + ) + checks['certificate_revocation'] = ( + rejected.error is not None and rejected.error.code == 'certificate_revoked' + ) + rejected = await call( + 'discovery.get', {'id': test_path + '/private'}, root, ROOT_SUBJECT + ) + checks['root_network_rejected'] = ( + rejected.error is not None and rejected.error.code == 'local_only' + ) + all_versions = await call('discovery.get', {'id': post.resources[0].id, 'view': 'meta'}) + checks['stable_id_read'] = all_versions.status == 'ok' from msg.admin.honor_check import check_honors - checks['honor_ceremony_display']=await check_honors(app,call,register) - checks.update(await _selftest_ca_chain(app,root,call,register,now)) + + checks['honor_ceremony_display'] = await check_honors(app, call, register) + checks.update(await _selftest_ca_chain(app, root, call, register, now)) from msg.admin.upgrade_check import check_upgrade_recovery - checks['identity_upgrade_recovery']=await check_upgrade_recovery(app,now) + + checks['identity_upgrade_recovery'] = await check_upgrade_recovery(app, now) from msg.admin.token_delivery_check import check_token_delivery - checks['credential_delivery_recovery']=await check_token_delivery(app,now) + + checks['credential_delivery_recovery'] = await check_token_delivery(app, now) from msg.admin.custodial_check import check_custodial_history - checks['custodial_history_recovery']=await check_custodial_history(app,now) + + checks['custodial_history_recovery'] = await check_custodial_history(app, now) from msg.admin.market_check import check_market, check_market_e2e - checks['market_e2e']=await check_market_e2e(app,root,call,register) - checks['market_lifecycle']=await check_market(app,root,call,register,now) + + checks['market_e2e'] = await check_market_e2e(app, root, call, register) + checks['market_lifecycle'] = await check_market(app, root, call, register, now) from msg.admin.git_check import check_git - checks['git_push_read_cas']=await check_git(app,call,register) - checks.update(await _selftest_search_hosting(app,call,register,test_path)) + + checks['git_push_read_cas'] = await check_git(app, call, register) + checks.update(await _selftest_search_hosting(app, call, register, test_path)) from msg.admin.dm_check import check_private_dm - checks['private_dm']=await check_private_dm(app,now) + + checks['private_dm'] = await check_private_dm(app, now) from msg.admin.content_edit_check import check_content_editing - checks['content_editing']=await check_content_editing(app,call,register,test_path) + + checks['content_editing'] = await check_content_editing(app, call, register, test_path) from msg.admin.transfer_check import check_transfer - checks['transfer_state_machine']=await check_transfer(app,call,register) + + checks['transfer_state_machine'] = await check_transfer(app, call, register) except Failure as exc: - checks['failure']={'code':exc.code} + checks['failure'] = {'code': exc.code} finally: await app.close() - features=feature_results(feature_manifest(),checks,'selftest_case') - return {'ok':bool(checks) and all(value is True for value in checks.values()) and - all(row['status']!='fail' for row in features.values()), - 'checks':checks,'features':features,'cleaned_up':not folder.exists(), - 'configuration_fields':configuration_fields,'tool_sandbox':tool_sandbox_details} + features = feature_results(feature_manifest(), checks, 'selftest_case') + return { + 'ok': bool(checks) + and all(value is True for value in checks.values()) + and all(row['status'] != 'fail' for row in features.values()), + 'checks': checks, + 'features': features, + 'cleaned_up': not folder.exists(), + 'configuration_fields': configuration_fields, + 'tool_sandbox': tool_sandbox_details, + } diff --git a/src/msg/admin/dm_check.py b/src/msg/admin/dm_check.py index 63b9ba37..e37de964 100644 --- a/src/msg/admin/dm_check.py +++ b/src/msg/admin/dm_check.py @@ -1,4 +1,5 @@ """Exercise private conversations using fresh clients in the isolated Test Root.""" + from pathlib import Path from tempfile import TemporaryDirectory from uuid import uuid4 @@ -19,29 +20,39 @@ async def check_private_dm(app, now): async with httpx.AsyncClient(transport=httpx.ASGITransport(app=create_app(app))) as http: clients = [] for name in ('sender', 'recipient', 'outsider'): - client = MsgClient(ClientState(Path(directory) / name, - server=app.settings.service_url), + client = MsgClient( + ClientState(Path(directory) / name, server=app.settings.service_url), HTTPTransport(app.settings.service_url, http=http), - clock=lambda: now, retries=0) + clock=lambda: now, + retries=0, + ) client.checked(await client.register('dm-' + uuid4().hex[:16])) clients.append(client) sender, recipient, outsider = clients - requested = sender.checked(await sender.call('communication.dm_request', - {'recipient': recipient.state.subject})) + requested = sender.checked( + await sender.call( + 'communication.dm_request', {'recipient': recipient.state.subject} + ) + ) conversation = requested.data['conversation_id'] if requested.data['state'] != 'pending': return False - early = await sender.call('communication.dm_send', - {'conversation_id': conversation, 'body': 'not accepted'}) + early = await sender.call( + 'communication.dm_send', {'conversation_id': conversation, 'body': 'not accepted'} + ) if early.status != 'error' or early.error.code != 'dm_controlled_resource': return False - accepted = recipient.checked(await recipient.call('communication.dm_accept', - {'conversation_id': conversation})) + accepted = recipient.checked( + await recipient.call('communication.dm_accept', {'conversation_id': conversation}) + ) if accepted.data['state'] != 'active': return False body = 'isolated private conversation ' + uuid4().hex - sent = sender.checked(await sender.call('communication.dm_send', - {'conversation_id': conversation, 'body': body})) + sent = sender.checked( + await sender.call( + 'communication.dm_send', {'conversation_id': conversation, 'body': body} + ) + ) post = sent.resources[0].id for participant in (sender, recipient): read = await participant.call('discovery.get', {'id': post}) @@ -54,12 +65,15 @@ async def check_private_dm(app, now): ('communication.dm_send', {'conversation_id': conversation, 'body': 'intrusion'}), ): denied = await outsider.call(operation, arguments) - expected = 'dm_not_found' if operation == 'communication.dm_send' else 'permission_denied' + expected = ( + 'dm_not_found' if operation == 'communication.dm_send' else 'permission_denied' + ) if denied.status != 'error' or denied.error.code != expected: return False hidden = await outsider.call('discovery.search', {'query': body}) if hidden.status != 'ok' or hidden.data.get('items'): return False anonymous = await HTTPTransport(app.settings.service_url, http=http).call( - request_for('discovery.get', {'id': post}, app.settings.service_url)) + request_for('discovery.get', {'id': post}, app.settings.service_url) + ) return anonymous.status == 'error' and anonymous.error.code == 'permission_denied' diff --git a/src/msg/admin/git_check.py b/src/msg/admin/git_check.py index 536b3906..f4b92f4f 100644 --- a/src/msg/admin/git_check.py +++ b/src/msg/admin/git_check.py @@ -1,10 +1,11 @@ """Real Git publication/fetch checks inside the disposable selftest installation.""" + import asyncio -from datetime import timedelta import hashlib -from pathlib import Path import subprocess import tempfile +from datetime import timedelta +from pathlib import Path from urllib.parse import urlsplit import httpx @@ -15,8 +16,8 @@ from msg.core.errors import Failure, require from msg.core.requests import request_for from msg.extensions.repositories import NativeGitStore -from msg.transports.http import create_app from msg.transports.git_http import GitHTTPAdapter +from msg.transports.http import create_app async def check_git(app, call, register): @@ -26,12 +27,22 @@ async def check_git(app, call, register): require(created.status == 'ok', 'selftest_git_create_failed') rid = created.resources[0].id store = NativeGitStore(app) - env = {**store.env, 'GIT_AUTHOR_NAME': 'Selftest', 'GIT_AUTHOR_EMAIL': 'selftest@example.invalid'} + env = { + **store.env, + 'GIT_AUTHOR_NAME': 'Selftest', + 'GIT_AUTHOR_EMAIL': 'selftest@example.invalid', + } def run(folder, *args, input=None): try: - return subprocess.run(['git', '-C', str(folder), *args], input=input, - capture_output=True, check=True, timeout=30, env=env).stdout + return subprocess.run( + ['git', '-C', str(folder), *args], + input=input, + capture_output=True, + check=True, + timeout=30, + env=env, + ).stdout except (OSError, subprocess.SubprocessError) as exc: raise Failure('selftest_git_command_failed') from exc @@ -40,9 +51,20 @@ def pkt(data): async def snapshot(): async with app.metadata.transaction(write=False) as tx: - return {table: tuple(tx.rows(f'SELECT * FROM {table} ORDER BY 1')) - for table in ('resources', 'events', 'audit', 'jobs', 'results', 'transfers', - 'credentials', 'reactions', 'watches')} + return { + table: tuple(tx.rows(f'SELECT * FROM {table} ORDER BY 1')) + for table in ( + 'resources', + 'events', + 'audit', + 'jobs', + 'results', + 'transfers', + 'credentials', + 'reactions', + 'watches', + ) + } with tempfile.TemporaryDirectory(prefix='msg-selftest-git-') as temporary: work = Path(temporary) @@ -52,62 +74,114 @@ async def snapshot(): await asyncio.to_thread(run, work, 'add', 'README.md') await asyncio.to_thread(run, work, 'commit', '-m', 'isolated selftest') commit = (await asyncio.to_thread(run, work, 'rev-parse', 'HEAD')).decode().strip() - pack = await asyncio.to_thread(run, work, 'pack-objects', '--stdout', '--revs', - input=(commit + '\n').encode()) - receive = pkt(f'{"0" * 40} {commit} refs/heads/main\0report-status\n'.encode()) + b'0000' + pack + pack = await asyncio.to_thread( + run, work, 'pack-objects', '--stdout', '--revs', input=(commit + '\n').encode() + ) + receive = ( + pkt(f'{"0" * 40} {commit} refs/heads/main\0report-status\n'.encode()) + b'0000' + pack + ) def headers(request_id): - packet = request_for('git.http_receive', {'id': rid, - 'pack_digest': 'sha256:' + hashlib.sha256(receive).hexdigest(), 'pack_size': len(receive)}, - app.settings.service_url, signer=key, subject=owner, request_id=request_id, - expires_at=app.clock() + timedelta(seconds=120)) - return {'X-Msg-Request': b64(canonical(packet)), - 'Content-Type': 'application/x-git-receive-pack-request'} + packet = request_for( + 'git.http_receive', + { + 'id': rid, + 'pack_digest': 'sha256:' + hashlib.sha256(receive).hexdigest(), + 'pack_size': len(receive), + }, + app.settings.service_url, + signer=key, + subject=owner, + request_id=request_id, + expires_at=app.clock() + timedelta(seconds=120), + ) + return { + 'X-Msg-Request': b64(canonical(packet)), + 'Content-Type': 'application/x-git-receive-pack-request', + } - async with httpx.AsyncClient(transport=httpx.ASGITransport(app=create_app(app)), - base_url=app.settings.service_url) as http: + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=create_app(app)), base_url=app.settings.service_url + ) as http: endpoint = f'/-/git/{rid}/git-receive-pack' - denied = await http.post(endpoint, content=receive, - headers={'Content-Type': 'application/x-git-receive-pack-request'}) + denied = await http.post( + endpoint, + content=receive, + headers={'Content-Type': 'application/x-git-receive-pack-request'}, + ) require(denied.status_code == 401, 'selftest_git_unsigned_push_allowed') - pushed = await http.post(endpoint, content=receive, headers=headers('selftest-git-push')) - require(pushed.status_code == 200 and b'ok refs/heads/main\n' in pushed.content, - 'selftest_git_push_failed') - require(await asyncio.to_thread(run, store.path(rid), 'show', 'refs/heads/main:README.md') == payload, - 'selftest_git_publication_mismatch') + pushed = await http.post( + endpoint, content=receive, headers=headers('selftest-git-push') + ) + require( + pushed.status_code == 200 and b'ok refs/heads/main\n' in pushed.content, + 'selftest_git_push_failed', + ) + require( + await asyncio.to_thread(run, store.path(rid), 'show', 'refs/heads/main:README.md') + == payload, + 'selftest_git_publication_mismatch', + ) before = await snapshot() - replay = await http.post(endpoint, content=receive, headers=headers('selftest-git-push')) - require(replay.status_code == 200 and replay.content == pushed.content and - await snapshot() == before, 'selftest_git_replay_mutated') + replay = await http.post( + endpoint, content=receive, headers=headers('selftest-git-push') + ) + require( + replay.status_code == 200 + and replay.content == pushed.content + and await snapshot() == before, + 'selftest_git_replay_mutated', + ) # A new request with an obsolete expected ref must not overwrite it. - stale = await http.post(endpoint, content=receive, headers=headers('selftest-git-stale')) - require(stale.status_code == 200 and b'ng refs/heads/main ' in stale.content, - 'selftest_git_stale_ref_allowed') - refs = await asyncio.to_thread(run, store.path(rid), 'for-each-ref', '--format=%(refname) %(objectname)') + stale = await http.post( + endpoint, content=receive, headers=headers('selftest-git-stale') + ) + require( + stale.status_code == 200 and b'ng refs/heads/main ' in stale.content, + 'selftest_git_stale_ref_allowed', + ) + refs = await asyncio.to_thread( + run, store.path(rid), 'for-each-ref', '--format=%(refname) %(objectname)' + ) require(refs == f'refs/heads/main {commit}\n'.encode(), 'selftest_git_ref_changed') before = await snapshot() + # Production's /@user/*.git router does not expose /_test/. # Mount the real read adapter over its isolated resource path only; # it still executes git.refs authorization and git http-backend. async def read(request): return await GitHTTPAdapter(app).http( - request, urlsplit(created.data['read_url']).path, 'git-upload-pack') + request, urlsplit(created.data['read_url']).path, 'git-upload-pack' + ) + fetch_app = Starlette(routes=[Route('/fetch', read, methods=['POST'])]) - async with httpx.AsyncClient(transport=httpx.ASGITransport(app=fetch_app), - base_url=app.settings.service_url) as reader: - fetch = await reader.post('/fetch', + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=fetch_app), base_url=app.settings.service_url + ) as reader: + fetch = await reader.post( + '/fetch', content=pkt(f'want {commit}\n'.encode()) + b'00000009done\n', - headers={'Content-Type': 'application/x-git-upload-pack-request'}) - require(fetch.status_code == 200 and fetch.content.startswith(b'0008NAK\nPACK'), - 'selftest_git_fetch_failed') + headers={'Content-Type': 'application/x-git-upload-pack-request'}, + ) + require( + fetch.status_code == 200 and fetch.content.startswith(b'0008NAK\nPACK'), + 'selftest_git_fetch_failed', + ) # Import the actual response into a fresh repository, not the source fixture. fetched = work / 'fetched' fetched.mkdir() await asyncio.to_thread(run, fetched, 'init', '--bare', '--object-format=sha1') await asyncio.to_thread(run, fetched, 'index-pack', '--stdin', input=fetch.content[8:]) - require(await asyncio.to_thread(run, fetched, 'show', commit + ':README.md') == payload, - 'selftest_git_fetch_mismatch') - require(await snapshot() == before and refs == await asyncio.to_thread( - run, store.path(rid), 'for-each-ref', '--format=%(refname) %(objectname)'), - 'selftest_git_read_mutated') + require( + await asyncio.to_thread(run, fetched, 'show', commit + ':README.md') == payload, + 'selftest_git_fetch_mismatch', + ) + require( + await snapshot() == before + and refs + == await asyncio.to_thread( + run, store.path(rid), 'for-each-ref', '--format=%(refname) %(objectname)' + ), + 'selftest_git_read_mutated', + ) return True diff --git a/src/msg/admin/honor_check.py b/src/msg/admin/honor_check.py index 0da52930..fb9ab035 100644 --- a/src/msg/admin/honor_check.py +++ b/src/msg/admin/honor_check.py @@ -1,4 +1,5 @@ """Read-only honor diagnostics and the isolated selftest's real protocol client.""" + from msg.bootstrap import manifest from msg.core.codec import canonical, decode, digest from msg.core.errors import require @@ -21,42 +22,67 @@ async def inspect_honors(app, tx): 'achievement_grants': {'id', 'subject', 'achievement_id', 'spec_version', 'body'}, 'achievement_pins': {'subject', 'grant_id', 'position'}, }.items(): - columns = {row[0] for row in tx.rows( - "SELECT column_name FROM information_schema.columns " - "WHERE table_schema='public' AND table_name=?", (table,))} + columns = { + row[0] + for row in tx.rows( + 'SELECT column_name FROM information_schema.columns ' + "WHERE table_schema='public' AND table_name=?", + (table,), + ) + } require(required <= columns, 'honor_schema_missing', table) for name in ('start', 'answer', 'finish', 'pin', 'unpin', 'reorder', 'list'): spec = app.registry.operation('achievement.' + name) - require(spec.effect == ('read' if name == 'list' else 'transaction'), - 'honor_contract_invalid') + require( + spec.effect == ('read' if name == 'list' else 'transaction'), 'honor_contract_invalid' + ) if name in {'finish', 'pin', 'unpin', 'reorder'}: require(spec.require_signature, 'honor_contract_invalid') sample = manifest()['honor_sample'] - require(sample['policy'] == { - 'achievement_id': I_AM_NOT_HUMAN.id, 'spec_version': I_AM_NOT_HUMAN.version, - 'rounds': 5, 'round_ttl_seconds': ROUND_TTL_SECONDS, - 'ceremony_ttl_seconds': CEREMONY_TTL_SECONDS, 'strategy': STRATEGY, - 'strategy_version': STRATEGY_VERSION, 'privileges': False, - } and sample['profile']['max_pins'] == MAX_PINS and - sample['profile']['pinned_grant_ids'] == [] and sample['grants'] == [] and - sample['issuer']['claim'] == I_AM_NOT_HUMAN.claim and - sample['issuer']['purpose'] == 'achievement-grant' and - sample['issuer']['key_source'] == 'service_receipt_signer', 'honor_sample_drift') + require( + sample['policy'] + == { + 'achievement_id': I_AM_NOT_HUMAN.id, + 'spec_version': I_AM_NOT_HUMAN.version, + 'rounds': 5, + 'round_ttl_seconds': ROUND_TTL_SECONDS, + 'ceremony_ttl_seconds': CEREMONY_TTL_SECONDS, + 'strategy': STRATEGY, + 'strategy_version': STRATEGY_VERSION, + 'privileges': False, + } + and sample['profile']['max_pins'] == MAX_PINS + and sample['profile']['pinned_grant_ids'] == [] + and sample['grants'] == [] + and sample['issuer']['claim'] == I_AM_NOT_HUMAN.claim + and sample['issuer']['purpose'] == 'achievement-grant' + and sample['issuer']['key_source'] == 'service_receipt_signer', + 'honor_sample_drift', + ) resource = await tx.resource('r_honor_sample') revision = await tx.revision(ResourceRef(id=resource.id, revision=resource.revision)) - require(revision.content is not None and - revision.content.digest == digest(canonical(sample)), - 'honor_sample_drift') + require( + revision.content is not None and revision.content.digest == digest(canonical(sample)), + 'honor_sample_drift', + ) # Doctor does not load Application storage (which can create directories). # Read the installed issuer directly, as the online-CA doctor does. issuer = Ed25519Signer.from_bytes((app.settings.service_keys / 'receipt.key').read_bytes()) probe = canonical({'purpose': 'read-only-honor-doctor'}) - verify(issuer.public_key, probe, - issuer.sign(probe, purpose='achievement-grant'), - purpose='achievement-grant') - return {'read_only': True, 'issuer': issuer.key_id, - 'strategy': STRATEGY, 'spec_version': I_AM_NOT_HUMAN.version, - 'pregrant': False, 'privileges': False} + verify( + issuer.public_key, + probe, + issuer.sign(probe, purpose='achievement-grant'), + purpose='achievement-grant', + ) + return { + 'read_only': True, + 'issuer': issuer.key_id, + 'strategy': STRATEGY, + 'spec_version': I_AM_NOT_HUMAN.version, + 'pregrant': False, + 'privileges': False, + } async def check_honors(app, call, register): @@ -64,11 +90,17 @@ async def check_honors(app, call, register): key, subject = await register('honor-selftest') stranger, other = await register('honor-stranger') async with app.metadata.transaction(write=False) as tx: - security_before = (tx.rows('SELECT body FROM certificates ORDER BY id'), - tx.rows('SELECT body FROM credentials ORDER BY id')) + security_before = ( + tx.rows('SELECT body FROM certificates ORDER BY id'), + tx.rows('SELECT body FROM credentials ORDER BY id'), + ) empty = await call('achievement.list', {'subject_id': subject}) - require(empty.status == 'ok' and not empty.data['achievements'] and - not empty.data['pinned_grant_ids'], 'selftest_honor_pregranted') + require( + empty.status == 'ok' + and not empty.data['achievements'] + and not empty.data['pinned_grant_ids'], + 'selftest_honor_pregranted', + ) started = await call('achievement.start', {}, key, subject) require(started.status == 'ok', 'selftest_honor_start_failed') challenge = started.data @@ -76,29 +108,46 @@ async def check_honors(app, call, register): require(challenge.get('round') == number, 'selftest_honor_round_invalid') answer = 'y' if number == 4: - bits = ''.join('1' if char == '\u200c' else '0' for char in challenge['question'] - if char in {'\u200b', '\u200c'}) + bits = ''.join( + '1' if char == '\u200c' else '0' + for char in challenge['question'] + if char in {'\u200b', '\u200c'} + ) require(len(bits) == 16, 'selftest_honor_payload_invalid') - answer = ''.join(str(int(bits[i:i + 4], 2)) for i in range(0, 16, 4)) - result = await call('achievement.answer', { - 'challenge_id': challenge['challenge_id'], 'round': number, - 'question_digest': challenge['question_digest'], 'nonce': challenge['nonce'], - 'answer': answer, - }, key, subject) - require(result.status == 'ok' and result.data.get('round') == number + 1, - 'selftest_honor_answer_failed') + answer = ''.join(str(int(bits[i : i + 4], 2)) for i in range(0, 16, 4)) + result = await call( + 'achievement.answer', + { + 'challenge_id': challenge['challenge_id'], + 'round': number, + 'question_digest': challenge['question_digest'], + 'nonce': challenge['nonce'], + 'answer': answer, + }, + key, + subject, + ) + require( + result.status == 'ok' and result.data.get('round') == number + 1, + 'selftest_honor_answer_failed', + ) challenge = result.data - finished = await call('achievement.finish', { - **{field: challenge[field] for field in - ('challenge_id', 'question_digest', 'nonce', 'ceremony_digest')}, - 'statement': FINAL_STATEMENT, - }, key, subject) - require(finished.status == 'ok' and 'grant' in finished.data, - 'selftest_honor_finish_failed') + finished = await call( + 'achievement.finish', + { + **{ + field: challenge[field] + for field in ('challenge_id', 'question_digest', 'nonce', 'ceremony_digest') + }, + 'statement': FINAL_STATEMENT, + }, + key, + subject, + ) + require(finished.status == 'ok' and 'grant' in finished.data, 'selftest_honor_finish_failed') grant = dict(finished.data['grant']) signature = decode(Signature, grant.pop('signature')) - verify(app.receipt_signer.public_key, canonical(grant), signature, - purpose='achievement-grant') + verify(app.receipt_signer.public_key, canonical(grant), signature, purpose='achievement-grant') gid = grant['id'] denied = await call('achievement.pin', {'grant_id': gid}, stranger, other) pinned = await call('achievement.pin', {'grant_id': gid}, key, subject) @@ -108,13 +157,20 @@ async def check_honors(app, call, register): final = await call('achievement.list', {'subject_id': subject}) duplicate = await call('achievement.start', {}, key, subject) async with app.metadata.transaction(write=False) as tx: - security_after = (tx.rows('SELECT body FROM certificates ORDER BY id'), - tx.rows('SELECT body FROM credentials ORDER BY id')) + security_after = ( + tx.rows('SELECT body FROM certificates ORDER BY id'), + tx.rows('SELECT body FROM credentials ORDER BY id'), + ) count = tx.one('SELECT COUNT(*) FROM achievement_grants WHERE subject=?', (subject,))[0] - return (denied.error is not None and denied.error.code == 'achievement_grant_not_found' and - all(result.status == 'ok' for result in (pinned, reordered, public, unpinned, final)) and - list(public.data['pinned_grant_ids']) == [gid] and - not final.data['pinned_grant_ids'] and len(final.data['achievements']) == 1 and - count == 1 and duplicate.error is not None and - duplicate.error.code == 'achievement_already_granted' and - security_after == security_before) + return ( + denied.error is not None + and denied.error.code == 'achievement_grant_not_found' + and all(result.status == 'ok' for result in (pinned, reordered, public, unpinned, final)) + and list(public.data['pinned_grant_ids']) == [gid] + and not final.data['pinned_grant_ids'] + and len(final.data['achievements']) == 1 + and count == 1 + and duplicate.error is not None + and duplicate.error.code == 'achievement_already_granted' + and security_after == security_before + ) diff --git a/src/msg/admin/legacy_approval.py b/src/msg/admin/legacy_approval.py index 13379059..345495d8 100644 --- a/src/msg/admin/legacy_approval.py +++ b/src/msg/admin/legacy_approval.py @@ -19,18 +19,20 @@ from msg.security.crypto import Ed25519Signer, open_private_key from msg.security.quarantine import require_live_authority from msg.security.root_files import read_private, rotation_lock -from msg.storage import legacy_git_import as git -from msg.storage import legacy_identity_plan as identities -from msg.storage import legacy_resource_import as content +from msg.storage import ( + legacy_git_import as git, + legacy_identity_plan as identities, + legacy_resource_import as content, +) def write_new(path, value): path = Path(path) - require(not path.exists() and not path.is_symlink(), "approval_destination_exists") + require(not path.exists() and not path.is_symlink(), 'approval_destination_exists') fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600) - with os.fdopen(fd, "wb") as stream: + with os.fdopen(fd, 'wb') as stream: stream.write(canonical(value)) - stream.write(b"\n") + stream.write(b'\n') stream.flush() os.fsync(stream.fileno()) @@ -40,109 +42,109 @@ def read_json(path): def recognized_payload(value): - require(isinstance(value, dict), "legacy_approval_invalid") - kind = value.get("format") + require(isinstance(value, dict), 'legacy_approval_invalid') + kind = value.get('format') common = { - key: value.get(key) for key in ("parent", "parent_generation", "operator", "expires_at") + key: value.get(key) for key in ('parent', 'parent_generation', 'operator', 'expires_at') } if kind == content.PURPOSE: expected = content.approval_payload( - sha256=value.get("source_sha256"), - service=value.get("target_service"), - identities=value.get("identities"), + sha256=value.get('source_sha256'), + service=value.get('target_service'), + identities=value.get('identities'), **common, ) elif kind == git.PURPOSE: expected = git.git_approval( - bundle_sha256=value.get("bundle_sha256"), - refs_digest=value.get("refs_digest"), - default_ref=value.get("default_ref"), - name=value.get("name"), - service=value.get("target_service"), + bundle_sha256=value.get('bundle_sha256'), + refs_digest=value.get('refs_digest'), + default_ref=value.get('default_ref'), + name=value.get('name'), + service=value.get('target_service'), **common, ) elif kind == identities.PURPOSE: expected = identities.approval_payload( - plan_digest=value.get("plan_digest"), - mappings=value.get("mappings"), - service=value.get("target_service"), + plan_digest=value.get('plan_digest'), + mappings=value.get('mappings'), + service=value.get('target_service'), **common, ) else: - require(False, "unsupported_legacy_approval_purpose") - require(value == expected, "legacy_approval_invalid") + require(False, 'unsupported_legacy_approval_purpose') + require(value == expected, 'legacy_approval_invalid') return kind async def validate_and_review(app, approval): kind = recognized_payload(approval) - require(approval["target_service"] == app.settings.service_url, "legacy_service_mismatch") + require(approval['target_service'] == app.settings.service_url, 'legacy_service_mismatch') require( - app.clock() < parse_time(approval["expires_at"]) <= app.clock() + timedelta(hours=24), - "legacy_approval_expired", + app.clock() < parse_time(approval['expires_at']) <= app.clock() + timedelta(hours=24), + 'legacy_approval_expired', ) - mappings = approval.get("identities", approval.get("mappings", {})) - require(isinstance(mappings, dict), "legacy_mapping_invalid") + mappings = approval.get('identities', approval.get('mappings', {})) + require(isinstance(mappings, dict), 'legacy_mapping_invalid') async with app.metadata.transaction(write=False) as tx: require_live_authority(tx) - require(tx.setting("runtime_config", {}).get("accept_writes", True), "writes_paused") - holder = await tx.subject(approval["operator"]) - require(holder.kind == "registered", "legacy_operator_must_be_registered") + require(tx.setting('runtime_config', {}).get('accept_writes', True), 'writes_paused') + holder = await tx.subject(approval['operator']) + require(holder.kind == 'registered', 'legacy_operator_must_be_registered') holder_resource = await tx.resource(holder.resource_id) - parent = await tx.resource(approval["parent"]) + parent = await tx.resource(approval['parent']) require( parent.owner == holder.resource_id and parent.mode == 0o700 - and parent.state == "active" - and parent.generation == approval["parent_generation"], - "legacy_parent_not_private", + and parent.state == 'active' + and parent.generation == approval['parent_generation'], + 'legacy_parent_not_private', ) require( app.registry.resource_type(parent.type, parent.type_version).container, - "not_a_container", + 'not_a_container', ) if kind != git.PURPOSE: require( - not tx.one("SELECT id FROM resources WHERE parent=?", (parent.id,)), - "legacy_parent_not_empty", + not tx.one('SELECT id FROM resources WHERE parent=?', (parent.id,)), + 'legacy_parent_not_empty', ) target_counts = Counter() for target in mappings.values(): - require(target is None or isinstance(target, str), "legacy_mapping_invalid") + require(target is None or isinstance(target, str), 'legacy_mapping_invalid') if target is not None: require( - (await tx.subject(target)).kind == "registered", "legacy_mapping_target_invalid" + (await tx.subject(target)).kind == 'registered', 'legacy_mapping_target_invalid' ) target_counts[target] += 1 return { - "purpose": kind, - "approval_digest": digest(approval), - "artifact_binding": { + 'purpose': kind, + 'approval_digest': digest(approval), + 'artifact_binding': { key: approval[key] for key in ( - "source_sha256", - "bundle_sha256", - "refs_digest", - "default_ref", - "name", - "plan_digest", + 'source_sha256', + 'bundle_sha256', + 'refs_digest', + 'default_ref', + 'name', + 'plan_digest', ) if key in approval }, - "target_service": app.settings.service_url, - "root_public_key_digest": digest(app.certificates.root_public_key), - "real_holder": {"subject": holder.resource_id, "handle": holder_resource.name}, - "private_parent": { - "id": parent.id, - "path": await tx.path(parent.id), - "generation": parent.generation, + 'target_service': app.settings.service_url, + 'root_public_key_digest': digest(app.certificates.root_public_key), + 'real_holder': {'subject': holder.resource_id, 'handle': holder_resource.name}, + 'private_parent': { + 'id': parent.id, + 'path': await tx.path(parent.id), + 'generation': parent.generation, }, - "historical_identity_count": len(mappings), - "unmapped_count": sum(v is None for v in mappings.values()), - "explicit_new_subject_associations": dict(target_counts), - "legacy_authority_enabled": False, - "ownership_proof_claimed": False, - "expires_at": approval["expires_at"], + 'historical_identity_count': len(mappings), + 'unmapped_count': sum(v is None for v in mappings.values()), + 'explicit_new_subject_associations': dict(target_counts), + 'legacy_authority_enabled': False, + 'ownership_proof_claimed': False, + 'expires_at': approval['expires_at'], } @@ -160,30 +162,30 @@ def sign_approval(config_dir, source, destination): require_local_console(config_dir) require( not Path(destination).exists() and not Path(destination).is_symlink(), - "approval_destination_exists", + 'approval_destination_exists', ) approval = read_json(source) review, public = asyncio.run(inspect_approval(config_dir, approval)) print(canonical(review).decode()) - phrase = "APPROVE LEGACY IMPORT " + digest(approval) - require(input("Type " + phrase + ": ") == phrase, "approval_cancelled") + phrase = 'APPROVE LEGACY IMPORT ' + digest(approval) + require(input('Type ' + phrase + ': ') == phrase, 'approval_cancelled') settings = load_settings(config_dir) with rotation_lock(settings.root_private_dir): envelope = loads(read_private(root_envelope(config_dir))) - private = open_private_key(envelope, getpass.getpass("Root PIN/passphrase: ")) + private = open_private_key(envelope, getpass.getpass('Root PIN/passphrase: ')) signer = Ed25519Signer.from_bytes(private) - require(signer.public_key == public, "root_key_mismatch") + require(signer.public_key == public, 'root_key_mismatch') result = { - "approval": approval, - "signature": wire( + 'approval': approval, + 'signature': wire( signer.sign(canonical(approval), purpose=recognized_payload(approval)) ), } write_new(destination, result) return { - "status": "legacy_approval_signed", - "purpose": approval["format"], - "approval_digest": digest(approval), + 'status': 'legacy_approval_signed', + 'purpose': approval['format'], + 'approval_digest': digest(approval), } @@ -194,42 +196,42 @@ async def prepare(args): async with app.metadata.transaction(write=False) as tx: parent = await tx.resource(args.parent) common = { - "service": app.settings.service_url, - "parent": args.parent, - "parent_generation": parent.generation, - "operator": args.operator, - "expires_at": wire(app.clock() + timedelta(hours=2)), + 'service': app.settings.service_url, + 'parent': args.parent, + 'parent_generation': parent.generation, + 'operator': args.operator, + 'expires_at': wire(app.clock() + timedelta(hours=2)), } - if args.command == "prepare-content": + if args.command == 'prepare-content': needed = content.identity_requirements( args.snapshot, args.sha256, {}, summary_only=False - )["required"] - mapping = {old: None for old in needed} if args.unmapped else read_json(args.mappings) - require(set(mapping) == set(needed), "legacy_identity_mapping_incomplete") + )['required'] + mapping = dict.fromkeys(needed) if args.unmapped else read_json(args.mappings) + require(set(mapping) == set(needed), 'legacy_identity_mapping_incomplete') approval = content.approval_payload(sha256=args.sha256, identities=mapping, **common) - elif args.command == "prepare-git": + elif args.command == 'prepare-git': manifest = read_json(args.manifest) verified = git.verify_bundle( args.bundle, - manifest["bundle_sha256"], + manifest['bundle_sha256'], protected_work=app.settings.server.staging_dir, ) require( - verified["refs"] == manifest["refs"] and manifest.get("refs_stable") is True, - "legacy_git_refs_mismatch", + verified['refs'] == manifest['refs'] and manifest.get('refs_stable') is True, + 'legacy_git_refs_mismatch', ) approval = git.git_approval( - bundle_sha256=verified["bundle_sha256"], - refs_digest=verified["refs_digest"], - default_ref=manifest["default_ref"], + bundle_sha256=verified['bundle_sha256'], + refs_digest=verified['refs_digest'], + default_ref=manifest['default_ref'], name=args.name, **common, ) else: plan = read_json(args.plan) - needed = {row["old_id"] for row in plan["identities"]} - mapping = {old: None for old in needed} if args.unmapped else read_json(args.mappings) - require(set(mapping) == needed, "legacy_identity_mapping_incomplete") + needed = {row['old_id'] for row in plan['identities']} + mapping = dict.fromkeys(needed) if args.unmapped else read_json(args.mappings) + require(set(mapping) == needed, 'legacy_identity_mapping_incomplete') approval = identities.approval_payload( plan_digest=digest(plan), mappings=mapping, **common ) @@ -246,7 +248,7 @@ async def apply_identities(args): await app.load() envelope = read_json(args.signed_approval) return await identities.import_identity_records( - app, read_json(args.plan), envelope["approval"], envelope["signature"] + app, read_json(args.plan), envelope['approval'], envelope['signature'] ) finally: await app.close() @@ -254,45 +256,45 @@ async def apply_identities(args): def parser(): command = argparse.ArgumentParser(description=__doc__) - command.add_argument("--config-dir", type=Path, required=True) - sub = command.add_subparsers(dest="command", required=True) - for kind in ("content", "git", "identities"): - item = sub.add_parser("prepare-" + kind) - item.add_argument("--operator", required=True) - item.add_argument("--parent", required=True) - item.add_argument("--output", type=Path, required=True) - if kind == "content": - item.add_argument("--snapshot", type=Path, required=True) - item.add_argument("--sha256", required=True) - elif kind == "git": - item.add_argument("--bundle", type=Path, required=True) - item.add_argument("--manifest", type=Path, required=True) - item.add_argument("--name", required=True) + command.add_argument('--config-dir', type=Path, required=True) + sub = command.add_subparsers(dest='command', required=True) + for kind in ('content', 'git', 'identities'): + item = sub.add_parser('prepare-' + kind) + item.add_argument('--operator', required=True) + item.add_argument('--parent', required=True) + item.add_argument('--output', type=Path, required=True) + if kind == 'content': + item.add_argument('--snapshot', type=Path, required=True) + item.add_argument('--sha256', required=True) + elif kind == 'git': + item.add_argument('--bundle', type=Path, required=True) + item.add_argument('--manifest', type=Path, required=True) + item.add_argument('--name', required=True) else: - item.add_argument("--plan", type=Path, required=True) - if kind != "git": + item.add_argument('--plan', type=Path, required=True) + if kind != 'git': mapping = item.add_mutually_exclusive_group(required=True) - mapping.add_argument("--mappings", type=Path) - mapping.add_argument("--unmapped", action="store_true") - sign = sub.add_parser("sign") - sign.add_argument("--draft", type=Path, required=True) - sign.add_argument("--output", type=Path, required=True) - apply = sub.add_parser("apply-identities") - apply.add_argument("--plan", type=Path, required=True) - apply.add_argument("--signed-approval", type=Path, required=True) + mapping.add_argument('--mappings', type=Path) + mapping.add_argument('--unmapped', action='store_true') + sign = sub.add_parser('sign') + sign.add_argument('--draft', type=Path, required=True) + sign.add_argument('--output', type=Path, required=True) + apply = sub.add_parser('apply-identities') + apply.add_argument('--plan', type=Path, required=True) + apply.add_argument('--signed-approval', type=Path, required=True) return command def main(): args = parser().parse_args() - if args.command == "sign": + if args.command == 'sign': result = sign_approval(args.config_dir, args.draft, args.output) - elif args.command == "apply-identities": + elif args.command == 'apply-identities': result = asyncio.run(apply_identities(args)) else: result = asyncio.run(prepare(args)) print(json.dumps(result, sort_keys=True)) -if __name__ == "__main__": +if __name__ == '__main__': main() diff --git a/src/msg/admin/market.py b/src/msg/admin/market.py index 1bc971f0..a045ba78 100644 --- a/src/msg/admin/market.py +++ b/src/msg/admin/market.py @@ -1,4 +1,5 @@ """Root-signed, physical-console arbitration configuration; never a network API.""" + from __future__ import annotations import asyncio @@ -17,46 +18,86 @@ async def apply_market(app, signer, *, action, operator, subject=None, policy=None, expected=None): _root_signer(app, signer) - require(action in {'grant','revoke','publish'}, 'market_admin_action_invalid') + require(action in {'grant', 'revoke', 'publish'}, 'market_admin_action_invalid') if action == 'publish': policy = validate(policy) require(policy['id'] != DEFAULT_POLICY['id'], 'builtin_policy_immutable') async with app.metadata.transaction(write=True) as tx: if action == 'publish': - require(tx.one('SELECT 1 FROM arbitration_policies WHERE id=?', (policy['id'],)) is None, - 'arbitration_policy_exists') + require( + tx.one('SELECT 1 FROM arbitration_policies WHERE id=?', (policy['id'],)) is None, + 'arbitration_policy_exists', + ) for candidate in policy['candidates']: identity = await tx.subject(candidate) - require(identity.kind == 'registered' and not identity.local_only, 'invalid_arbitrator') - tx.execute('INSERT INTO arbitration_policies(id,body,digest) VALUES (?,?,?)', - (policy['id'],canonical(policy).decode(),digest(policy)), write=True) + require( + identity.kind == 'registered' and not identity.local_only, 'invalid_arbitrator' + ) + tx.execute( + 'INSERT INTO arbitration_policies(id,body,digest) VALUES (?,?,?)', + (policy['id'], canonical(policy).decode(), digest(policy)), + write=True, + ) before, after = None, policy else: identity = await tx.subject(subject) - require(identity.kind == 'registered' and not identity.local_only and subject != ROOT_SUBJECT, - 'invalid_arbitrator') + require( + identity.kind == 'registered' + and not identity.local_only + and subject != ROOT_SUBJECT, + 'invalid_arbitrator', + ) row = tx.one('SELECT epoch,active FROM arbitrator_roles WHERE subject=?', (subject,)) - before = {'epoch':row[0],'active':row[1]} if row else None + before = {'epoch': row[0], 'active': row[1]} if row else None if expected is not None: require(before == expected, 'market_preview_stale') - require((action=='grant' and (not row or not row[1])) or - (action=='revoke' and row and row[1]), 'arbitrator_role_unchanged') - after = {'epoch': new_id('role') if action=='grant' else row[0], 'active': action=='grant'} - tx.execute('''INSERT INTO arbitrator_roles(subject,epoch,active) VALUES (?,?,?) - ON CONFLICT(subject) DO UPDATE SET epoch=excluded.epoch,active=excluded.active''', - (subject,after['epoch'],after['active']), write=True) - statement = {'action':action,'subject':subject,'before':before,'after':after, - 'operator':operator,'at':wire(app.clock())} + require( + (action == 'grant' and (not row or not row[1])) + or (action == 'revoke' and row and row[1]), + 'arbitrator_role_unchanged', + ) + after = { + 'epoch': new_id('role') if action == 'grant' else row[0], + 'active': action == 'grant', + } + tx.execute( + """INSERT INTO arbitrator_roles(subject,epoch,active) VALUES (?,?,?) + ON CONFLICT(subject) DO UPDATE SET epoch=excluded.epoch,active=excluded.active""", + (subject, after['epoch'], after['active']), + write=True, + ) + statement = { + 'action': action, + 'subject': subject, + 'before': before, + 'after': after, + 'operator': operator, + 'at': wire(app.clock()), + } proof = signer.sign(canonical(statement), purpose='market-admin') - event = Event(id=new_id('audit'), type='root.market.'+action, time=app.clock(), - request_id=new_id('local-market'), actor=ROOT_SUBJECT, subject=ROOT_SUBJECT, - resources=(ResourceRef(id=ROOT_SUBJECT),), data={**statement,'root_signature':wire(proof)}) + event = Event( + id=new_id('audit'), + type='root.market.' + action, + time=app.clock(), + request_id=new_id('local-market'), + actor=ROOT_SUBJECT, + subject=ROOT_SUBJECT, + resources=(ResourceRef(id=ROOT_SUBJECT),), + data={**statement, 'root_signature': wire(proof)}, + ) await tx.append_event(event) - await tx.append_audit(AuditEvent(event=event, - authority=(ResourceRef(id=app.certificates.root_certificate.resource_id),), - before_digest=digest(before),after_digest=digest(after),previous_digest=None, - entry_digest='',result=action)) - return {'action':action,'audit_event_id':event.id,'result':after} + await tx.append_audit( + AuditEvent( + event=event, + authority=(ResourceRef(id=app.certificates.root_certificate.resource_id),), + before_digest=digest(before), + after_digest=digest(after), + previous_digest=None, + entry_digest='', + result=action, + ) + ) + return {'action': action, 'audit_event_id': event.id, 'result': after} class MarketAdmin: @@ -66,6 +107,7 @@ def __init__(self, config_dir): def execute(self, action, *, subject=None, policy_file=None): operator = require_local_console(self.config_dir) app = RootAdmin(self.config_dir)._app() + async def run(): await app.load() try: @@ -73,19 +115,37 @@ async def run(): resolved, before = subject, None async with app.metadata.transaction(write=False) as tx: if subject and subject.startswith('@'): - resolved = await tx.resolve('/'+subject) + resolved = await tx.resolve('/' + subject) if resolved: - row = tx.one('SELECT epoch,active FROM arbitrator_roles WHERE subject=?', (resolved,)) - before = {'epoch':row[0],'active':row[1]} if row else None - preview = {'action':action,'subject':resolved,'before':before,'policy':policy} + row = tx.one( + 'SELECT epoch,active FROM arbitrator_roles WHERE subject=?', (resolved,) + ) + before = {'epoch': row[0], 'active': row[1]} if row else None + preview = { + 'action': action, + 'subject': resolved, + 'before': before, + 'policy': policy, + } print(canonical(preview).decode()) - approval = 'CONFIRM MARKET '+digest(preview) - require(input('Type '+approval+' to continue: ') == approval, 'approval_cancelled') + approval = 'CONFIRM MARKET ' + digest(preview) + require( + input('Type ' + approval + ' to continue: ') == approval, 'approval_cancelled' + ) pin = getpass.getpass('Root PIN/passphrase: ') - signer = Ed25519Signer.from_bytes(open_private_key( - loads(root_envelope(self.config_dir).read_bytes()), pin)) - return await apply_market(app, signer, action=action, operator=operator, - subject=resolved,policy=policy,expected=before) + signer = Ed25519Signer.from_bytes( + open_private_key(loads(root_envelope(self.config_dir).read_bytes()), pin) + ) + return await apply_market( + app, + signer, + action=action, + operator=operator, + subject=resolved, + policy=policy, + expected=before, + ) finally: await app.close() + return asyncio.run(run()) diff --git a/src/msg/admin/market_check.py b/src/msg/admin/market_check.py index e40fdcc2..0a920077 100644 --- a/src/msg/admin/market_check.py +++ b/src/msg/admin/market_check.py @@ -1,4 +1,5 @@ """Read-only financial invariants and a deliberately isolated Test Root scenario.""" + from __future__ import annotations from copy import deepcopy @@ -16,161 +17,365 @@ async def inspect_market(tx): """No initialization, repair, timer, statement, event, job or network call.""" count = 0 - for oid,buyer,total,escrow,state in tx.rows( - 'SELECT id,buyer,total_price_minor,escrow_subject,state FROM store_orders ORDER BY id'): - expected = total if state in {'funded','delivered','accepted','disputed'} else 0 - require(_balance(tx,escrow)==expected, 'market_escrow_invariant') - require(tx.one('SELECT 1 FROM identities WHERE id=?',(escrow,)) is None and - tx.one('SELECT kind,subject_id FROM ledger_accounts WHERE id=?',(escrow,))==('order_escrow',None), - 'market_escrow_identity') - snapshot = tx.one('SELECT body FROM order_contracts WHERE order_id=?',(oid,)) + for oid, buyer, total, escrow, state in tx.rows( + 'SELECT id,buyer,total_price_minor,escrow_subject,state FROM store_orders ORDER BY id' + ): + expected = total if state in {'funded', 'delivered', 'accepted', 'disputed'} else 0 + require(_balance(tx, escrow) == expected, 'market_escrow_invariant') + require( + tx.one('SELECT 1 FROM identities WHERE id=?', (escrow,)) is None + and tx.one('SELECT kind,subject_id FROM ledger_accounts WHERE id=?', (escrow,)) + == ('order_escrow', None), + 'market_escrow_identity', + ) + snapshot = tx.one('SELECT body FROM order_contracts WHERE order_id=?', (oid,)) if snapshot: - locked = contract(tx,oid) + locked = contract(tx, oid) policy = locked['policy'] if locked['version'] != 5: validate(policy['policy']) - require(digest(policy['policy'])==policy['policy_digest'] and - locked['buyer']==buyer and locked['total_price_minor']==total, 'market_snapshot_invariant') - row = tx.one('SELECT body FROM order_settlements WHERE order_id=?',(oid,)) + require( + digest(policy['policy']) == policy['policy_digest'] + and locked['buyer'] == buyer + and locked['total_price_minor'] == total, + 'market_snapshot_invariant', + ) + row = tx.one('SELECT body FROM order_settlements WHERE order_id=?', (oid,)) if row: fact = loads(row[0]) - require(state in {'settled','refunded'} and fact['refund_minor']+fact['release_minor']==total, - 'market_settlement_invariant') - actual = tx.one("SELECT COALESCE(SUM(amount_minor),0) FROM money_ledger WHERE debit_account=?",(escrow,))[0] - require(actual==total,'market_settlement_invariant') + require( + state in {'settled', 'refunded'} + and fact['refund_minor'] + fact['release_minor'] == total, + 'market_settlement_invariant', + ) + actual = tx.one( + 'SELECT COALESCE(SUM(amount_minor),0) FROM money_ledger WHERE debit_account=?', + (escrow,), + )[0] + require(actual == total, 'market_settlement_invariant') elif snapshot: - require(state not in {'settled','refunded'},'market_settlement_missing') + require(state not in {'settled', 'refunded'}, 'market_settlement_missing') count += 1 - for raw,expected in tx.rows('SELECT body,digest FROM arbitration_policies'): - policy = loads(raw); validate(policy) - require(digest(policy)==expected,'arbitration_policy_corrupt') - for case_id,order_id,state in tx.rows('SELECT id,order_id,state FROM arbitration_cases'): - settlement = tx.one('SELECT decision_id FROM order_settlements WHERE order_id=?',(order_id,)) - if state=='executed': - require(settlement is not None and settlement[0] is not None,'market_decision_unfunded') + for raw, expected in tx.rows('SELECT body,digest FROM arbitration_policies'): + policy = loads(raw) + validate(policy) + require(digest(policy) == expected, 'arbitration_policy_corrupt') + for case_id, order_id, state in tx.rows('SELECT id,order_id,state FROM arbitration_cases'): + settlement = tx.one( + 'SELECT decision_id FROM order_settlements WHERE order_id=?', (order_id,) + ) + if state == 'executed': + require( + settlement is not None and settlement[0] is not None, 'market_decision_unfunded' + ) from msg.market.arbitration import _read from msg.market.rationale import record - row=tx.one('SELECT body FROM arbitration_decisions WHERE id=? AND case_id=?', - (settlement[0],case_id)) - require(row is not None,'market_decision_missing') - decision=loads(row[0]) - require(all(decision.get(k)==decision['proposal'].get(k) - for k in ('rationale_ref','rationale_digest')),'decision_rationale_mismatch') - record(tx,_read(tx,case_id),decision['proposal']) - return {'orders':count,'held_cases':tx.one("SELECT COUNT(*) FROM arbitration_cases WHERE state='held'")[0], - 'escrow_invariants':True,'private_reads_only':True} - - -async def check_market(app,root,call,register,now): + + row = tx.one( + 'SELECT body FROM arbitration_decisions WHERE id=? AND case_id=?', + (settlement[0], case_id), + ) + require(row is not None, 'market_decision_missing') + decision = loads(row[0]) + require( + all( + decision.get(k) == decision['proposal'].get(k) + for k in ('rationale_ref', 'rationale_digest') + ), + 'decision_rationale_mismatch', + ) + record(tx, _read(tx, case_id), decision['proposal']) + return { + 'orders': count, + 'held_cases': tx.one("SELECT COUNT(*) FROM arbitration_cases WHERE state='held'")[0], + 'escrow_invariants': True, + 'private_reads_only': True, + } + + +async def check_market(app, root, call, register, now): """20 MSG issue -> bank-funded 10 reward -> 5 bundle, entirely disposable.""" from msg.admin.market import apply_market from msg.admin.money import apply_money from msg.market.policy import DEFAULT_POLICY from msg.workers.effects import EffectWorker - require(app.selftest_run_id is not None,'selftest_namespace_required') + + require(app.selftest_run_id is not None, 'selftest_namespace_required') async with app.metadata.transaction(write=False) as tx: baseline_supply = _supply(tx) - bank,bank_id = await register('market-test-bank') - buyer,buyer_id = await register('market-test-buyer') - async def invoke(name,args,key=buyer,subject=buyer_id,**kw): - result = await call(name,args,key,subject,**kw) - require(result.status=='ok',result.error.code if result.error else 'market_selftest_failed') + bank, bank_id = await register('market-test-bank') + buyer, buyer_id = await register('market-test-buyer') + + async def invoke(name, args, key=buyer, subject=buyer_id, **kw): + result = await call(name, args, key, subject, **kw) + require( + result.status == 'ok', result.error.code if result.error else 'market_selftest_failed' + ) return result - await apply_money(app,root,action='bank_add',operator='isolated-selftest',subject_id=bank_id) - await apply_money(app,root,action='mint',operator='isolated-selftest',amount_minor=20_000_000) - await apply_money(app,root,action='transfer',operator='isolated-selftest',subject_id=bank_id,amount_minor=20_000_000) - bounty = await invoke('bounty.create',{'name':'identity-test','terms':'One-use signature test', - 'reward_minor':10_000_000,'budget_minor':10_000_000,'max_claims':1},bank,bank_id) - challenge = await invoke('bounty.challenge',{'listing_id':bounty.data['bounty']['listing_id']}) + + await apply_money( + app, root, action='bank_add', operator='isolated-selftest', subject_id=bank_id + ) + await apply_money( + app, root, action='mint', operator='isolated-selftest', amount_minor=20_000_000 + ) + await apply_money( + app, + root, + action='transfer', + operator='isolated-selftest', + subject_id=bank_id, + amount_minor=20_000_000, + ) + bounty = await invoke( + 'bounty.create', + { + 'name': 'identity-test', + 'terms': 'One-use signature test', + 'reward_minor': 10_000_000, + 'budget_minor': 10_000_000, + 'max_claims': 1, + }, + bank, + bank_id, + ) + challenge = await invoke( + 'bounty.challenge', {'listing_id': bounty.data['bounty']['listing_id']} + ) payload = challenge.data['challenge'] - await invoke('bounty.claim',{'challenge_id':payload['challenge_id'], - 'proof':wire(buyer.sign(canonical(payload),purpose='bounty-pop-v1'))}) - listing = await invoke('store.listing_create',{'name':'store-selftest','item_kind':'bundle', - 'price_minor':5_000_000,'currency_id':'primary','quantity':1,'delivery_mode':'managed_instant', - 'escrow_policy':'escrow-v1','dispute_policy':'dispute-v1','terms':'Immutable test bundle.'},bank,bank_id) + await invoke( + 'bounty.claim', + { + 'challenge_id': payload['challenge_id'], + 'proof': wire(buyer.sign(canonical(payload), purpose='bounty-pop-v1')), + }, + ) + listing = await invoke( + 'store.listing_create', + { + 'name': 'store-selftest', + 'item_kind': 'bundle', + 'price_minor': 5_000_000, + 'currency_id': 'primary', + 'quantity': 1, + 'delivery_mode': 'managed_instant', + 'escrow_policy': 'escrow-v1', + 'dispute_policy': 'dispute-v1', + 'terms': 'Immutable test bundle.', + }, + bank, + bank_id, + ) async with app.metadata.transaction(write=False) as tx: - parent = tx.one("SELECT id FROM resources WHERE parent=? AND name='files'",(bank_id,))[0] - file = await invoke('content.file_put',{'parent':parent,'name':'hello.txt','media_type':'text/plain', - 'data':b64(b'delivery-ok\n')},bank,bank_id) - package = await invoke('store.package_deposit',{'listing_id':listing.resources[0].id, - 'listing_revision':listing.resources[0].revision,'manifest':{'text':'msg.lmm.best store selftest'}, - 'payload_refs':[wire(file.resources[0])]},bank,bank_id) - active = await invoke('store.listing_update',{'id':listing.resources[0].id,'state':'active', - 'package_ref':package.data['package']['id']},bank,bank_id, - expected=((listing.resources[0].id,listing.data['generation']),)) + parent = tx.one("SELECT id FROM resources WHERE parent=? AND name='files'", (bank_id,))[0] + file = await invoke( + 'content.file_put', + { + 'parent': parent, + 'name': 'hello.txt', + 'media_type': 'text/plain', + 'data': b64(b'delivery-ok\n'), + }, + bank, + bank_id, + ) + package = await invoke( + 'store.package_deposit', + { + 'listing_id': listing.resources[0].id, + 'listing_revision': listing.resources[0].revision, + 'manifest': {'text': 'msg.lmm.best store selftest'}, + 'payload_refs': [wire(file.resources[0])], + }, + bank, + bank_id, + ) + active = await invoke( + 'store.listing_update', + { + 'id': listing.resources[0].id, + 'state': 'active', + 'package_ref': package.data['package']['id'], + }, + bank, + bank_id, + expected=((listing.resources[0].id, listing.data['generation']),), + ) old_settings = app.settings + class NoNetworkMail: - def __init__(self): self.messages=[] - async def send(self,job): - self.messages.append(dict(job.arguments)); return 'sent' - sender=NoNetworkMail() - app.settings=replace(app.settings,server=replace(app.settings.server,mail=MailConfig(enabled=True, - host='selftest.invalid',port=2525,tls='starttls',sender='test@selftest.invalid',credential_file=None))) + def __init__(self): + self.messages = [] + + async def send(self, job): + self.messages.append(dict(job.arguments)) + return 'sent' + + sender = NoNetworkMail() + app.settings = replace( + app.settings, + server=replace( + app.settings.server, + mail=MailConfig( + enabled=True, + host='selftest.invalid', + port=2525, + tls='starttls', + sender='test@selftest.invalid', + credential_file=None, + ), + ), + ) try: sale = active.data['listing'] - bought = await invoke('orders.buy',{'listing_id':sale['listing_id'], - 'listing_revision':sale['listing_revision'],'quantity':1,'total_price_minor':5_000_000, - 'currency_id':'primary','email':'buyer@selftest.invalid'},contract_version=3) + bought = await invoke( + 'orders.buy', + { + 'listing_id': sale['listing_id'], + 'listing_revision': sale['listing_revision'], + 'quantity': 1, + 'total_price_minor': 5_000_000, + 'currency_id': 'primary', + 'email': 'buyer@selftest.invalid', + }, + contract_version=3, + ) oid = bought.data['order']['id'] - worker=EffectWorker(app,mail_sender=sender) - while await worker.run_once(): pass - require(len(sender.messages)==1 and oid not in sender.messages[0]['text'],'market_email_leak') - proof=loads(sender.messages[0]['text']) - await invoke('orders.email_verify',{'order_id':oid,'challenge_id':proof['challenge_id'],'token':proof['token']}) - while await worker.run_once(): pass - require(len(sender.messages)==2 and set(loads(sender.messages[1]['text']))=={'order_id','handle','pickup'}, - 'market_email_leak') - delivery=await invoke('delivery.get',{'order_id':oid},contract_version=2) - require(delivery.data['delivery']['state']=='prepared' and - delivery.data['delivery']['payloads'][0]['data']==b64(b'delivery-ok\n'),'market_claim_invariant') + worker = EffectWorker(app, mail_sender=sender) + while await worker.run_once(): + pass + require( + len(sender.messages) == 1 and oid not in sender.messages[0]['text'], 'market_email_leak' + ) + proof = loads(sender.messages[0]['text']) + await invoke( + 'orders.email_verify', + {'order_id': oid, 'challenge_id': proof['challenge_id'], 'token': proof['token']}, + ) + while await worker.run_once(): + pass + require( + len(sender.messages) == 2 + and set(loads(sender.messages[1]['text'])) == {'order_id', 'handle', 'pickup'}, + 'market_email_leak', + ) + delivery = await invoke('delivery.get', {'order_id': oid}, contract_version=2) + require( + delivery.data['delivery']['state'] == 'prepared' + and delivery.data['delivery']['payloads'][0]['data'] == b64(b'delivery-ok\n'), + 'market_claim_invariant', + ) finally: - app.settings=old_settings + app.settings = old_settings # A separate 1 MSG service dispute refunds the test buyer through a real # fixed 2-of-3 panel. It leaves the official 5/15 final balances unchanged. - members={} + members = {} for i in range(3): - key,uid=await register('market-test-arbitrator-'+str(i)); members[uid]=key - await apply_market(app,root,action='grant',operator='isolated-selftest',subject=uid) - policy=deepcopy(DEFAULT_POLICY); policy.update(id='selftest-panel-v1',candidates=list(members)) - await apply_market(app,root,action='publish',operator='isolated-selftest',policy=policy) - service=await invoke('store.listing_create',{'name':'case-selftest','item_kind':'service', - 'price_minor':1_000_000,'currency_id':'primary','quantity':1,'delivery_mode':'service', - 'escrow_policy':'escrow-v1','dispute_policy':policy['id'],'terms':'Test service dispute.'},bank,bank_id) - service=await invoke('store.listing_update',{'id':service.resources[0].id,'state':'active'},bank,bank_id, - expected=((service.resources[0].id,service.data['generation']),)) - sale=service.data['listing'] - bought=await invoke('orders.buy',{'listing_id':sale['listing_id'],'listing_revision':sale['listing_revision'], - 'quantity':1,'total_price_minor':1_000_000,'currency_id':'primary'},contract_version=3) - opened=await invoke('orders.dispute_open',{'order_id':bought.data['order']['id'],'reason':'quality'}) - case=await invoke('orders.dispute_get',{'case_id':opened.data['case_id']}) - author=opened.data['panel'][0] + key, uid = await register('market-test-arbitrator-' + str(i)) + members[uid] = key + await apply_market(app, root, action='grant', operator='isolated-selftest', subject=uid) + policy = deepcopy(DEFAULT_POLICY) + policy.update(id='selftest-panel-v1', candidates=list(members)) + await apply_market(app, root, action='publish', operator='isolated-selftest', policy=policy) + service = await invoke( + 'store.listing_create', + { + 'name': 'case-selftest', + 'item_kind': 'service', + 'price_minor': 1_000_000, + 'currency_id': 'primary', + 'quantity': 1, + 'delivery_mode': 'service', + 'escrow_policy': 'escrow-v1', + 'dispute_policy': policy['id'], + 'terms': 'Test service dispute.', + }, + bank, + bank_id, + ) + service = await invoke( + 'store.listing_update', + {'id': service.resources[0].id, 'state': 'active'}, + bank, + bank_id, + expected=((service.resources[0].id, service.data['generation']),), + ) + sale = service.data['listing'] + bought = await invoke( + 'orders.buy', + { + 'listing_id': sale['listing_id'], + 'listing_revision': sale['listing_revision'], + 'quantity': 1, + 'total_price_minor': 1_000_000, + 'currency_id': 'primary', + }, + contract_version=3, + ) + opened = await invoke( + 'orders.dispute_open', {'order_id': bought.data['order']['id'], 'reason': 'quality'} + ) + case = await invoke('orders.dispute_get', {'case_id': opened.data['case_id']}) + author = opened.data['panel'][0] async with app.metadata.transaction(write=False) as tx: - parent=tx.one("SELECT id FROM resources WHERE parent=? AND name='files'",(author,))[0] - reason=await invoke('content.file_put',{'parent':parent,'name':'case-reason.txt', - 'media_type':'text/plain','data':b64(b'Selftest evidence warrants a full refund.')},members[author],author) - bound=await invoke('orders.dispute_rationale',{'case_id':opened.data['case_id'], - 'ref':wire(reason.resources[0])},members[author],author) - proposal={**case.data['proposal_base'],'outcome':'refund','refund_minor':1_000_000, - **{name:bound.data[name] for name in ('rationale_ref','rationale_digest')}} + parent = tx.one("SELECT id FROM resources WHERE parent=? AND name='files'", (author,))[0] + reason = await invoke( + 'content.file_put', + { + 'parent': parent, + 'name': 'case-reason.txt', + 'media_type': 'text/plain', + 'data': b64(b'Selftest evidence warrants a full refund.'), + }, + members[author], + author, + ) + bound = await invoke( + 'orders.dispute_rationale', + {'case_id': opened.data['case_id'], 'ref': wire(reason.resources[0])}, + members[author], + author, + ) + proposal = { + **case.data['proposal_base'], + 'outcome': 'refund', + 'refund_minor': 1_000_000, + **{name: bound.data[name] for name in ('rationale_ref', 'rationale_digest')}, + } for uid in opened.data['panel'][:2]: - voted=await invoke('orders.dispute_vote',{'proposal':proposal, - 'signature':wire(members[uid].sign(canonical(proposal),purpose='arbitration-decision'))},members[uid],uid) - args={'case_id':opened.data['case_id'],'decision_id':voted.data['decision']['id']} - await invoke('orders.dispute_execute',args) - await invoke('orders.dispute_execute',args) + voted = await invoke( + 'orders.dispute_vote', + { + 'proposal': proposal, + 'signature': wire( + members[uid].sign(canonical(proposal), purpose='arbitration-decision') + ), + }, + members[uid], + uid, + ) + args = {'case_id': opened.data['case_id'], 'decision_id': voted.data['decision']['id']} + await invoke('orders.dispute_execute', args) + await invoke('orders.dispute_execute', args) async with app.metadata.transaction(write=False) as tx: await inspect_market(tx) - require(_balance(tx,buyer_id)==5_000_000 and _balance(tx,bank_id)==15_000_000 and - _supply(tx)==baseline_supply+20_000_000, - 'market_conservation_failed') - for (escrow,) in tx.rows("SELECT id FROM ledger_accounts WHERE kind IN ('order_escrow','bounty_escrow')"): - require(_balance(tx,escrow)==0,'market_escrow_invariant') + require( + _balance(tx, buyer_id) == 5_000_000 + and _balance(tx, bank_id) == 15_000_000 + and _supply(tx) == baseline_supply + 20_000_000, + 'market_conservation_failed', + ) + for (escrow,) in tx.rows( + "SELECT id FROM ledger_accounts WHERE kind IN ('order_escrow','bounty_escrow')" + ): + require(_balance(tx, escrow) == 0, 'market_escrow_invariant') return True async def inspect_bounty_contracts(app, tx): """Check restored accounting projections against immutable published terms.""" from msg.plugins.bounty import _row, validate_contract + rows = tx.rows('SELECT listing_id FROM bounty_listings ORDER BY listing_id') if not rows: return @@ -178,62 +383,133 @@ async def inspect_bounty_contracts(app, tx): if app.contents is None: # doctor intentionally never opens the application writer stores. from types import SimpleNamespace + from msg.storage.read_only import GitContentReader - reader = SimpleNamespace(contents=GitContentReader( - app.settings.server.content_dir, binary_dir=app.settings.server.blob_dir)) + + reader = SimpleNamespace( + contents=GitContentReader( + app.settings.server.content_dir, binary_dir=app.settings.server.blob_dir + ) + ) for (listing_id,) in rows: await validate_contract(reader, tx, _row(tx, listing_id)) def inspect_clearing(app, tx): """Never migrate, mint, seed a bank, repair a row or unlock the Root key.""" - tables = ('ledger_accounts','money_ledger','money_bank_roles','server_offers', - 'money_purchases','resource_entitlements','bounty_listings','bounty_claims', - 'store_orders','store_deliveries','store_order_events') + tables = ( + 'ledger_accounts', + 'money_ledger', + 'money_bank_roles', + 'server_offers', + 'money_purchases', + 'resource_entitlements', + 'bounty_listings', + 'bounty_claims', + 'store_orders', + 'store_deliveries', + 'store_order_events', + ) for table in tables: - require(tx.one('SELECT to_regclass(?)',(table,))[0] is not None, 'market_schema_missing') - require(tx.one("SELECT 1 FROM information_schema.columns WHERE table_name='money_ledger' " - "AND column_name='entry_key'") is not None, 'market_schema_missing') - balances = {account:int(amount) for account,amount in tx.rows(''' + require(tx.one('SELECT to_regclass(?)', (table,))[0] is not None, 'market_schema_missing') + require( + tx.one( + "SELECT 1 FROM information_schema.columns WHERE table_name='money_ledger' " + "AND column_name='entry_key'" + ) + is not None, + 'market_schema_missing', + ) + balances = { + account: int(amount) + for account, amount in tx.rows(""" SELECT account,SUM(amount) FROM ( SELECT credit_account account,amount_minor amount FROM money_ledger WHERE credit_account IS NOT NULL UNION ALL SELECT debit_account account,-amount_minor amount FROM money_ledger WHERE debit_account IS NOT NULL - ) entries GROUP BY account''')} + ) entries GROUP BY account""") + } supply = _supply(tx) - require(0 <= supply <= MAX_MINOR and all(0 <= n <= MAX_MINOR for n in balances.values()) - and sum(balances.values()) == supply, 'money_conservation_failure') - require(tx.one("SELECT 1 FROM ledger_accounts a JOIN identities i ON i.id=a.id " - "WHERE a.kind<>'subject' LIMIT 1") is None, 'escrow_has_identity') - public = Ed25519Signer.from_bytes((app.settings.service_keys/'receipt.key').read_bytes()).public_key + require( + 0 <= supply <= MAX_MINOR + and all(0 <= n <= MAX_MINOR for n in balances.values()) + and sum(balances.values()) == supply, + 'money_conservation_failure', + ) + require( + tx.one( + 'SELECT 1 FROM ledger_accounts a JOIN identities i ON i.id=a.id ' + "WHERE a.kind<>'subject' LIMIT 1" + ) + is None, + 'escrow_has_identity', + ) + public = Ed25519Signer.from_bytes( + (app.settings.service_keys / 'receipt.key').read_bytes() + ).public_key ledger = {} - for row in tx.rows('''SELECT id,kind,amount_minor,debit_account,credit_account,seq, - policy_version,policy_digest,receipt,actor,request_id,entry_key FROM money_ledger'''): - receipt = loads(row[8]); body = receipt['body'] - expected = dict(zip(('transaction_id','kind','amount_minor','from_subject','to_subject', - 'ledger_sequence','policy_version','policy_digest'),row[:8])) - require(all(body.get(k)==v for k,v in expected.items()), 'money_receipt_mismatch') + for row in tx.rows("""SELECT id,kind,amount_minor,debit_account,credit_account,seq, + policy_version,policy_digest,receipt,actor,request_id,entry_key FROM money_ledger"""): + receipt = loads(row[8]) + body = receipt['body'] + expected = dict( + zip( + ( + 'transaction_id', + 'kind', + 'amount_minor', + 'from_subject', + 'to_subject', + 'ledger_sequence', + 'policy_version', + 'policy_digest', + ), + row[:8], + strict=True, + ) + ) + require(all(body.get(k) == v for k, v in expected.items()), 'money_receipt_mismatch') if row[6] >= 2: - require((body.get('actor'),body.get('request_id'),body.get('entry_key')) == row[9:], - 'money_receipt_mismatch') - verify(public,canonical(body),decode(Signature,receipt['signature']),purpose='money-receipt') + require( + (body.get('actor'), body.get('request_id'), body.get('entry_key')) == row[9:], + 'money_receipt_mismatch', + ) + verify( + public, + canonical(body), + decode(Signature, receipt['signature']), + purpose='money-receipt', + ) ledger[row[0]] = body - for oid,escrow,total,state,quote,refs in tx.rows('''SELECT id,escrow_subject,total_price_minor, - state,quote_digest,receipt_refs FROM store_orders'''): + for ( + oid, + escrow, + total, + state, + quote, + refs, + ) in tx.rows("""SELECT id,escrow_subject,total_price_minor, + state,quote_digest,receipt_refs FROM store_orders"""): require(state != 'accepted', 'incomplete_order_commit') immutable = tx.one('SELECT body FROM order_contracts WHERE order_id=?', (oid,)) if immutable is not None and loads(immutable[0]).get('version') == 5: require(state == 'settled', 'incomplete_entitlement_commit') - expected = total if state in {'funded','delivered','disputed'} else 0 - require(balances.get(escrow,0)==expected, 'order_escrow_mismatch') + expected = total if state in {'funded', 'delivered', 'disputed'} else 0 + require(balances.get(escrow, 0) == expected, 'order_escrow_mismatch') require(all(ref in ledger for ref in loads(refs)), 'order_receipt_missing') if quote is not None: - last=tx.one('SELECT to_state FROM store_order_events WHERE order_id=? ORDER BY seq DESC LIMIT 1',(oid,)) - require(last is not None and last[0]==state, 'order_history_mismatch') - if state=='settled': - delivery=tx.one('SELECT d.state,o.escrow_policy FROM store_deliveries d ' - 'JOIN store_orders o ON o.id=d.order_id WHERE o.id=?',(oid,)) - current=tx.one('SELECT 1 FROM order_contracts WHERE order_id=?',(oid,)) + last = tx.one( + 'SELECT to_state FROM store_order_events WHERE order_id=? ORDER BY seq DESC LIMIT 1', + (oid,), + ) + require(last is not None and last[0] == state, 'order_history_mismatch') + if state == 'settled': + delivery = tx.one( + 'SELECT d.state,o.escrow_policy FROM store_deliveries d ' + 'JOIN store_orders o ON o.id=d.order_id WHERE o.id=?', + (oid,), + ) + current = tx.one('SELECT 1 FROM order_contracts WHERE order_id=?', (oid,)) if current is not None: # V3 managed-instant settlement is a fulfillment/payment fact, # not buyer acknowledgement: its delivery intentionally stays @@ -241,39 +517,73 @@ def inspect_clearing(app, tx): # immutable contract validator instead of weakening this check # for arbitrary prepared legacy orders. from msg.market.policy import contract - locked=contract(tx,oid) + + locked = contract(tx, oid) if locked['version'] == 5: # The contract validator requires the exact entitlement grant; # synchronous issuance must not invent a Delivery or acceptance. require(delivery is None, 'unexpected_entitlement_delivery') else: - prepared=(delivery is not None and delivery[0]=='prepared' and - locked['listing']['delivery_mode']=='managed_instant') - require(delivery is not None and (delivery[0]=='claimed' or prepared), - 'settled_delivery_missing') + prepared = ( + delivery is not None + and delivery[0] == 'prepared' + and locked['listing']['delivery_mode'] == 'managed_instant' + ) + require( + delivery is not None and (delivery[0] == 'claimed' or prepared), + 'settled_delivery_missing', + ) else: - require(delivery is not None and (delivery[0]=='claimed' or - delivery==('prepared','escrow-instant-v1')), 'settled_delivery_missing') - for escrow,total,state,eid in tx.rows('SELECT escrow_account,total_minor,state,entitlement_id FROM money_purchases'): - require(balances.get(escrow,0)==(total if state=='pending' else 0),'purchase_escrow_mismatch') - require((state=='settled') == (eid is not None), 'purchase_entitlement_mismatch') - for lid,escrow,budget,reward,maximum,state in tx.rows('''SELECT listing_id,escrow_subject, - budget_minor,reward_minor,max_claims,state FROM bounty_listings'''): - claims=tx.rows('SELECT claimant,reward_minor,transaction_id,challenge_id FROM bounty_claims WHERE listing_id=?',(lid,)) - require(len(claims)<=maximum, 'bounty_limit_mismatch') - require(balances.get(escrow,0)==(0 if state=='closed' else budget-sum(c[1] for c in claims)), - 'bounty_escrow_mismatch') - for claimant,amount,tid,cid in claims: - receipt=ledger.get(tid,{}) - require(amount==reward and (receipt.get('from_subject'),receipt.get('to_subject'),receipt.get('amount_minor')) - == (escrow,claimant,amount), 'bounty_receipt_mismatch') - challenge=tx.one('SELECT consumed_at FROM bounty_challenges WHERE id=?',(cid,)) + require( + delivery is not None + and (delivery[0] == 'claimed' or delivery == ('prepared', 'escrow-instant-v1')), + 'settled_delivery_missing', + ) + for escrow, total, state, eid in tx.rows( + 'SELECT escrow_account,total_minor,state,entitlement_id FROM money_purchases' + ): + require( + balances.get(escrow, 0) == (total if state == 'pending' else 0), + 'purchase_escrow_mismatch', + ) + require((state == 'settled') == (eid is not None), 'purchase_entitlement_mismatch') + for lid, escrow, budget, reward, maximum, state in tx.rows("""SELECT listing_id,escrow_subject, + budget_minor,reward_minor,max_claims,state FROM bounty_listings"""): + claims = tx.rows( + 'SELECT claimant,reward_minor,transaction_id,challenge_id FROM bounty_claims WHERE listing_id=?', + (lid,), + ) + require(len(claims) <= maximum, 'bounty_limit_mismatch') + require( + balances.get(escrow, 0) + == (0 if state == 'closed' else budget - sum(c[1] for c in claims)), + 'bounty_escrow_mismatch', + ) + for claimant, amount, tid, cid in claims: + receipt = ledger.get(tid, {}) + require( + amount == reward + and ( + receipt.get('from_subject'), + receipt.get('to_subject'), + receipt.get('amount_minor'), + ) + == (escrow, claimant, amount), + 'bounty_receipt_mismatch', + ) + challenge = tx.one('SELECT consumed_at FROM bounty_challenges WHERE id=?', (cid,)) require(challenge is not None and challenge[0] is not None, 'bounty_nonce_mismatch') - return {'currency_id':'primary','scale':6,'total_supply_minor':supply, - 'banks':tx.one("SELECT COUNT(*) FROM money_bank_roles WHERE status='active'")[0], - 'enabled_offers':tx.one('SELECT COUNT(*) FROM server_offers WHERE enabled=TRUE')[0], - 'ledger_entries':len(ledger),'conserved':True,'signed_receipts_verified':True, - 'zero_default':'No mint, bank or offer is created by initialization or inspection.'} + return { + 'currency_id': 'primary', + 'scale': 6, + 'total_supply_minor': supply, + 'banks': tx.one("SELECT COUNT(*) FROM money_bank_roles WHERE status='active'")[0], + 'enabled_offers': tx.one('SELECT COUNT(*) FROM server_offers WHERE enabled=TRUE')[0], + 'ledger_entries': len(ledger), + 'conserved': True, + 'signed_receipts_verified': True, + 'zero_default': 'No mint, bank or offer is created by initialization or inspection.', + } async def check_market_e2e(app, root, call, register): @@ -282,131 +592,283 @@ async def check_market_e2e(app, root, call, register): Production MoneyAdmin still requires an OS-root physical console and hidden PIN. This fixture cannot be selected by configuration or a network request. """ - require(app.selftest_run_id is not None and app.settings.service_url=='https://selftest.invalid', - 'isolated_market_selftest_required') + require( + app.selftest_run_id is not None and app.settings.service_url == 'https://selftest.invalid', + 'isolated_market_selftest_required', + ) from msg.admin.money import _confirmed_money from msg.daemon import parser + async with app.metadata.transaction(write=False) as tx: - baseline=inspect_clearing(app,tx) - require(baseline['total_supply_minor']==0,'selftest_requires_zero_supply') - bank_key,bank=await register('bank-test') - buyer_key,buyer=await register('market-buyer') + baseline = inspect_clearing(app, tx) + require(baseline['total_supply_minor'] == 0, 'selftest_requires_zero_supply') + bank_key, bank = await register('bank-test') + buyer_key, buyer = await register('market-buyer') async def command(argv, expected_actions): - args=parser().parse_args(argv) - action='bank_fund' if args.money_command=='bank' and args.bank_command=='fund' else args.money_command - previews=[]; approvals=[] - def emit(value): previews.append(loads(value)) + args = parser().parse_args(argv) + action = ( + 'bank_fund' + if args.money_command == 'bank' and args.bank_command == 'fund' + else args.money_command + ) + previews = [] + approvals = [] + + def emit(value): + previews.append(loads(value)) + def confirm(prompt): - approval='CONFIRM MONEY '+digest(previews[-1]) - require(prompt=='Type '+approval+' to continue: ', 'selftest_approval_unbound') + approval = 'CONFIRM MONEY ' + digest(previews[-1]) + require(prompt == 'Type ' + approval + ' to continue: ', 'selftest_approval_unbound') approvals.append(previews[-1]['approval_for']) return approval + def unlock(): - require(approvals==expected_actions, 'selftest_unlock_before_confirmation') + require(approvals == expected_actions, 'selftest_unlock_before_confirmation') return root # Fresh disposable Test Root; never opens production key material. - return await _confirmed_money(app,action,operator='isolated-TestConsole',amount=args.amount, - subject_id=getattr(args,'subject_id',None),emit=emit,confirm=confirm,unlock=unlock) - - await command(['money','mint','20'], ['mint']) - funding=await command(['money','bank','fund','@bank-test','20'], ['bank_add','transfer']) - require(len(funding['audit_event_ids'])==2 and funding['root_balance_minor']==0, 'selftest_bank_fund_failed') - async def checked(name,args,key=None,subject=None,**kwargs): - result=await call(name,args,key,subject,**kwargs) - require(result.status=='ok','market_selftest_'+name.replace('.','_') + - (':' + result.error.code if result.error else '')) + + return await _confirmed_money( + app, + action, + operator='isolated-TestConsole', + amount=args.amount, + subject_id=getattr(args, 'subject_id', None), + emit=emit, + confirm=confirm, + unlock=unlock, + ) + + await command(['money', 'mint', '20'], ['mint']) + funding = await command(['money', 'bank', 'fund', '@bank-test', '20'], ['bank_add', 'transfer']) + require( + len(funding['audit_event_ids']) == 2 and funding['root_balance_minor'] == 0, + 'selftest_bank_fund_failed', + ) + + async def checked(name, args, key=None, subject=None, **kwargs): + result = await call(name, args, key, subject, **kwargs) + require( + result.status == 'ok', + 'market_selftest_' + + name.replace('.', '_') + + (':' + result.error.code if result.error else ''), + ) return result - bounty=await checked('bounty.create',{'name':'signature-reward', - 'terms':'Control of the current IdentityKey only, not human or Sybil proof.', - 'reward_minor':10_000_000,'budget_minor':10_000_000,'max_claims':1, - 'claim_limit_per_subject':1},bank_key,bank) - lid=bounty.data['bounty']['listing_id'] - challenge=await checked('bounty.challenge',{'listing_id':lid},buyer_key,buyer) - proof=challenge.data['challenge'] - claimed=await checked('bounty.claim',{'challenge_id':proof['challenge_id'], - 'proof':wire(buyer_key.sign(canonical(proof),purpose='bounty-pop-v1'))},buyer_key,buyer) - require(claimed.data['reward']['body']['amount_minor']==10_000_000 and - claimed.data['escrow_balance_minor']==0,'selftest_bounty_not_paid') - listing=await checked('store.listing_create',{'name':'test-delivery','item_kind':'bundle', - 'price_minor':5_000_000,'currency_id':'primary','quantity':1, - 'delivery_mode':'managed_instant','escrow_policy':'escrow-v1', - 'dispute_policy':'dispute-v1','terms':'Fixed text and file; immediate in-site delivery.'},bank_key,bank) + + bounty = await checked( + 'bounty.create', + { + 'name': 'signature-reward', + 'terms': 'Control of the current IdentityKey only, not human or Sybil proof.', + 'reward_minor': 10_000_000, + 'budget_minor': 10_000_000, + 'max_claims': 1, + 'claim_limit_per_subject': 1, + }, + bank_key, + bank, + ) + lid = bounty.data['bounty']['listing_id'] + challenge = await checked('bounty.challenge', {'listing_id': lid}, buyer_key, buyer) + proof = challenge.data['challenge'] + claimed = await checked( + 'bounty.claim', + { + 'challenge_id': proof['challenge_id'], + 'proof': wire(buyer_key.sign(canonical(proof), purpose='bounty-pop-v1')), + }, + buyer_key, + buyer, + ) + require( + claimed.data['reward']['body']['amount_minor'] == 10_000_000 + and claimed.data['escrow_balance_minor'] == 0, + 'selftest_bounty_not_paid', + ) + listing = await checked( + 'store.listing_create', + { + 'name': 'test-delivery', + 'item_kind': 'bundle', + 'price_minor': 5_000_000, + 'currency_id': 'primary', + 'quantity': 1, + 'delivery_mode': 'managed_instant', + 'escrow_policy': 'escrow-v1', + 'dispute_policy': 'dispute-v1', + 'terms': 'Fixed text and file; immediate in-site delivery.', + }, + bank_key, + bank, + ) async with app.metadata.transaction(write=False) as tx: - files=await tx.resolve((await tx.path(bank))+'/files') - payload=await checked('content.file_put',{'parent':files,'name':'hello.txt','data':b64(b'delivery-ok\n'), - 'media_type':'text/plain'},bank_key,bank) - listing_id=listing.resources[0].id - package=await checked('store.package_deposit',{'listing_id':listing_id, - 'listing_revision':listing.resources[0].revision,'manifest':{'text':'msg.lmm.best store selftest'}, - 'payload_refs':[wire(payload.resources[0])]},bank_key,bank) - active=await checked('store.listing_update',{'id':listing_id,'state':'active','package_ref':package.data['package']['id']}, - bank_key,bank,expected=((listing_id,listing.data['generation']),)) - quote=active.data['listing'] - args={'listing_id':listing_id,'listing_revision':quote['listing_revision'],'quantity':1, - 'currency_id':'primary','total_price_minor':5_000_000} + files = await tx.resolve((await tx.path(bank)) + '/files') + payload = await checked( + 'content.file_put', + { + 'parent': files, + 'name': 'hello.txt', + 'data': b64(b'delivery-ok\n'), + 'media_type': 'text/plain', + }, + bank_key, + bank, + ) + listing_id = listing.resources[0].id + package = await checked( + 'store.package_deposit', + { + 'listing_id': listing_id, + 'listing_revision': listing.resources[0].revision, + 'manifest': {'text': 'msg.lmm.best store selftest'}, + 'payload_refs': [wire(payload.resources[0])], + }, + bank_key, + bank, + ) + active = await checked( + 'store.listing_update', + {'id': listing_id, 'state': 'active', 'package_ref': package.data['package']['id']}, + bank_key, + bank, + expected=((listing_id, listing.data['generation']),), + ) + quote = active.data['listing'] + args = { + 'listing_id': listing_id, + 'listing_revision': quote['listing_revision'], + 'quantity': 1, + 'currency_id': 'primary', + 'total_price_minor': 5_000_000, + } # No seller request occurs after publishing. The signed buyer request alone # validates the immutable deposit and delivers; payment remains in escrow. - bought=await checked('orders.buy',args,buyer_key,buyer,request_id='market-e2e-buy', - contract_version=4) - require(bought.data['order']['state']=='delivered','selftest_order_not_delivered') - oid=bought.data['order']['id'] + bought = await checked( + 'orders.buy', args, buyer_key, buyer, request_id='market-e2e-buy', contract_version=4 + ) + require(bought.data['order']['state'] == 'delivered', 'selftest_order_not_delivered') + oid = bought.data['order']['id'] async with app.metadata.transaction(write=False) as tx: - escrow=tx.one('SELECT escrow_subject FROM store_orders WHERE id=?',(oid,))[0] - require((_balance(tx,buyer),_balance(tx,bank),_balance(tx,escrow)) == - (5_000_000,10_000_000,5_000_000),'selftest_awaiting_acceptance_balance_mismatch') - delivery=await checked('delivery.get',{'order_id':oid},buyer_key,buyer,contract_version=2) - content=delivery.data['delivery'] - require(content['manifest']=={'text':'msg.lmm.best store selftest'} and - content['payloads'][0]['data']==b64(b'delivery-ok\n') and - content['package_digest']==package.data['package']['digest'],'selftest_delivery_mismatch') - repeated=await checked('orders.buy',args,buyer_key,buyer,request_id='market-e2e-buy', - contract_version=4) - require(repeated.replayed and repeated.data==bought.data,'selftest_market_replay_failed') - require(content['state']=='prepared','selftest_false_buyer_ack') - accepted=await checked('delivery.accept',{'order_id':oid, - 'delivery_digest':content['delivery_digest']},buyer_key,buyer, - contract_version=2,request_id='market-e2e-accept') - require(accepted.data['state']=='settled','selftest_order_not_settled') + escrow = tx.one('SELECT escrow_subject FROM store_orders WHERE id=?', (oid,))[0] + require( + (_balance(tx, buyer), _balance(tx, bank), _balance(tx, escrow)) + == (5_000_000, 10_000_000, 5_000_000), + 'selftest_awaiting_acceptance_balance_mismatch', + ) + delivery = await checked( + 'delivery.get', {'order_id': oid}, buyer_key, buyer, contract_version=2 + ) + content = delivery.data['delivery'] + require( + content['manifest'] == {'text': 'msg.lmm.best store selftest'} + and content['payloads'][0]['data'] == b64(b'delivery-ok\n') + and content['package_digest'] == package.data['package']['digest'], + 'selftest_delivery_mismatch', + ) + repeated = await checked( + 'orders.buy', args, buyer_key, buyer, request_id='market-e2e-buy', contract_version=4 + ) + require(repeated.replayed and repeated.data == bought.data, 'selftest_market_replay_failed') + require(content['state'] == 'prepared', 'selftest_false_buyer_ack') + accepted = await checked( + 'delivery.accept', + {'order_id': oid, 'delivery_digest': content['delivery_digest']}, + buyer_key, + buyer, + contract_version=2, + request_id='market-e2e-accept', + ) + require(accepted.data['state'] == 'settled', 'selftest_order_not_settled') async with app.metadata.transaction(write=False) as tx: - require(_balance(tx,escrow)==0,'selftest_escrow_not_empty') - buyer_balance=await checked('money.balance',{},buyer_key,buyer) - bank_balance=await checked('money.balance',{},bank_key,bank) - require(buyer_balance.data['balance_minor']==5_000_000 and bank_balance.data['balance_minor']==15_000_000, - 'selftest_market_balance_mismatch') + require(_balance(tx, escrow) == 0, 'selftest_escrow_not_empty') + buyer_balance = await checked('money.balance', {}, buyer_key, buyer) + bank_balance = await checked('money.balance', {}, bank_key, bank) + require( + buyer_balance.data['balance_minor'] == 5_000_000 + and bank_balance.data['balance_minor'] == 15_000_000, + 'selftest_market_balance_mismatch', + ) async with app.metadata.transaction(write=False) as tx: - checked_state=inspect_clearing(app,tx) - require(checked_state['total_supply_minor']==20_000_000 and - checked_state['banks']==baseline['banks']+1 and - checked_state['enabled_offers']==baseline['enabled_offers'], - 'selftest_market_supply_mismatch') + checked_state = inspect_clearing(app, tx) + require( + checked_state['total_supply_minor'] == 20_000_000 + and checked_state['banks'] == baseline['banks'] + 1 + and checked_state['enabled_offers'] == baseline['enabled_offers'], + 'selftest_market_supply_mismatch', + ) # The 20 -> 10 -> 5 buyer checkpoint above remains the official PoP/store # acceptance flow. Separately exercise an explicit synchronous grant intent. from msg.admin.money import apply_offer from msg.client_market import entitlement_intent from msg.plugins.hosting_capacity import ENTITLEMENT_KIND, extra_capacity - await apply_offer(app, root, action='set', operator='isolated-TestConsole', - offer_id='selftest-byte-grant', fields={'resource_kind': 'website', 'unit': 'byte', - 'price_minor': 1, 'min_quantity': 1, 'max_quantity': 1, - 'entitlement_kind': ENTITLEMENT_KIND, 'duration_seconds': None}) + + await apply_offer( + app, + root, + action='set', + operator='isolated-TestConsole', + offer_id='selftest-byte-grant', + fields={ + 'resource_kind': 'website', + 'unit': 'byte', + 'price_minor': 1, + 'min_quantity': 1, + 'max_quantity': 1, + 'entitlement_kind': ENTITLEMENT_KIND, + 'duration_seconds': None, + }, + ) + class BuyerClient: async def call(self, name, params, **options): return await checked(name, params, buyer_key, buyer, **options) - intent = await entitlement_intent(BuyerClient(), {'offer_id': 'selftest-byte-grant', 'quantity': 1}) - grant = await checked('money.redeem', intent, buyer_key, buyer, - contract_version=3, request_id='market-e2e-entitlement') - repeated = await checked('money.redeem', intent, buyer_key, buyer, - contract_version=3, request_id='market-e2e-entitlement') + + intent = await entitlement_intent( + BuyerClient(), {'offer_id': 'selftest-byte-grant', 'quantity': 1} + ) + grant = await checked( + 'money.redeem', + intent, + buyer_key, + buyer, + contract_version=3, + request_id='market-e2e-entitlement', + ) + repeated = await checked( + 'money.redeem', + intent, + buyer_key, + buyer, + contract_version=3, + request_id='market-e2e-entitlement', + ) order = grant.data['order'] - require(order['state'] == 'settled' and order['contract_version'] == 5 and - order['delivered_at'] is None and repeated.replayed and repeated.data == grant.data, - 'selftest_entitlement_order_failed') + require( + order['state'] == 'settled' + and order['contract_version'] == 5 + and order['delivered_at'] is None + and repeated.replayed + and repeated.data == grant.data, + 'selftest_entitlement_order_failed', + ) async with app.metadata.transaction(write=False) as tx: locked = contract(tx, order['id']) - require(extra_capacity(tx, buyer, app.clock()) == 1 and - (_balance(tx, buyer), _balance(tx, bank), _balance(tx, ROOT_SUBJECT), - _balance(tx, locked['escrow_subject'])) == (4_999_999, 15_000_000, 1, 0) and - tx.one('SELECT COUNT(*) FROM store_deliveries WHERE order_id=?', (order['id'],))[0] == 0, - 'selftest_entitlement_grant_failed') - require(inspect_clearing(app, tx)['total_supply_minor'] == 20_000_000, - 'selftest_entitlement_supply_mismatch') + require( + extra_capacity(tx, buyer, app.clock()) == 1 + and ( + _balance(tx, buyer), + _balance(tx, bank), + _balance(tx, ROOT_SUBJECT), + _balance(tx, locked['escrow_subject']), + ) + == (4_999_999, 15_000_000, 1, 0) + and tx.one('SELECT COUNT(*) FROM store_deliveries WHERE order_id=?', (order['id'],))[0] + == 0, + 'selftest_entitlement_grant_failed', + ) + require( + inspect_clearing(app, tx)['total_supply_minor'] == 20_000_000, + 'selftest_entitlement_supply_mismatch', + ) return True diff --git a/src/msg/admin/money.py b/src/msg/admin/money.py index 9ea4556d..53e9b3f2 100644 --- a/src/msg/admin/money.py +++ b/src/msg/admin/money.py @@ -1,4 +1,5 @@ """Physical-console central bank operations. Never register these as network ops.""" + from __future__ import annotations import asyncio @@ -12,55 +13,75 @@ from msg.core.codec import canonical, digest, loads, wire from msg.core.errors import require from msg.core.models import AuditEvent, Event, ResourceRef -from msg.plugins.money import ( - CURRENCY_ID, - MAX_MINOR, - SCALE, - _balance, - _post_entry, - _supply, -) +from msg.market.offer_resources import OFFER_COLUMNS as _OFFER_COLUMNS +from msg.plugins.money import CURRENCY_ID, MAX_MINOR, SCALE, _balance, _post_entry, _supply from msg.plugins.offers import validate_local_offer from msg.security.crypto import Ed25519Signer, open_private_key def parse_amount(value: str) -> int: """Parse a displayed MSG amount without binary floats or silent rounding.""" - require(isinstance(value, str) and re.fullmatch(r'(?:0|[1-9][0-9]*)(?:\.[0-9]{1,6})?', value), - 'invalid_money_amount') - minor = int(Decimal(value) * (10 ** SCALE)) + require( + isinstance(value, str) and re.fullmatch(r'(?:0|[1-9][0-9]*)(?:\.[0-9]{1,6})?', value), + 'invalid_money_amount', + ) + minor = int(Decimal(value) * (10**SCALE)) require(0 < minor <= MAX_MINOR, 'invalid_money_amount') return minor def _root_signer(app, signer): - require(isinstance(signer, Ed25519Signer) and - signer.public_key == app.certificates.root_public_key, 'root_key_mismatch') + require( + isinstance(signer, Ed25519Signer) and signer.public_key == app.certificates.root_public_key, + 'root_key_mismatch', + ) -from msg.market.offer_resources import OFFER_COLUMNS as _OFFER_COLUMNS - _UNSET = object() def _offer_snapshot(tx, offer_id): - row = tx.one('''SELECT offer_id,resource_kind,unit,price_minor,min_quantity,max_quantity, + row = tx.one( + """SELECT offer_id,resource_kind,unit,price_minor,min_quantity,max_quantity, entitlement_kind,duration_seconds,enabled,price_revision - FROM server_offers WHERE offer_id=?''', (offer_id,)) - return dict(zip(_OFFER_COLUMNS, row)) if row else None + FROM server_offers WHERE offer_id=?""", + (offer_id,), + ) + return dict(zip(_OFFER_COLUMNS, row, strict=True)) if row else None -async def apply_offer(app, signer, *, action: str, operator: str, offer_id: str, - fields: dict | None = None, expected_offer=_UNSET): +async def apply_offer( + app, + signer, + *, + action: str, + operator: str, + offer_id: str, + fields: dict | None = None, + expected_offer=_UNSET, +): """Root-signed local offer mutation; set remains Registry/fulfiller gated.""" _root_signer(app, signer) require(action in {'set', 'disable'}, 'invalid_offer_action') - require(isinstance(offer_id, str) and - re.fullmatch(r'[a-zA-Z][a-zA-Z0-9_-]{0,79}', offer_id), 'invalid_offer_id') + require( + isinstance(offer_id, str) and re.fullmatch(r'[a-zA-Z][a-zA-Z0-9_-]{0,79}', offer_id), + 'invalid_offer_id', + ) if action == 'set': - require(isinstance(fields, dict) and set(fields) == { - 'resource_kind', 'unit', 'price_minor', 'min_quantity', - 'max_quantity', 'entitlement_kind', 'duration_seconds'}, 'invalid_offer_fields') + require( + isinstance(fields, dict) + and set(fields) + == { + 'resource_kind', + 'unit', + 'price_minor', + 'min_quantity', + 'max_quantity', + 'entitlement_kind', + 'duration_seconds', + }, + 'invalid_offer_fields', + ) validate_local_offer(app, **fields) else: require(fields is None, 'invalid_offer_fields') @@ -69,6 +90,7 @@ async def apply_offer(app, signer, *, action: str, operator: str, offer_id: str, async with app.metadata.transaction(write=True) as tx: previous = _offer_snapshot(tx, offer_id) from msg.market.offer_resources import authoritative_offer, write_offer_revision + source = await authoritative_offer(app, tx, offer_id) if source is not None: require(source == previous, 'offer_projection_mismatch') @@ -77,7 +99,8 @@ async def apply_offer(app, signer, *, action: str, operator: str, offer_id: str, if action == 'set': validate_local_offer(app, **fields) revision = 'price_' + uuid4().hex - tx.execute('''INSERT INTO server_offers + tx.execute( + """INSERT INTO server_offers (offer_id,resource_kind,unit,price_minor,min_quantity,max_quantity, entitlement_kind,duration_seconds,enabled,price_revision) VALUES (?,?,?,?,?,?,?,?,TRUE,?) @@ -86,51 +109,95 @@ async def apply_offer(app, signer, *, action: str, operator: str, offer_id: str, price_minor=excluded.price_minor,min_quantity=excluded.min_quantity, max_quantity=excluded.max_quantity,entitlement_kind=excluded.entitlement_kind, duration_seconds=excluded.duration_seconds,enabled=TRUE, - price_revision=excluded.price_revision''', - (offer_id, fields['resource_kind'], fields['unit'], fields['price_minor'], - fields['min_quantity'], fields['max_quantity'], fields['entitlement_kind'], - fields['duration_seconds'], revision), write=True) + price_revision=excluded.price_revision""", + ( + offer_id, + fields['resource_kind'], + fields['unit'], + fields['price_minor'], + fields['min_quantity'], + fields['max_quantity'], + fields['entitlement_kind'], + fields['duration_seconds'], + revision, + ), + write=True, + ) else: require(previous is not None, 'offer_not_found') require(previous['enabled'], 'offer_already_disabled') - tx.execute('UPDATE server_offers SET enabled=FALSE WHERE offer_id=?', - (offer_id,), write=True) + tx.execute( + 'UPDATE server_offers SET enabled=FALSE WHERE offer_id=?', (offer_id,), write=True + ) current = _offer_snapshot(tx, offer_id) if previous is None or source is not None: - await write_offer_revision(app, tx, signer, current, now=now, - request_id=request_id, new=previous is None) - statement = {'action': 'offer_' + action, 'offer_id': offer_id, - 'before': previous, 'after': current, 'operator': operator, - 'request_id': request_id, 'time': wire(now)} + await write_offer_revision( + app, tx, signer, current, now=now, request_id=request_id, new=previous is None + ) + statement = { + 'action': 'offer_' + action, + 'offer_id': offer_id, + 'before': previous, + 'after': current, + 'operator': operator, + 'request_id': request_id, + 'time': wire(now), + } signature = wire(signer.sign(canonical(statement), purpose='money-admin')) - event = Event(id='audit_' + uuid4().hex, type='root.money.offer.' + action, - time=now, request_id=request_id, actor=ROOT_SUBJECT, - subject=ROOT_SUBJECT, resources=(ResourceRef(id=ROOT_SUBJECT),), - data={**statement, 'root_signature': signature}) + event = Event( + id='audit_' + uuid4().hex, + type='root.money.offer.' + action, + time=now, + request_id=request_id, + actor=ROOT_SUBJECT, + subject=ROOT_SUBJECT, + resources=(ResourceRef(id=ROOT_SUBJECT),), + data={**statement, 'root_signature': signature}, + ) await tx.append_event(event) - await tx.append_audit(AuditEvent(event=event, - authority=(ResourceRef(id=app.certificates.root_certificate.resource_id),), - before_digest=digest(previous), after_digest=digest(current), - previous_digest=None, entry_digest='', result='offer_' + action)) + await tx.append_audit( + AuditEvent( + event=event, + authority=(ResourceRef(id=app.certificates.root_certificate.resource_id),), + before_digest=digest(previous), + after_digest=digest(current), + previous_digest=None, + entry_digest='', + result='offer_' + action, + ) + ) return {'action': action, 'offer': current, 'audit_event_id': event.id} -async def apply_money(app, signer, *, action: str, operator: str, amount_minor: int | None = None, - subject_id: str | None = None, expected_state: dict | None = None): +async def apply_money( + app, + signer, + *, + action: str, + operator: str, + amount_minor: int | None = None, + subject_id: str | None = None, + expected_state: dict | None = None, +): """Internal local use case; callers must establish the OS-console boundary. Each invocation creates a fresh local request id. PG serializes all writers, so balance and supply checks are re-evaluated within the committing transaction. """ _root_signer(app, signer) - require(action in {'mint', 'burn', 'transfer', 'bank_add', 'bank_remove', 'bank_fund'}, 'invalid_money_action') + require( + action in {'mint', 'burn', 'transfer', 'bank_add', 'bank_remove', 'bank_fund'}, + 'invalid_money_action', + ) if action in {'mint', 'burn', 'transfer', 'bank_fund'}: require(type(amount_minor) is int and 0 < amount_minor <= MAX_MINOR, 'invalid_money_amount') else: require(amount_minor is None, 'invalid_money_amount') if action in {'transfer', 'bank_add', 'bank_remove', 'bank_fund'}: - require(isinstance(subject_id, str) and subject_id not in {ROOT_SUBJECT, '@root'}, - 'invalid_money_recipient') + require( + isinstance(subject_id, str) and subject_id not in {ROOT_SUBJECT, '@root'}, + 'invalid_money_recipient', + ) else: require(subject_id is None, 'invalid_money_recipient') @@ -139,35 +206,56 @@ async def apply_money(app, signer, *, action: str, operator: str, amount_minor: async with app.metadata.transaction(write=True) as tx: if subject_id is not None: subject = await tx.subject(subject_id) - require(subject.kind in {'registered', 'custodial'} and not subject.local_only, - 'invalid_money_recipient') + require( + subject.kind in {'registered', 'custodial'} and not subject.local_only, + 'invalid_money_recipient', + ) before_supply = _supply(tx) before_root = _balance(tx, ROOT_SUBJECT) - row = (tx.one('SELECT status FROM money_bank_roles WHERE subject_id=?', (subject_id,)) - if action.startswith('bank_') else None) + row = ( + tx.one('SELECT status FROM money_bank_roles WHERE subject_id=?', (subject_id,)) + if action.startswith('bank_') + else None + ) before_role = row[0] if row else None if expected_state is not None: - current = {'supply': before_supply, 'root': before_root, - 'subject': _balance(tx, subject_id) if subject_id else None, - 'role': before_role} + current = { + 'supply': before_supply, + 'root': before_root, + 'subject': _balance(tx, subject_id) if subject_id else None, + 'role': before_role, + } require(current == expected_state, 'money_preview_stale') if action.startswith('bank_'): - require(action == 'bank_fund' or - (action == 'bank_add' and before_role != 'active') or - (action == 'bank_remove' and before_role == 'active'), 'bank_role_unchanged') + require( + action == 'bank_fund' + or (action == 'bank_add' and before_role != 'active') + or (action == 'bank_remove' and before_role == 'active'), + 'bank_role_unchanged', + ) if action in {'bank_add', 'bank_fund'}: - tx.execute('''INSERT INTO money_bank_roles(subject_id,status,granted_at,granted_by,revoked_at) + tx.execute( + """INSERT INTO money_bank_roles(subject_id,status,granted_at,granted_by,revoked_at) VALUES (?,?,?,?,NULL) ON CONFLICT(subject_id) DO UPDATE SET status='active',granted_at=excluded.granted_at,granted_by=excluded.granted_by, - revoked_at=NULL''', (subject_id, 'active', wire(now), ROOT_SUBJECT), write=True) + revoked_at=NULL""", + (subject_id, 'active', wire(now), ROOT_SUBJECT), + write=True, + ) else: - tx.execute("UPDATE money_bank_roles SET status='revoked',revoked_at=? WHERE subject_id=?", - (wire(now), subject_id), write=True) + tx.execute( + "UPDATE money_bank_roles SET status='revoked',revoked_at=? WHERE subject_id=?", + (wire(now), subject_id), + write=True, + ) receipt = None if action not in {'bank_add', 'bank_remove'}: if action == 'mint': - require(before_supply + amount_minor <= MAX_MINOR and - before_root + amount_minor <= MAX_MINOR, 'money_overflow') + require( + before_supply + amount_minor <= MAX_MINOR + and before_root + amount_minor <= MAX_MINOR, + 'money_overflow', + ) debit, credit, kind = None, ROOT_SUBJECT, 'mint' elif action == 'burn': require(before_root >= amount_minor, 'insufficient_funds') @@ -176,49 +264,87 @@ async def apply_money(app, signer, *, action: str, operator: str, amount_minor: require(before_root >= amount_minor, 'insufficient_funds') require(_balance(tx, subject_id) + amount_minor <= MAX_MINOR, 'money_overflow') debit, credit, kind = ROOT_SUBJECT, subject_id, 'transfer' - receipt = _post_entry(tx, sender=debit, recipient=credit, - amount=amount_minor, actor=ROOT_SUBJECT, request_id=request_id, - now=now, receipt_signer=app.receipt_signer, kind=kind, _local_issuer=True) - statement = {'action': action, 'subject_id': subject_id, 'amount_minor': amount_minor, - 'currency_id': CURRENCY_ID, 'operator': operator, 'request_id': request_id, - 'time': wire(now), 'receipt_digest': digest(receipt) if receipt else None, - 'before_supply_minor': before_supply, 'after_supply_minor': _supply(tx), - 'before_root_minor': before_root, 'after_root_minor': _balance(tx, ROOT_SUBJECT), - 'before_role': before_role} + receipt = _post_entry( + tx, + sender=debit, + recipient=credit, + amount=amount_minor, + actor=ROOT_SUBJECT, + request_id=request_id, + now=now, + receipt_signer=app.receipt_signer, + kind=kind, + _local_issuer=True, + ) + statement = { + 'action': action, + 'subject_id': subject_id, + 'amount_minor': amount_minor, + 'currency_id': CURRENCY_ID, + 'operator': operator, + 'request_id': request_id, + 'time': wire(now), + 'receipt_digest': digest(receipt) if receipt else None, + 'before_supply_minor': before_supply, + 'after_supply_minor': _supply(tx), + 'before_root_minor': before_root, + 'after_root_minor': _balance(tx, ROOT_SUBJECT), + 'before_role': before_role, + } audit_ids = [] actions = ('bank_add', 'transfer') if action == 'bank_fund' else (action,) for audit_action in actions: fact = {**statement, 'action': audit_action, 'combined_action': action} proof = wire(signer.sign(canonical(fact), purpose='money-admin')) - event = Event(id='audit_' + uuid4().hex, type='root.money.' + audit_action, - time=now, request_id=request_id, actor=ROOT_SUBJECT, - subject=ROOT_SUBJECT, - resources=(ResourceRef(id=subject_id or ROOT_SUBJECT),), - data={**fact, 'root_signature': proof}) + event = Event( + id='audit_' + uuid4().hex, + type='root.money.' + audit_action, + time=now, + request_id=request_id, + actor=ROOT_SUBJECT, + subject=ROOT_SUBJECT, + resources=(ResourceRef(id=subject_id or ROOT_SUBJECT),), + data={**fact, 'root_signature': proof}, + ) await tx.append_event(event) - await tx.append_audit(AuditEvent(event=event, - authority=(ResourceRef(id=app.certificates.root_certificate.resource_id),), - before_digest=digest({'supply': before_supply, 'root': before_root, 'role': before_role}), - after_digest=digest(fact), previous_digest=None, entry_digest='', result=audit_action)) + await tx.append_audit( + AuditEvent( + event=event, + authority=(ResourceRef(id=app.certificates.root_certificate.resource_id),), + before_digest=digest({ + 'supply': before_supply, + 'root': before_root, + 'role': before_role, + }), + after_digest=digest(fact), + previous_digest=None, + entry_digest='', + result=audit_action, + ) + ) audit_ids.append(event.id) - return {'action': action, 'receipt': receipt, 'audit_event_id': audit_ids[-1], - 'audit_event_ids': audit_ids, - 'root_balance_minor': statement['after_root_minor'], - 'total_supply_minor': statement['after_supply_minor'], - 'subject_id': subject_id} + return { + 'action': action, + 'receipt': receipt, + 'audit_event_id': audit_ids[-1], + 'audit_event_ids': audit_ids, + 'root_balance_minor': statement['after_root_minor'], + 'total_supply_minor': statement['after_supply_minor'], + 'subject_id': subject_id, + } - -async def _confirmed_money(app, action, *, operator, amount, subject_id, - emit, confirm, unlock): +async def _confirmed_money(app, action, *, operator, amount, subject_id, emit, confirm, unlock): """The command's approval pipeline, shared with an isolated TestConsole. The production caller must establish the physical-console boundary first. There is no network operation, config option or command flag selecting IO adapters. The diagnostic replaces only IO/unlock on its fresh Test Root. """ - require(action in {'mint','burn','transfer','bank_add','bank_remove','bank_fund'}, - 'invalid_money_action') + require( + action in {'mint', 'burn', 'transfer', 'bank_add', 'bank_remove', 'bank_fund'}, + 'invalid_money_action', + ) amount_minor = parse_amount(amount) if amount is not None else None resolved_subject = subject_id async with app.metadata.transaction(write=False) as tx: @@ -227,41 +353,71 @@ async def _confirmed_money(app, action, *, operator, amount, subject_id, resolved_subject = await tx.resolve(namespace + '/' + subject_id) if resolved_subject is not None: subject = await tx.subject(resolved_subject) - require(subject.kind in {'registered','custodial'} and not subject.local_only, - 'invalid_money_recipient') + require( + subject.kind in {'registered', 'custodial'} and not subject.local_only, + 'invalid_money_recipient', + ) root_before = _balance(tx, ROOT_SUBJECT) supply_before = _supply(tx) subject_before = _balance(tx, resolved_subject) if resolved_subject else None - role_row = (tx.one('SELECT status FROM money_bank_roles WHERE subject_id=?', - (resolved_subject,)) if action.startswith('bank_') else None) + role_row = ( + tx.one('SELECT status FROM money_bank_roles WHERE subject_id=?', (resolved_subject,)) + if action.startswith('bank_') + else None + ) role_before = role_row[0] if role_row else None - root_delta = (amount_minor if action == 'mint' else - -amount_minor if action in {'burn','transfer','bank_fund'} else 0) - supply_delta = (amount_minor if action == 'mint' else - -amount_minor if action == 'burn' else 0) - preview = {'action':action, 'amount_minor':amount_minor, 'currency_id':CURRENCY_ID, - 'subject_id':resolved_subject, - 'subject_handle':subject_id if subject_id != resolved_subject else None, - 'root_balance_before_minor':root_before, 'root_balance_after_minor':root_before+root_delta, - 'total_supply_before_minor':supply_before, 'total_supply_after_minor':supply_before+supply_delta, - 'recipient_balance_before_minor':subject_before, - 'recipient_balance_after_minor':subject_before+amount_minor - if action in {'transfer','bank_fund'} else subject_before, - 'bank_role_before':role_before, 'bank_role_after':'active' - if action in {'bank_add','bank_fund'} else 'revoked' if action=='bank_remove' else role_before} + root_delta = ( + amount_minor + if action == 'mint' + else -amount_minor + if action in {'burn', 'transfer', 'bank_fund'} + else 0 + ) + supply_delta = amount_minor if action == 'mint' else -amount_minor if action == 'burn' else 0 + preview = { + 'action': action, + 'amount_minor': amount_minor, + 'currency_id': CURRENCY_ID, + 'subject_id': resolved_subject, + 'subject_handle': subject_id if subject_id != resolved_subject else None, + 'root_balance_before_minor': root_before, + 'root_balance_after_minor': root_before + root_delta, + 'total_supply_before_minor': supply_before, + 'total_supply_after_minor': supply_before + supply_delta, + 'recipient_balance_before_minor': subject_before, + 'recipient_balance_after_minor': subject_before + amount_minor + if action in {'transfer', 'bank_fund'} + else subject_before, + 'bank_role_before': role_before, + 'bank_role_after': 'active' + if action in {'bank_add', 'bank_fund'} + else 'revoked' + if action == 'bank_remove' + else role_before, + } # Distinct confirmations, bound to one before/after snapshot. Unlock occurs # only after BOTH role and funding approvals, before the atomic use case. - approvals = ('bank_add','transfer') if action == 'bank_fund' else (action,) + approvals = ('bank_add', 'transfer') if action == 'bank_fund' else (action,) for approved_action in approvals: - part = {**preview, 'approval_for':approved_action} + part = {**preview, 'approval_for': approved_action} emit(canonical(part).decode()) # Escaped JSON, never terminal control text. approval = 'CONFIRM MONEY ' + digest(part) require(confirm('Type ' + approval + ' to continue: ') == approval, 'approval_cancelled') signer = unlock() - return await apply_money(app, signer, action=action, operator=operator, - amount_minor=amount_minor, subject_id=resolved_subject, - expected_state={'supply':supply_before, 'root':root_before, - 'subject':subject_before, 'role':role_before}) + return await apply_money( + app, + signer, + action=action, + operator=operator, + amount_minor=amount_minor, + subject_id=resolved_subject, + expected_state={ + 'supply': supply_before, + 'root': root_before, + 'subject': subject_before, + 'role': role_before, + }, + ) class MoneyAdmin: @@ -271,6 +427,7 @@ def __init__(self, config_dir): def execute(self, action, *, amount=None, subject_id=None): operator = require_local_console(self.config_dir) from msg.admin.root import RootAdmin + app = RootAdmin(self.config_dir)._app() def unlock(): @@ -281,43 +438,71 @@ def unlock(): async def run(): await app.load() try: - return await _confirmed_money(app, action, operator=operator, amount=amount, - subject_id=subject_id, emit=print, confirm=input, unlock=unlock) + return await _confirmed_money( + app, + action, + operator=operator, + amount=amount, + subject_id=subject_id, + emit=print, + confirm=input, + unlock=unlock, + ) finally: await app.close() + return asyncio.run(run()) def execute_offer(self, action, *, offer_id, fields=None, dry_run=False): operator = require_local_console(self.config_dir) require(action in {'set', 'disable', 'import'}, 'invalid_offer_action') require(not dry_run or action == 'import', 'invalid_offer_action') - require(isinstance(offer_id, str) and - re.fullmatch(r'[a-zA-Z][a-zA-Z0-9_-]{0,79}', offer_id), 'invalid_offer_id') + require( + isinstance(offer_id, str) and re.fullmatch(r'[a-zA-Z][a-zA-Z0-9_-]{0,79}', offer_id), + 'invalid_offer_id', + ) from msg.admin.root import RootAdmin + app = RootAdmin(self.config_dir)._app() async def run(): await app.load() try: if action == 'import': - from msg.admin.offer_import import preview_import, apply_import + from msg.admin.offer_import import apply_import, preview_import + async with app.metadata.transaction(write=False) as tx: plan = await preview_import(app, tx, offer_id) print(canonical(plan).decode()) if dry_run: return {'plan': plan, 'approval_digest': digest(plan)} approval = 'CONFIRM MONEY ' + digest(plan) - require(input('Type ' + approval + ' to continue: ') == approval, - 'approval_cancelled') + require( + input('Type ' + approval + ' to continue: ') == approval, + 'approval_cancelled', + ) pin = getpass.getpass('Root PIN/passphrase: ') - private = open_private_key(loads(root_envelope(self.config_dir).read_bytes()), pin) - return await apply_import(app, Ed25519Signer.from_bytes(private), plan, - operator=operator) + private = open_private_key( + loads(root_envelope(self.config_dir).read_bytes()), pin + ) + return await apply_import( + app, Ed25519Signer.from_bytes(private), plan, operator=operator + ) if action == 'set': - require(isinstance(fields, dict) and set(fields) == { - 'resource_kind', 'unit', 'price', 'min_quantity', - 'max_quantity', 'entitlement_kind', 'duration_seconds'}, - 'invalid_offer_fields') + require( + isinstance(fields, dict) + and set(fields) + == { + 'resource_kind', + 'unit', + 'price', + 'min_quantity', + 'max_quantity', + 'entitlement_kind', + 'duration_seconds', + }, + 'invalid_offer_fields', + ) fields['price_minor'] = parse_amount(fields.pop('price')) validate_local_offer(app, **fields) async with app.metadata.transaction(write=False) as tx: @@ -325,19 +510,30 @@ async def run(): if action == 'disable': require(previous is not None, 'offer_not_found') require(previous['enabled'], 'offer_already_disabled') - preview = {'action': 'offer_' + action, 'offer_id': offer_id, - 'before': previous, 'proposed': fields if action == 'set' else - {**previous, 'enabled': False}} + preview = { + 'action': 'offer_' + action, + 'offer_id': offer_id, + 'before': previous, + 'proposed': fields if action == 'set' else {**previous, 'enabled': False}, + } print(canonical(preview).decode()) approval = 'CONFIRM MONEY ' + digest(preview) - require(input('Type ' + approval + ' to continue: ') == approval, - 'approval_cancelled') + require( + input('Type ' + approval + ' to continue: ') == approval, 'approval_cancelled' + ) pin = getpass.getpass('Root PIN/passphrase: ') private = open_private_key(loads(root_envelope(self.config_dir).read_bytes()), pin) signer = Ed25519Signer.from_bytes(private) - return await apply_offer(app, signer, action=action, operator=operator, - offer_id=offer_id, fields=fields, - expected_offer=previous) + return await apply_offer( + app, + signer, + action=action, + operator=operator, + offer_id=offer_id, + fields=fields, + expected_offer=previous, + ) finally: await app.close() + return asyncio.run(run()) diff --git a/src/msg/admin/offer_import.py b/src/msg/admin/offer_import.py index 63d2f262..2377241b 100644 --- a/src/msg/admin/offer_import.py +++ b/src/msg/admin/offer_import.py @@ -1,4 +1,5 @@ """Explicit local adoption of a legacy offer; no financial rows are rewritten.""" + import re from uuid import uuid4 @@ -12,24 +13,53 @@ async def preview_import(app, tx, offer_id, *, revision_id=None): - require(isinstance(offer_id, str) and re.fullmatch(r'[a-zA-Z][a-zA-Z0-9_-]{0,79}', offer_id), - 'invalid_offer_id') + require( + isinstance(offer_id, str) and re.fullmatch(r'[a-zA-Z][a-zA-Z0-9_-]{0,79}', offer_id), + 'invalid_offer_id', + ) require(mapped_listing(tx, offer_id) is None, 'offer_already_resource') - row = tx.one('SELECT ' + ','.join(OFFER_COLUMNS) + ' FROM server_offers WHERE offer_id=?', (offer_id,)) + row = tx.one( + 'SELECT ' + ','.join(OFFER_COLUMNS) + ' FROM server_offers WHERE offer_id=?', (offer_id,) + ) require(row is not None, 'offer_not_found') - offer = dict(zip(OFFER_COLUMNS, row)) - validate_local_offer(app, **{k: offer[k] for k in ('resource_kind', 'unit', 'price_minor', - 'min_quantity', 'max_quantity', 'entitlement_kind', 'duration_seconds')}) - require(tx.one('SELECT 1 FROM resources WHERE id=? OR (parent=? AND name=?)', - (offer_id, 't_store', offer_id)) is None, 'offer_resource_conflict') + offer = dict(zip(OFFER_COLUMNS, row, strict=True)) + validate_local_offer( + app, + **{ + k: offer[k] + for k in ( + 'resource_kind', + 'unit', + 'price_minor', + 'min_quantity', + 'max_quantity', + 'entitlement_kind', + 'duration_seconds', + ) + }, + ) + require( + tx.one( + 'SELECT 1 FROM resources WHERE id=? OR (parent=? AND name=?)', + (offer_id, 't_store', offer_id), + ) + is None, + 'offer_resource_conflict', + ) revision_id = revision_id or 'v_' + uuid4().hex - require(isinstance(revision_id, str) and re.fullmatch(r'v_[a-f0-9]{32}', revision_id), - 'invalid_revision_id') - require(tx.one('SELECT 1 FROM revisions WHERE id=?', (revision_id,)) is None, - 'offer_revision_conflict') + require( + isinstance(revision_id, str) and re.fullmatch(r'v_[a-f0-9]{32}', revision_id), + 'invalid_revision_id', + ) + require( + tx.one('SELECT 1 FROM revisions WHERE id=?', (revision_id,)) is None, + 'offer_revision_conflict', + ) # The original serialized snapshots and receipt bytes are hashed unchanged. purchases, transactions, grants = [], {}, set() - for pid, owner, encoded in tx.rows('SELECT id,subject_id,offer_snapshot FROM money_purchases ORDER BY id'): + for pid, owner, encoded in tx.rows( + 'SELECT id,subject_id,offer_snapshot FROM money_purchases ORDER BY id' + ): snapshot = loads(encoded) require(isinstance(snapshot, dict), 'legacy_purchase_invalid') if snapshot.get('offer_id') != offer_id: @@ -39,24 +69,54 @@ async def preview_import(app, tx, offer_id, *, revision_id=None): purchases.append(tx.one('SELECT * FROM money_purchases WHERE id=?', (pid,))) for index, tid in enumerate(order['receipt_refs']): kind, reference = tx.one('SELECT kind,reference FROM money_ledger WHERE id=?', (tid,)) - expected_kind = 'transfer' if index == 0 else 'redeem' if purchase['state'] == 'settled' else 'refund' - expected_reference = ('purchase_fund:' if index == 0 else 'purchase_settle:' - if purchase['state'] == 'settled' else 'purchase_refund:') + pid - require((kind, reference) == (expected_kind, expected_reference), 'offer_import_inconsistent_ledger') + expected_kind = ( + 'transfer' + if index == 0 + else 'redeem' + if purchase['state'] == 'settled' + else 'refund' + ) + expected_reference = ( + 'purchase_fund:' + if index == 0 + else 'purchase_settle:' + if purchase['state'] == 'settled' + else 'purchase_refund:' + ) + pid + require( + (kind, reference) == (expected_kind, expected_reference), + 'offer_import_inconsistent_ledger', + ) transactions[tid] = tx.one('SELECT * FROM money_ledger WHERE id=?', (tid,)) if purchase['entitlement_id'] is not None: - grant = tx.one('SELECT offer_id,quantity,entitlement_kind FROM resource_entitlements WHERE id=?', - (purchase['entitlement_id'],)) - require(grant == (offer_id, purchase['quantity'], snapshot['entitlement_kind']), - 'offer_import_inconsistent_entitlements') + grant = tx.one( + 'SELECT offer_id,quantity,entitlement_kind FROM resource_entitlements WHERE id=?', + (purchase['entitlement_id'],), + ) + require( + grant == (offer_id, purchase['quantity'], snapshot['entitlement_kind']), + 'offer_import_inconsistent_entitlements', + ) grants.add(purchase['entitlement_id']) - entitlements = tx.rows('SELECT * FROM resource_entitlements WHERE offer_id=? ORDER BY id', (offer_id,)) + entitlements = tx.rows( + 'SELECT * FROM resource_entitlements WHERE offer_id=? ORDER BY id', (offer_id,) + ) require({row[0] for row in entitlements} == grants, 'offer_import_inconsistent_entitlements') - return {'action': 'offer_import', 'version': 1, 'offer_id': offer_id, - 'before': offer, 'listing_id': offer_id, 'revision_id': revision_id, - 'purchase_count': len(purchases), 'entitlement_count': len(entitlements), - 'dependencies_digest': digest({'purchases': purchases, - 'transactions': transactions, 'entitlements': entitlements})} + return { + 'action': 'offer_import', + 'version': 1, + 'offer_id': offer_id, + 'before': offer, + 'listing_id': offer_id, + 'revision_id': revision_id, + 'purchase_count': len(purchases), + 'entitlement_count': len(entitlements), + 'dependencies_digest': digest({ + 'purchases': purchases, + 'transactions': transactions, + 'entitlements': entitlements, + }), + } async def apply_import(app, signer, plan, *, operator): @@ -66,21 +126,53 @@ async def apply_import(app, signer, plan, *, operator): current = await preview_import(app, tx, plan['offer_id'], revision_id=plan['revision_id']) require(current == plan, 'offer_import_preview_stale') approved_digest = digest(plan) - await write_offer_revision(app, tx, signer, current['before'], now=now, - request_id=request_id, new=True, revision_id=plan['revision_id'], - source_digest=approved_digest) - statement = {'action': 'offer_import', 'plan': current, 'approved_digest': approved_digest, - 'operator': operator, 'request_id': request_id, 'time': wire(now)} + await write_offer_revision( + app, + tx, + signer, + current['before'], + now=now, + request_id=request_id, + new=True, + revision_id=plan['revision_id'], + source_digest=approved_digest, + ) + statement = { + 'action': 'offer_import', + 'plan': current, + 'approved_digest': approved_digest, + 'operator': operator, + 'request_id': request_id, + 'time': wire(now), + } signature = wire(signer.sign(canonical(statement), purpose='money-admin')) - event = Event(id='audit_' + uuid4().hex, type='root.money.offer.import', time=now, - request_id=request_id, actor=ROOT_SUBJECT, subject=ROOT_SUBJECT, + event = Event( + id='audit_' + uuid4().hex, + type='root.money.offer.import', + time=now, + request_id=request_id, + actor=ROOT_SUBJECT, + subject=ROOT_SUBJECT, resources=(ResourceRef(id=plan['listing_id'], revision=plan['revision_id']),), - data={**statement, 'root_signature': signature}) + data={**statement, 'root_signature': signature}, + ) await tx.append_event(event) - await tx.append_audit(AuditEvent(event=event, - authority=(ResourceRef(id=app.certificates.root_certificate.resource_id),), - before_digest=digest(plan['before']), after_digest=digest(plan['before']), - previous_digest=None, entry_digest='', result='offer_import')) - return {'action': 'offer_import', 'offer_id': plan['offer_id'], 'listing_id': plan['listing_id'], - 'revision_id': plan['revision_id'], 'approved_digest': approved_digest, - 'audit_event_id': event.id} + await tx.append_audit( + AuditEvent( + event=event, + authority=(ResourceRef(id=app.certificates.root_certificate.resource_id),), + before_digest=digest(plan['before']), + after_digest=digest(plan['before']), + previous_digest=None, + entry_digest='', + result='offer_import', + ) + ) + return { + 'action': 'offer_import', + 'offer_id': plan['offer_id'], + 'listing_id': plan['listing_id'], + 'revision_id': plan['revision_id'], + 'approved_digest': approved_digest, + 'audit_event_id': event.id, + } diff --git a/src/msg/admin/preflight.py b/src/msg/admin/preflight.py index eae60c37..db2335b3 100644 --- a/src/msg/admin/preflight.py +++ b/src/msg/admin/preflight.py @@ -4,6 +4,7 @@ no DDL, migration, filesystem repair, Root unlock, effect emission or sequence allocation. It never supplies missing operational evidence with fixture results. """ + from __future__ import annotations import argparse @@ -23,21 +24,43 @@ ('ingress_log_chain', '#64', 'CDN/proxy/APM/nginx/service redaction and listener evidence'), ('protected_snapshot', '#65/#84', 'independent provenance and digest of the field backup'), ('restore_conservation', '#65/#84', 'isolated old/new ledger and signed-history comparison'), - ('current_checkpoint', '#69', 'independently current signed revocation head outside the rollback set'), + ( + 'current_checkpoint', + '#69', + 'independently current signed revocation head outside the rollback set', + ), ('physical_console', '#70', 'real authorized local VT/serial console and rejection evidence'), ('ca_transition', '#70', 'finite old CA inventory and individually approved reissue/revoke'), - ('bank_fund_boundary', '#70/B', 'joint role versus injection gate evidence; no actual funding here'), - ('capacity_and_failure_drill', '#84', 'measured capacity and isolated install/crash/rollback drill'), + ( + 'bank_fund_boundary', + '#70/B', + 'joint role versus injection gate evidence; no actual funding here', + ), + ( + 'capacity_and_failure_drill', + '#84', + 'measured capacity and isolated install/crash/rollback drill', + ), ) def inspect_database(connection): - require(connection.execute('SHOW transaction_read_only').fetchone() == ('on',), - 'preflight_read_only_required') - tables = {row[0] for row in connection.execute( - 'SELECT tablename FROM pg_tables WHERE schemaname=current_schema()')} - result = {'read_only': True, 'state': 'empty' if not tables else 'unclassified', - 'table_count': len(tables), 'server_version': connection.info.server_version} + require( + connection.execute('SHOW transaction_read_only').fetchone() == ('on',), + 'preflight_read_only_required', + ) + tables = { + row[0] + for row in connection.execute( + 'SELECT tablename FROM pg_tables WHERE schemaname=current_schema()' + ) + } + result = { + 'read_only': True, + 'state': 'empty' if not tables else 'unclassified', + 'table_count': len(tables), + 'server_version': connection.info.server_version, + } if not tables: return result required = {'money_ledger', 'money_accounts', 'store_orders', 'bounty_listings', 'identities'} @@ -47,11 +70,19 @@ def inspect_database(connection): _validate_identity_references, _validate_ledger, ) + escrow = _escrow_sources(connection) - legacy = sum(connection.execute('SELECT 1 FROM identities WHERE id=%s', (key,)).fetchone() - is not None for key in escrow) - ledger = {'escrow_accounts': len(escrow), 'legacy_escrow_identities': legacy, - 'checks': 'blocked', 'migration_performed': False} + legacy = sum( + connection.execute('SELECT 1 FROM identities WHERE id=%s', (key,)).fetchone() + is not None + for key in escrow + ) + ledger = { + 'escrow_accounts': len(escrow), + 'legacy_escrow_identities': legacy, + 'checks': 'blocked', + 'migration_performed': False, + } result['state'] = 'typed_ledger' if 'ledger_accounts' in tables else 'legacy_ledger' try: _validate_identity_references(connection, escrow) @@ -77,54 +108,76 @@ def inspect_database(connection): ledger.update(rows=count, history_digest='sha256:' + hasher.hexdigest()) result['ledger'] = ledger if 'settings' in tables: - gate = connection.execute("SELECT value FROM settings WHERE key='recovery_quarantine'").fetchone() + gate = connection.execute( + "SELECT value FROM settings WHERE key='recovery_quarantine'" + ).fetchone() result['quarantine_present'] = gate is not None - row = connection.execute("SELECT value FROM settings WHERE key='active_root_certificate'").fetchone() + row = connection.execute( + "SELECT value FROM settings WHERE key='active_root_certificate'" + ).fetchone() result['active_root_certificate'] = loads(row[0]) if row else None if 'certificates' in tables: inventory = [] for identifier, subject, parent, revoked, body in connection.execute( - 'SELECT id,subject,parent,revoked,body FROM certificates ORDER BY id'): + 'SELECT id,subject,parent,revoked,body FROM certificates ORDER BY id' + ): certificate = decode(Certificate, loads(body)) - require((certificate.resource_id, certificate.subject_id, certificate.parent_certificate_id) == - (identifier, subject, parent), 'preflight_certificate_mismatch') + require( + (certificate.resource_id, certificate.subject_id, certificate.parent_certificate_id) + == (identifier, subject, parent), + 'preflight_certificate_mismatch', + ) if certificate.kind != 'ca': continue policy = certificate.issuance grants = (*certificate.grants, *(policy.issue_grants if policy is not None else ())) operations = sorted({operation for grant in grants for operation in grant.operations}) - inventory.append({'id': identifier, 'subject': subject, 'parent': parent, - 'revoked': bool(revoked), 'key_id': certificate.key_id, + inventory.append({ + 'id': identifier, + 'subject': subject, + 'parent': parent, + 'revoked': bool(revoked), + 'key_id': certificate.key_id, 'signed_grants_digest': digest(certificate.grants), - 'issuance_digest': digest(policy), 'operation_count': len(operations), + 'issuance_digest': digest(policy), + 'operation_count': len(operations), 'explicit_operations': all('*' not in operation for operation in operations), 'max_child_ca_depth': policy.max_child_ca_depth if policy is not None else None, 'max_cert_ttl_seconds': policy.max_cert_ttl_seconds if policy is not None else None, - 'next_step': 'explicit_operator_review_only'}) + 'next_step': 'explicit_operator_review_only', + }) result['ca_inventory'] = inventory return result def preflight(config_dir): """The config directory is explicit; no accidental default production probe.""" - result = {'format': 'msg-preflight-v1', 'captured_at': datetime.now(UTC).isoformat(), - 'decision': 'blocked', 'source_kind': 'unattested', 'mutation_performed': False, - 'root_private_material': 'not_opened', 'outbound_effects': 'not_run', - 'field_evidence': [{'id': key, 'issue': issue, 'required': description, 'status': 'missing'} - for key, issue, description in _FIELD_EVIDENCE]} + result = { + 'format': 'msg-preflight-v1', + 'captured_at': datetime.now(UTC).isoformat(), + 'decision': 'blocked', + 'source_kind': 'unattested', + 'mutation_performed': False, + 'root_private_material': 'not_opened', + 'outbound_effects': 'not_run', + 'field_evidence': [ + {'id': key, 'issue': issue, 'required': description, 'status': 'missing'} + for key, issue, description in _FIELD_EVIDENCE + ], + } try: settings = load_settings(Path(config_dir)) config = settings.config_dir / 'msgd.toml' if not config.is_file(): config = settings.config_dir / 'server.toml' result['config_digest'] = digest(config.read_bytes()) - except (OSError, Failure, ValueError, KeyError): + except OSError, Failure, ValueError, KeyError: return dict(result, error='preflight_configuration_unavailable') try: with psycopg.connect(settings.server.postgres_dsn, connect_timeout=5) as connection: connection.execute('SET TRANSACTION ISOLATION LEVEL REPEATABLE READ, READ ONLY') result['database'] = inspect_database(connection) - except (psycopg.Error, Failure, ValueError, KeyError): + except psycopg.Error, Failure, ValueError, KeyError: result['error'] = 'preflight_database_unavailable_or_inconsistent' return result diff --git a/src/msg/admin/recovery_policy.py b/src/msg/admin/recovery_policy.py index 8ec3bb36..1bd4a80f 100644 --- a/src/msg/admin/recovery_policy.py +++ b/src/msg/admin/recovery_policy.py @@ -1,16 +1,27 @@ """Typed policy restrictions for pinned recovery v2; never a promotion contract.""" -from dataclasses import replace + import re +from dataclasses import replace from msg.constants import ROOT_SUBJECT from msg.core.codec import decode, digest, loads, parse_time from msg.core.errors import require from msg.core.models import Certificate -FACTS = frozenset({'resource.acl.restrict', 'topic.policy.restrict', 'topic_ban.set', 'certificate.current', 'resource.authority.reconcile'}) -DOMAINS = {'resource.acl.restrict': 'resource_acl', 'topic.policy.restrict': 'topic_policy', - 'topic_ban.set': 'topic_ban', 'certificate.current': 'certificates', - 'resource.authority.reconcile': 'resource_authority'} +FACTS = frozenset({ + 'resource.acl.restrict', + 'topic.policy.restrict', + 'topic_ban.set', + 'certificate.current', + 'resource.authority.reconcile', +}) +DOMAINS = { + 'resource.acl.restrict': 'resource_acl', + 'topic.policy.restrict': 'topic_policy', + 'topic_ban.set': 'topic_ban', + 'certificate.current': 'certificates', + 'resource.authority.reconcile': 'resource_authority', +} def validate(fact): @@ -19,31 +30,55 @@ def validate(fact): if kind == 'resource.authority.reconcile': require(set(value) == {'previous', 'current'}, 'recovery_checkpoint_invalid') for state in value.values(): - require(type(state) is dict and set(state) == {'owner', 'group', 'parent'} and - all(type(state[k]) is str and 0 < len(state[k]) <= 256 for k in ('owner', 'group')) and - (state['parent'] is None or - (type(state['parent']) is str and 0 < len(state['parent']) <= 256)), - 'recovery_checkpoint_invalid') + require( + type(state) is dict + and set(state) == {'owner', 'group', 'parent'} + and all( + type(state[k]) is str and 0 < len(state[k]) <= 256 for k in ('owner', 'group') + ) + and ( + state['parent'] is None + or (type(state['parent']) is str and 0 < len(state['parent']) <= 256) + ), + 'recovery_checkpoint_invalid', + ) require(fact['subject'] == value['previous']['owner'], 'recovery_checkpoint_invalid') elif kind == 'certificate.current': - require(set(value) == {'body_digest'} and - (value['body_digest'] is None or - (type(value['body_digest']) is str and - re.fullmatch(r'sha256:[0-9a-f]{64}', value['body_digest']) is not None)), - 'recovery_checkpoint_invalid') + require( + set(value) == {'body_digest'} + and ( + value['body_digest'] is None + or ( + type(value['body_digest']) is str + and re.fullmatch(r'sha256:[0-9a-f]{64}', value['body_digest']) is not None + ) + ), + 'recovery_checkpoint_invalid', + ) elif kind == 'resource.acl.restrict': - require(set(value) == {'owner', 'group', 'mode'} and - all(type(value[k]) is str and 0 < len(value[k]) <= 256 for k in ('owner', 'group')) and - type(value['mode']) is int and 0 <= value['mode'] <= 0o777, - 'recovery_checkpoint_invalid') + require( + set(value) == {'owner', 'group', 'mode'} + and all(type(value[k]) is str and 0 < len(value[k]) <= 256 for k in ('owner', 'group')) + and type(value['mode']) is int + and 0 <= value['mode'] <= 0o777, + 'recovery_checkpoint_invalid', + ) elif kind == 'topic.policy.restrict': - require(set(value) == {'membership_policy'} and - value['membership_policy'] in ('open', 'approval', 'invite', 'closed'), - 'recovery_checkpoint_invalid') + require( + set(value) == {'membership_policy'} + and value['membership_policy'] in ('open', 'approval', 'invite', 'closed'), + 'recovery_checkpoint_invalid', + ) else: - require(kind == 'topic_ban.set' and set(value) == {'expires_at'} and - (value['expires_at'] is None or parse_time(value['expires_at']) > parse_time(fact['at'])), - 'recovery_checkpoint_invalid') + require( + kind == 'topic_ban.set' + and set(value) == {'expires_at'} + and ( + value['expires_at'] is None + or parse_time(value['expires_at']) > parse_time(fact['at']) + ), + 'recovery_checkpoint_invalid', + ) async def apply(tx, fact): @@ -53,8 +88,10 @@ async def apply(tx, fact): row = tx.one('SELECT subject,revoked,body FROM certificates WHERE id=?', (target,)) require(row is not None, 'recovery_fact_missing') original = decode(Certificate, loads(row[2])) - require(row[0] == original.subject_id == subject and original.resource_id == target, - 'recovery_fact_subject_mismatch') + require( + row[0] == original.subject_id == subject and original.resource_id == target, + 'recovery_fact_subject_mismatch', + ) # The exact signed body includes grants, constraints, issuance policy, # issuer/parent/source bindings and validity. Never rewrite it to invent # a constrained signature, or reactivate an already revoked certificate. @@ -65,6 +102,7 @@ async def apply(tx, fact): resource = await tx.resource(target) if kind == 'resource.authority.reconcile': from msg.security.quarantine import active + require(active(tx), 'recovery_quarantine_required') previous, current = value['previous'], value['current'] actual = {field: getattr(resource, field) for field in ('owner', 'group', 'parent')} @@ -74,27 +112,45 @@ async def apply(tx, fact): if current['parent'] is not None: require(current['parent'] != target, 'parent_cycle') await tx.resource(current['parent']) - require(all(item.id != target for item in await tx.ancestors(current['parent'])), 'parent_cycle') + require( + all(item.id != target for item in await tx.ancestors(current['parent'])), + 'parent_cycle', + ) if actual == current: return False # These are staged current facts, not authority to reopen the service. # Old and new scope-derived rights remain blocked globally until a later # complete, independently verified authority reconstruction is possible. - await tx.replace(replace(resource, **current, generation=resource.generation + 1, - modified_at=max(resource.modified_at, parse_time(fact['at'])), - modified_by=ROOT_SUBJECT), resource.generation) + await tx.replace( + replace( + resource, + **current, + generation=resource.generation + 1, + modified_at=max(resource.modified_at, parse_time(fact['at'])), + modified_by=ROOT_SUBJECT, + ), + resource.generation, + ) return True require(resource.owner == subject, 'recovery_fact_subject_mismatch') if kind == 'resource.acl.restrict': - require(resource.owner == value['owner'] and resource.group == value['group'], - 'recovery_policy_authority_change_unsupported') + require( + resource.owner == value['owner'] and resource.group == value['group'], + 'recovery_policy_authority_change_unsupported', + ) mode = (resource.mode & 0o7000) | (resource.mode & value['mode']) if resource.mode == mode: return False - await tx.replace(replace(resource, mode=mode, generation=resource.generation + 1, - modified_at=max(resource.modified_at, parse_time(fact['at'])), - modified_by=ROOT_SUBJECT), - resource.generation) + await tx.replace( + replace( + resource, + mode=mode, + generation=resource.generation + 1, + modified_at=max(resource.modified_at, parse_time(fact['at'])), + modified_by=ROOT_SUBJECT, + ), + resource.generation, + ) return True require(resource.type == 'topic', 'not_a_topic') if kind == 'topic.policy.restrict': @@ -110,8 +166,11 @@ async def apply(tx, fact): desired = 'closed' if previous == desired: return False - tx.execute('INSERT INTO topic_settings (topic,membership_policy) VALUES (?,?) ' - 'ON CONFLICT(topic) DO UPDATE SET membership_policy=excluded.membership_policy', - (target, desired), write=True) + tx.execute( + 'INSERT INTO topic_settings (topic,membership_policy) VALUES (?,?) ' + 'ON CONFLICT(topic) DO UPDATE SET membership_policy=excluded.membership_policy', + (target, desired), + write=True, + ) return True raise AssertionError('topic_ban.set uses the existing replay membership-removal path') diff --git a/src/msg/admin/recovery_proof.py b/src/msg/admin/recovery_proof.py index fe68809c..6447447a 100644 --- a/src/msg/admin/recovery_proof.py +++ b/src/msg/admin/recovery_proof.py @@ -1,14 +1,15 @@ """Strict complete-state recovery proof; physical local promotion is a separate ceremony.""" + from dataclasses import dataclass, replace from datetime import timedelta from uuid import uuid4 -from msg.admin.recovery_state import snapshot, runtime_controls, _runtime_row -from msg.constants import ROOT_SUBJECT, ONLINE_CA -from msg.core.codec import canonical, decode, digest, loads, parse_time, wire, unb64 +from msg.admin.recovery_state import _runtime_row, runtime_controls, snapshot +from msg.constants import ONLINE_CA, ROOT_SUBJECT +from msg.core.codec import canonical, decode, digest, loads, parse_time, unb64, wire from msg.core.errors import Failure, require from msg.core.models import AuditEvent, Certificate, Event, ResourceRef, Signature -from msg.security.crypto import verify, Ed25519Signer +from msg.security.crypto import Ed25519Signer, verify from msg.security.quarantine import SETTING, active FORMAT = 'msg-complete-recovery-state-v1' @@ -28,27 +29,64 @@ class IndependentRecoveryPin: def verify_proof(packet, pin, now): require(isinstance(pin, IndependentRecoveryPin), 'recovery_independent_pin_required') try: - require(type(packet) is dict and set(packet) == {'state', 'signature'}, 'recovery_proof_invalid') + require( + type(packet) is dict and set(packet) == {'state', 'signature'}, 'recovery_proof_invalid' + ) body = packet['state'] - require(type(body) is dict and set(body) == {'format', 'service', 'source_backup_sha256', - 'sequence', 'issued_at', 'expires_at', 'runtime_controls', 'metadata', 'files'}, 'recovery_proof_invalid') - require(body['format'] == FORMAT and body['service'] == pin.service and - body['source_backup_sha256'] == pin.source_backup_sha256 and - type(pin.public_key) is bytes and len(pin.public_key) == 32 and - type(pin.sequence) is int and pin.sequence >= 1 and type(body['sequence']) is int and - body['sequence'] == pin.sequence and digest(body) == pin.digest, - 'recovery_proof_pin_mismatch') + require( + type(body) is dict + and set(body) + == { + 'format', + 'service', + 'source_backup_sha256', + 'sequence', + 'issued_at', + 'expires_at', + 'runtime_controls', + 'metadata', + 'files', + }, + 'recovery_proof_invalid', + ) + require( + body['format'] == FORMAT + and body['service'] == pin.service + and body['source_backup_sha256'] == pin.source_backup_sha256 + and type(pin.public_key) is bytes + and len(pin.public_key) == 32 + and type(pin.sequence) is int + and pin.sequence >= 1 + and type(body['sequence']) is int + and body['sequence'] == pin.sequence + and digest(body) == pin.digest, + 'recovery_proof_pin_mismatch', + ) controls = body['runtime_controls'] - require(type(controls) is dict and set(controls) == {'present', 'fields'} and - type(controls['present']) is bool and type(controls['fields']) is dict and - set(controls['fields']) == {'accept_writes', 'cleanup_enabled'}, 'recovery_proof_invalid') + require( + type(controls) is dict + and set(controls) == {'present', 'fields'} + and type(controls['present']) is bool + and type(controls['fields']) is dict + and set(controls['fields']) == {'accept_writes', 'cleanup_enabled'}, + 'recovery_proof_invalid', + ) for field in controls['fields'].values(): - require(type(field) is dict and set(field) == {'present', 'value'} and - type(field['present']) is bool and type(field['value']) is bool, 'recovery_proof_invalid') - require(type(body['metadata']) is dict and type(body['files']) is dict, 'recovery_proof_invalid') + require( + type(field) is dict + and set(field) == {'present', 'value'} + and type(field['present']) is bool + and type(field['value']) is bool, + 'recovery_proof_invalid', + ) + require( + type(body['metadata']) is dict and type(body['files']) is dict, 'recovery_proof_invalid' + ) start, end = parse_time(body['issued_at']), parse_time(body['expires_at']) - require(start <= now < end and end-start <= timedelta(hours=24), 'recovery_proof_expired') - verify(pin.public_key, canonical(body), decode(Signature, packet['signature']), purpose=PURPOSE) + require(start <= now < end and end - start <= timedelta(hours=24), 'recovery_proof_expired') + verify( + pin.public_key, canonical(body), decode(Signature, packet['signature']), purpose=PURPOSE + ) return loads(canonical(body)) except (KeyError, TypeError, ValueError, OverflowError) as exc: raise Failure('recovery_proof_invalid') from exc @@ -56,10 +94,16 @@ def verify_proof(packet, pin, now): async def verify_current_authority(app, tx, public_key): from msg.security.backup_retirement import root_verifier - require(await root_verifier(tx, now=app.clock()) == public_key == app.certificates.root_public_key, - 'recovery_root_mismatch') + + require( + await root_verifier(tx, now=app.clock()) == public_key == app.certificates.root_public_key, + 'recovery_root_mismatch', + ) generation = tx.rows("SELECT value FROM settings WHERE key='recovery_runtime_generation'") - require(not generation or (type(loads(generation[0][0])) is str and bool(loads(generation[0][0]))), 'recovery_runtime_invalid') + require( + not generation or (type(loads(generation[0][0])) is str and bool(loads(generation[0][0]))), + 'recovery_runtime_invalid', + ) root = await tx.subject(ROOT_SUBJECT) require(root.local_only, 'root_policy_corrupt') for (identifier,) in tx.rows('SELECT id FROM certificates WHERE revoked=0 ORDER BY id'): @@ -71,17 +115,23 @@ async def verify_current_authority(app, tx, public_key): online = tx.setting('online_ca_certificate') require(type(online) is str, 'recovery_online_issuer_missing') certificate = await app.certificates.validate(online, tx) - require(certificate.subject_id == ONLINE_CA and certificate.key_id == app.online_signer.key_id, - 'recovery_online_key_mismatch') + require( + certificate.subject_id == ONLINE_CA and certificate.key_id == app.online_signer.key_id, + 'recovery_online_key_mismatch', + ) # The receipt signer has a registered credential, independent of online CA. receipt = tx.setting('receipt_public_key') - require(type(receipt) is dict and receipt.get('key_id') == app.receipt_signer.key_id and - unb64(receipt.get('public_key', ''), limit=32) == app.receipt_signer.public_key, - 'recovery_receipt_key_mismatch') + require( + type(receipt) is dict + and receipt.get('key_id') == app.receipt_signer.key_id + and unb64(receipt.get('public_key', ''), limit=32) == app.receipt_signer.public_key, + 'recovery_receipt_key_mismatch', + ) for (identifier,) in tx.rows('SELECT id FROM resources ORDER BY id'): await tx.ancestors(identifier) from msg.admin.market_check import inspect_market from msg.storage.ledger_migration import _validate_ledger + _validate_ledger(tx._connection) await inspect_market(tx) @@ -92,165 +142,348 @@ def configuration_state(settings): for key in ('listen', 'port'): value.pop(key) value['hosting_recovery_marker'] = settings.hosting_recovery_marker is not None - for key in ('config_dir', 'postgres_dsn', 'valkey_url', 'mail', 'content_dir', - 'repositories_dir', 'blob_dir', 'staging_dir', 'service_keys_dir'): + for key in ( + 'config_dir', + 'postgres_dsn', + 'valkey_url', + 'mail', + 'content_dir', + 'repositories_dir', + 'blob_dir', + 'staging_dir', + 'service_keys_dir', + ): value['server'].pop(key) return value def file_state(app, tx): - from msg.admin.backups import _db_refs, _verify_storage, _regular_tree, _hash + from msg.admin.backups import _db_refs, _hash, _regular_tree, _verify_storage + settings = app.settings from msg.config import load_settings + actual_settings = load_settings(settings.config_dir) if active(tx): - require(actual_settings.server.mail is None and actual_settings.server.valkey_url is None, 'recovery_target_not_isolated') + require( + actual_settings.server.mail is None and actual_settings.server.valkey_url is None, + 'recovery_target_not_isolated', + ) policy = configuration_state(actual_settings) require(actual_settings == settings, 'recovery_cached_configuration_mismatch') - require(settings.trust_file.is_file() and not settings.trust_file.is_symlink(), 'invalid_trust_anchor') + require( + settings.trust_file.is_file() and not settings.trust_file.is_symlink(), + 'invalid_trust_anchor', + ) trees = {} - for name, path in (('content', settings.server.content_dir), ('blobs', settings.server.blob_dir), - ('repositories', settings.server.repositories_dir), ('staging', settings.server.staging_dir), - ('service', settings.service_keys)): + for name, path in ( + ('content', settings.server.content_dir), + ('blobs', settings.server.blob_dir), + ('repositories', settings.server.repositories_dir), + ('staging', settings.server.staging_dir), + ('service', settings.service_keys), + ): if name in {'repositories', 'staging'} and not path.exists() and not path.is_symlink(): trees[name] = {} continue _regular_tree(path) - trees[name] = {str(file.relative_to(path)): {'size': file.stat().st_size, 'sha256': _hash(file)} - for file in sorted(path.rglob('*')) if file.is_file()} + trees[name] = { + str(file.relative_to(path)): {'size': file.stat().st_size, 'sha256': _hash(file)} + for file in sorted(path.rglob('*')) + if file.is_file() + } refs = _db_refs(tx._connection) for (identifier,) in tx.rows("SELECT id FROM resources WHERE type='repo' AND state<>'purged'"): - require((settings.server.repositories_dir/(identifier+'.git')).is_dir(), 'backup_repository_missing') - _verify_storage(settings.server.content_dir.parent, refs, settings=settings, repair_git_layout=False) - require(Ed25519Signer.from_bytes((settings.service_keys/'online.key').read_bytes()).public_key == - app.online_signer.public_key and - Ed25519Signer.from_bytes((settings.service_keys/'receipt.key').read_bytes()).public_key == - app.receipt_signer.public_key and - (settings.service_keys/'tokens.key').read_bytes() == app._token_secret, 'recovery_cached_keys_mismatch') - keys = {name: digest((settings.service_keys/name).read_bytes()) - for name in ('online.key', 'receipt.key', 'tokens.key')} - require(len((settings.service_keys/'tokens.key').read_bytes()) == 32, 'invalid_service_key') + require( + (settings.server.repositories_dir / (identifier + '.git')).is_dir(), + 'backup_repository_missing', + ) + _verify_storage( + settings.server.content_dir.parent, refs, settings=settings, repair_git_layout=False + ) + require( + Ed25519Signer.from_bytes((settings.service_keys / 'online.key').read_bytes()).public_key + == app.online_signer.public_key + and Ed25519Signer.from_bytes( + (settings.service_keys / 'receipt.key').read_bytes() + ).public_key + == app.receipt_signer.public_key + and (settings.service_keys / 'tokens.key').read_bytes() == app._token_secret, + 'recovery_cached_keys_mismatch', + ) + keys = { + name: digest((settings.service_keys / name).read_bytes()) + for name in ('online.key', 'receipt.key', 'tokens.key') + } + require(len((settings.service_keys / 'tokens.key').read_bytes()) == 32, 'invalid_service_key') # Digests commit random service key material without exporting it. - return {'configuration': policy, 'references': refs, 'trees': trees, 'service_key_digests': keys, - 'trust': loads(settings.trust_file.read_bytes())} + return { + 'configuration': policy, + 'references': refs, + 'trees': trees, + 'service_key_digests': keys, + 'trust': loads(settings.trust_file.read_bytes()), + } async def draft(app, *, public_key, source_backup_sha256, sequence): """Internal use case; public RootAdmin entry first enforces console and PIN.""" - require(type(sequence) is int and sequence > 0 and type(source_backup_sha256) is str and - len(source_backup_sha256) == 64 and all(c in '0123456789abcdef' for c in source_backup_sha256), - 'recovery_proof_invalid') + require( + type(sequence) is int + and sequence > 0 + and type(source_backup_sha256) is str + and len(source_backup_sha256) == 64 + and all(c in '0123456789abcdef' for c in source_backup_sha256), + 'recovery_proof_invalid', + ) async with app.metadata.transaction(write=True) as tx: - require(not active(tx) and not (app.settings.config_dir/'recovery-drill.json').exists() and not (app.settings.config_dir/'recovery-drill.json').is_symlink(), 'recovery_source_is_quarantined') + require( + not active(tx) + and not (app.settings.config_dir / 'recovery-drill.json').exists() + and not (app.settings.config_dir / 'recovery-drill.json').is_symlink(), + 'recovery_source_is_quarantined', + ) state = snapshot(tx) await verify_current_authority(app, tx, public_key) - body = {'format': FORMAT, 'service': app.settings.service_url, - 'source_backup_sha256': source_backup_sha256, 'sequence': sequence, - 'issued_at': wire(app.clock()), 'expires_at': wire(app.clock()+timedelta(hours=24)), - 'runtime_controls': runtime_controls(tx), 'metadata': state, 'files': file_state(app, tx)} + body = { + 'format': FORMAT, + 'service': app.settings.service_url, + 'source_backup_sha256': source_backup_sha256, + 'sequence': sequence, + 'issued_at': wire(app.clock()), + 'expires_at': wire(app.clock() + timedelta(hours=24)), + 'runtime_controls': runtime_controls(tx), + 'metadata': state, + 'files': file_state(app, tx), + } return body async def seal(app, body, signer): async with app.metadata.transaction(write=True) as tx: - require(not active(tx) and not (app.settings.config_dir/'recovery-drill.json').exists() and not (app.settings.config_dir/'recovery-drill.json').is_symlink(), 'recovery_source_is_quarantined') - require(body['service'] == app.settings.service_url and snapshot(tx) == body['metadata'] and - runtime_controls(tx) == body['runtime_controls'] and file_state(app, tx) == body['files'], - 'recovery_complete_state_mismatch') + require( + not active(tx) + and not (app.settings.config_dir / 'recovery-drill.json').exists() + and not (app.settings.config_dir / 'recovery-drill.json').is_symlink(), + 'recovery_source_is_quarantined', + ) + require( + body['service'] == app.settings.service_url + and snapshot(tx) == body['metadata'] + and runtime_controls(tx) == body['runtime_controls'] + and file_state(app, tx) == body['files'], + 'recovery_complete_state_mismatch', + ) await verify_current_authority(app, tx, signer.public_key) - require(parse_time(body['issued_at']) <= app.clock() < parse_time(body['expires_at']), 'recovery_proof_expired') + require( + parse_time(body['issued_at']) <= app.clock() < parse_time(body['expires_at']), + 'recovery_proof_expired', + ) return {'state': body, 'signature': wire(signer.sign(canonical(body), purpose=PURPOSE))} async def capture(app, signer, *, source_backup_sha256, sequence): - body = await draft(app, public_key=signer.public_key, source_backup_sha256=source_backup_sha256, sequence=sequence) + body = await draft( + app, + public_key=signer.public_key, + source_backup_sha256=source_backup_sha256, + sequence=sequence, + ) return await seal(app, body, signer) def _receipt(value, pin): - require(type(value) is dict and set(value) == {'receipt', 'signature'}, 'recovery_promotion_receipt_invalid') + require( + type(value) is dict and set(value) == {'receipt', 'signature'}, + 'recovery_promotion_receipt_invalid', + ) body = value['receipt'] - require(type(body) is dict and set(body) == {'manifest_digest', 'sequence', 'source_backup_sha256', - 'generation', 'prior_generation', 'audit_row', 'prior_audit_sequence'}, 'recovery_promotion_receipt_invalid') - require(type(body['generation']) is str and bool(body['generation']) and - (body['prior_generation'] is None or (type(body['prior_generation']) is str and bool(body['prior_generation']))) and - type(body['audit_row']) is list and len(body['audit_row']) == 4 and - type(body['audit_row'][0]) is int and body['audit_row'][0] > 0 and - type(body['prior_audit_sequence']) is list and len(body['prior_audit_sequence']) == 2 and - type(body['prior_audit_sequence'][0]) is int and type(body['prior_audit_sequence'][1]) is bool, - 'recovery_promotion_receipt_invalid') - verify(pin.public_key, canonical(body), decode(Signature, value['signature']), purpose=RECEIPT_PURPOSE) - require(body['manifest_digest'] == pin.digest and body['sequence'] == pin.sequence and - body['source_backup_sha256'] == pin.source_backup_sha256, 'recovery_promotion_receipt_mismatch') + require( + type(body) is dict + and set(body) + == { + 'manifest_digest', + 'sequence', + 'source_backup_sha256', + 'generation', + 'prior_generation', + 'audit_row', + 'prior_audit_sequence', + }, + 'recovery_promotion_receipt_invalid', + ) + require( + type(body['generation']) is str + and bool(body['generation']) + and ( + body['prior_generation'] is None + or (type(body['prior_generation']) is str and bool(body['prior_generation'])) + ) + and type(body['audit_row']) is list + and len(body['audit_row']) == 4 + and type(body['audit_row'][0]) is int + and body['audit_row'][0] > 0 + and type(body['prior_audit_sequence']) is list + and len(body['prior_audit_sequence']) == 2 + and type(body['prior_audit_sequence'][0]) is int + and type(body['prior_audit_sequence'][1]) is bool, + 'recovery_promotion_receipt_invalid', + ) + verify( + pin.public_key, + canonical(body), + decode(Signature, value['signature']), + purpose=RECEIPT_PURPOSE, + ) + require( + body['manifest_digest'] == pin.digest + and body['sequence'] == pin.sequence + and body['source_backup_sha256'] == pin.source_backup_sha256, + 'recovery_promotion_receipt_mismatch', + ) return body async def promote(app, packet, *, pin, signer, operator): """Internal fixture-capable use case. Caller must enforce physical console ceremony.""" body = verify_proof(packet, pin, app.clock()) - require(app.settings.server.mail is None and app.settings.server.valkey_url is None, - 'recovery_target_not_isolated') + require( + app.settings.server.mail is None and app.settings.server.valkey_url is None, + 'recovery_target_not_isolated', + ) require(signer.public_key == pin.public_key, 'recovery_root_mismatch') require(type(operator) is str and bool(operator), 'recovery_operator_required') - marker = app.settings.config_dir/'recovery-drill.json' + marker = app.settings.config_dir / 'recovery-drill.json' from msg.security.root_files import read_private + marker_present = marker.exists() or marker.is_symlink() if marker_present: marker_value = loads(read_private(marker)) - require(type(marker_value) is dict and marker_value.get('source_backup_sha256') == pin.source_backup_sha256, - 'recovery_checkpoint_backup_mismatch') + require( + type(marker_value) is dict + and marker_value.get('source_backup_sha256') == pin.source_backup_sha256, + 'recovery_checkpoint_backup_mismatch', + ) async with app.metadata.transaction(write=True) as tx: previous = tx.setting('recovery_promotion') # A backup of an already recovered source contains a historical receipt. # Only the exact current manifest can resume its pending delta; a new # recovery must independently match the entire initial source state. - pending = type(previous) is dict and type(previous.get('receipt')) is dict and previous['receipt'].get('manifest_digest') == pin.digest + pending = ( + type(previous) is dict + and type(previous.get('receipt')) is dict + and previous['receipt'].get('manifest_digest') == pin.digest + ) resume = _receipt(previous, pin) if pending else None gate = tx.setting(SETTING) - require(active(tx) and type(gate) is dict and gate.get('source_backup_sha256') == pin.source_backup_sha256, - 'recovery_quarantine_required') + require( + active(tx) + and type(gate) is dict + and gate.get('source_backup_sha256') == pin.source_backup_sha256, + 'recovery_quarantine_required', + ) if resume is None: require(marker_present, 'recovery_marker_required') - require(active(tx) and type(gate) is dict and gate.get('source_backup_sha256') == pin.source_backup_sha256 and - gate.get('format') == 'msg-recovery-quarantine-v1' and gate.get('authority') == 'health_only' and gate.get('outbound_enabled') is False, - 'recovery_quarantine_required') - actual = snapshot(tx, controls=body['runtime_controls'] if resume is None else None, resume=resume) + require( + active(tx) + and type(gate) is dict + and gate.get('source_backup_sha256') == pin.source_backup_sha256 + and gate.get('format') == 'msg-recovery-quarantine-v1' + and gate.get('authority') == 'health_only' + and gate.get('outbound_enabled') is False, + 'recovery_quarantine_required', + ) + actual = snapshot( + tx, controls=body['runtime_controls'] if resume is None else None, resume=resume + ) require(actual == body['metadata'], 'recovery_complete_state_mismatch') require(file_state(app, tx) == body['files'], 'recovery_complete_files_mismatch') await verify_current_authority(app, tx, pin.public_key) verify_proof(packet, pin, app.clock()) if resume is None: prior_generation = tx.setting('recovery_runtime_generation') - require(prior_generation is None or type(prior_generation) is str, 'recovery_runtime_invalid') + require( + prior_generation is None or type(prior_generation) is str, + 'recovery_runtime_invalid', + ) generation = uuid4().hex - event = Event(id='audit_'+uuid4().hex, type='recovery.complete_promotion', time=app.clock(), - request_id=pin.digest, actor=ROOT_SUBJECT, subject=ROOT_SUBJECT, resources=(), - data={'manifest_digest': pin.digest, 'sequence': pin.sequence, 'operator': operator, - 'runtime_generation': generation, 'requires_runtime_restart': True}) - audit_event = AuditEvent(event=event, + event = Event( + id='audit_' + uuid4().hex, + type='recovery.complete_promotion', + time=app.clock(), + request_id=pin.digest, + actor=ROOT_SUBJECT, + subject=ROOT_SUBJECT, + resources=(), + data={ + 'manifest_digest': pin.digest, + 'sequence': pin.sequence, + 'operator': operator, + 'runtime_generation': generation, + 'requires_runtime_restart': True, + }, + ) + audit_event = AuditEvent( + event=event, authority=(ResourceRef(id=tx.setting('active_root_certificate')),), - before_digest=pin.digest, after_digest=pin.digest, previous_digest=None, - entry_digest='', result='verified_complete_state') + before_digest=pin.digest, + after_digest=pin.digest, + previous_digest=None, + entry_digest='', + result='verified_complete_state', + ) previous = tx.one('SELECT seq,digest FROM audit ORDER BY seq DESC LIMIT 1') prior_sequence = actual['sequences']['audit_seq_seq'] - audit_seq = max(prior_sequence[0] + int(prior_sequence[1]), (previous[0]+1) if previous else 1) - audit_data = wire(replace(audit_event, previous_digest=previous[1] if previous else None, entry_digest='')) + audit_seq = max( + prior_sequence[0] + int(prior_sequence[1]), (previous[0] + 1) if previous else 1 + ) + audit_data = wire( + replace( + audit_event, previous_digest=previous[1] if previous else None, entry_digest='' + ) + ) audit_data.pop('entry_digest') - audit_event = replace(audit_event, previous_digest=previous[1] if previous else None, entry_digest=digest(audit_data)) + audit_event = replace( + audit_event, + previous_digest=previous[1] if previous else None, + entry_digest=digest(audit_data), + ) # Explicit allocation is transactional. PostgreSQL nextval is not: # using it here would invalidate the source commitment on rollback. - tx.execute('INSERT INTO audit (seq,digest,previous,body) VALUES (?,?,?,?)', - (audit_seq, audit_event.entry_digest, audit_event.previous_digest, canonical(audit_event).decode()), write=True) - audit_row = tx.one('SELECT seq,digest,previous,body FROM audit ORDER BY seq DESC LIMIT 1') - receipt = {'manifest_digest': pin.digest, 'sequence': pin.sequence, - 'source_backup_sha256': pin.source_backup_sha256, 'generation': generation, - 'prior_generation': prior_generation, 'audit_row': loads(canonical(audit_row)), - 'prior_audit_sequence': actual['sequences']['audit_seq_seq']} - tx.set_setting('recovery_promotion', {'receipt': receipt, - 'signature': wire(signer.sign(canonical(receipt), purpose=RECEIPT_PURPOSE))}) + tx.execute( + 'INSERT INTO audit (seq,digest,previous,body) VALUES (?,?,?,?)', + ( + audit_seq, + audit_event.entry_digest, + audit_event.previous_digest, + canonical(audit_event).decode(), + ), + write=True, + ) + audit_row = tx.one( + 'SELECT seq,digest,previous,body FROM audit ORDER BY seq DESC LIMIT 1' + ) + receipt = { + 'manifest_digest': pin.digest, + 'sequence': pin.sequence, + 'source_backup_sha256': pin.source_backup_sha256, + 'generation': generation, + 'prior_generation': prior_generation, + 'audit_row': loads(canonical(audit_row)), + 'prior_audit_sequence': actual['sequences']['audit_seq_seq'], + } + tx.set_setting( + 'recovery_promotion', + { + 'receipt': receipt, + 'signature': wire(signer.sign(canonical(receipt), purpose=RECEIPT_PURPOSE)), + }, + ) tx.set_setting('recovery_runtime_generation', generation) runtime = tx.setting('runtime_config') - normalized = _runtime_row(('runtime_config', canonical(runtime).decode()), body['runtime_controls']) + normalized = _runtime_row( + ('runtime_config', canonical(runtime).decode()), body['runtime_controls'] + ) if normalized is None: tx.execute("DELETE FROM settings WHERE key='runtime_config'", write=True) else: @@ -260,13 +493,19 @@ async def promote(app, packet, *, pin, signer, operator): # succeed. A crash between unlink and commit remains closed through the DB; # only an authenticated exact receipt can resume with the marker missing. import os + async with app.metadata.transaction(write=True) as tx: require(active(tx), 'recovery_quarantine_required') - require(_receipt(tx.setting('recovery_promotion'), pin) == resume, 'recovery_promotion_receipt_mismatch') + require( + _receipt(tx.setting('recovery_promotion'), pin) == resume, + 'recovery_promotion_receipt_mismatch', + ) require(snapshot(tx, resume=resume) == body['metadata'], 'recovery_complete_state_mismatch') require(file_state(app, tx) == body['files'], 'recovery_complete_files_mismatch') verify_proof(packet, pin, app.clock()) - tx.execute("SELECT setval('public.audit_seq_seq', ?, true)", (resume['audit_row'][0],), write=True) + tx.execute( + "SELECT setval('public.audit_seq_seq', ?, true)", (resume['audit_row'][0],), write=True + ) try: fd = os.open(marker.parent, os.O_RDONLY | os.O_DIRECTORY) try: @@ -278,27 +517,48 @@ async def promote(app, packet, *, pin, signer, operator): except OSError as exc: raise Failure('recovery_promotion_finish_required') from exc tx.execute('DELETE FROM settings WHERE key=?', (SETTING,), write=True) - return {'status': 'recovery_promoted', 'manifest_digest': pin.digest, - 'runtime_generation': resume['generation'], 'requires_runtime_restart': True} + return { + 'status': 'recovery_promoted', + 'manifest_digest': pin.digest, + 'runtime_generation': resume['generation'], + 'requires_runtime_restart': True, + } async def open_for_proof(config_dir): """Load only existing authority; no DDL, release-source sync, repair or worker.""" from msg.application import Application from msg.config import load_settings - from msg.storage.postgres import PostgresMetadataStore from msg.security.certificates import CertificateValidator + from msg.storage.postgres import PostgresMetadataStore + settings = load_settings(config_dir) app = Application(settings) app.metadata = PostgresMetadataStore(settings.server.postgres_dsn, initialize=False) - require(settings.trust_file.is_file() and not settings.trust_file.is_symlink(), 'invalid_trust_anchor') + require( + settings.trust_file.is_file() and not settings.trust_file.is_symlink(), + 'invalid_trust_anchor', + ) trust = loads(settings.trust_file.read_bytes()) - require(set(trust) == {'version', 'public_key', 'certificate'} and trust['version'] == 1, 'invalid_trust_anchor') - app.certificates = CertificateValidator(app.registry, decode(Certificate, trust['certificate']), - unb64(trust['public_key'], limit=32), settings.service_url, app.clock) + require( + set(trust) == {'version', 'public_key', 'certificate'} and trust['version'] == 1, + 'invalid_trust_anchor', + ) + app.certificates = CertificateValidator( + app.registry, + decode(Certificate, trust['certificate']), + unb64(trust['public_key'], limit=32), + settings.service_url, + app.clock, + ) from msg.admin.backups import _regular_tree + _regular_tree(settings.service_keys) - app.online_signer = Ed25519Signer.from_bytes((settings.service_keys/'online.key').read_bytes()) - app.receipt_signer = Ed25519Signer.from_bytes((settings.service_keys/'receipt.key').read_bytes()) - app._token_secret = (settings.service_keys/'tokens.key').read_bytes() + app.online_signer = Ed25519Signer.from_bytes( + (settings.service_keys / 'online.key').read_bytes() + ) + app.receipt_signer = Ed25519Signer.from_bytes( + (settings.service_keys / 'receipt.key').read_bytes() + ) + app._token_secret = (settings.service_keys / 'tokens.key').read_bytes() return app diff --git a/src/msg/admin/recovery_replay.py b/src/msg/admin/recovery_replay.py index 7ca9d14a..392b5e35 100644 --- a/src/msg/admin/recovery_replay.py +++ b/src/msg/admin/recovery_replay.py @@ -6,6 +6,7 @@ network operation, configuration override, or quarantine-clearing operation here. Unknown fact kinds require a newer reviewed replay implementation and stay blocked. """ + from __future__ import annotations import re @@ -23,17 +24,30 @@ FORMAT = 'msg-revocation-checkpoint-v1' POLICY_FORMAT = 'msg-revocation-checkpoint-v2' -SUPPORTED_FACTS = frozenset({'credential.revoke', 'certificate.revoke', 'share_grant.revoke', - 'share_grant_v2.revoke', 'share_link.revoke', 'membership.remove', - 'topic_membership.remove', 'topic_ban.apply', 'topic_ban.lift', 'identity_key.retire', - 'encryption_key.retire', 'vault.destroy'}) +SUPPORTED_FACTS = frozenset({ + 'credential.revoke', + 'certificate.revoke', + 'share_grant.revoke', + 'share_grant_v2.revoke', + 'share_link.revoke', + 'membership.remove', + 'topic_membership.remove', + 'topic_ban.apply', + 'topic_ban.lift', + 'identity_key.retire', + 'encryption_key.retire', + 'vault.destroy', +}) PROMOTION_BLOCKED_REASONS = ( 'quarantine_remains', 'supported_fact_inventory_incomplete', 'backup_retirement_not_attested', ) -_SHARE_TABLES = {'share_grant.revoke': 'share_grants', 'share_grant_v2.revoke': 'share_grants_v2', - 'share_link.revoke': 'share_links'} +_SHARE_TABLES = { + 'share_grant.revoke': 'share_grants', + 'share_grant_v2.revoke': 'share_grants_v2', + 'share_link.revoke': 'share_links', +} _MAX_ENTRIES = 10000 @@ -45,6 +59,7 @@ class TrustedCheckpointPin: It does not assert that an offline source was current or operator-approved. Production remains blocked without evidence for that separate trust ceremony. """ + service: str public_key: bytes digest: str @@ -54,26 +69,36 @@ class TrustedCheckpointPin: def verify_checkpoint(packet, *, pin): require(isinstance(pin, TrustedCheckpointPin), 'recovery_checkpoint_pin_required') try: - require(type(packet) is dict and set(packet) == {'checkpoint', 'signature'}, - 'recovery_checkpoint_invalid') + require( + type(packet) is dict and set(packet) == {'checkpoint', 'signature'}, + 'recovery_checkpoint_invalid', + ) body = packet['checkpoint'] policy = type(body) is dict and body.get('format') == POLICY_FORMAT fields = {'format', 'service', 'source_backup_sha256', 'sequence', 'entries'} if policy: fields.add('coverage') - require(type(body) is dict and set(body) == fields, - 'recovery_checkpoint_invalid') - require(type(pin.public_key) is bytes and len(pin.public_key) == 32 and - type(pin.sequence) is int and 0 <= pin.sequence <= _MAX_ENTRIES and - type(body['sequence']) is int and body['sequence'] == pin.sequence and - body['format'] in {FORMAT, POLICY_FORMAT} and body['service'] == pin.service and - isinstance(pin.service, str) and bool(pin.service) and - isinstance(body['source_backup_sha256'], str) and - re.fullmatch('[0-9a-f]{64}', body['source_backup_sha256']) is not None and - digest(body) == pin.digest, 'recovery_checkpoint_invalid') + require(type(body) is dict and set(body) == fields, 'recovery_checkpoint_invalid') + require( + type(pin.public_key) is bytes + and len(pin.public_key) == 32 + and type(pin.sequence) is int + and 0 <= pin.sequence <= _MAX_ENTRIES + and type(body['sequence']) is int + and body['sequence'] == pin.sequence + and body['format'] in {FORMAT, POLICY_FORMAT} + and body['service'] == pin.service + and isinstance(pin.service, str) + and bool(pin.service) + and isinstance(body['source_backup_sha256'], str) + and re.fullmatch('[0-9a-f]{64}', body['source_backup_sha256']) is not None + and digest(body) == pin.digest, + 'recovery_checkpoint_invalid', + ) entries = body['entries'] - require(type(entries) is list and len(entries) == pin.sequence, - 'recovery_checkpoint_invalid') + require( + type(entries) is list and len(entries) == pin.sequence, 'recovery_checkpoint_invalid' + ) domains = set() authority_chains = {} for sequence, fact in enumerate(entries, 1): @@ -81,25 +106,42 @@ def verify_checkpoint(packet, *, pin): fact_fields = {'sequence', 'kind', 'subject', 'target', 'at'} if typed: fact_fields.add('value') - require(type(fact) is dict and set(fact) == fact_fields and - type(fact['sequence']) is int and fact['sequence'] == sequence and - fact['kind'] in (SUPPORTED_FACTS | recovery_policy.FACTS if policy else SUPPORTED_FACTS) and - all(isinstance(fact[name], str) and 0 < len(fact[name]) <= 256 - for name in ('subject', 'target')), 'recovery_checkpoint_invalid') + require( + type(fact) is dict + and set(fact) == fact_fields + and type(fact['sequence']) is int + and fact['sequence'] == sequence + and fact['kind'] + in (SUPPORTED_FACTS | recovery_policy.FACTS if policy else SUPPORTED_FACTS) + and all( + isinstance(fact[name], str) and 0 < len(fact[name]) <= 256 + for name in ('subject', 'target') + ), + 'recovery_checkpoint_invalid', + ) require(parse_time(fact['at']) <= datetime.now(UTC), 'recovery_checkpoint_invalid') if typed: recovery_policy.validate(fact) if fact['kind'] == 'resource.authority.reconcile': chain = authority_chains.setdefault(fact['target'], []) - require(not chain or chain[-1]['value']['current'] == fact['value']['previous'], - 'recovery_authority_chain_gap') + require( + not chain or chain[-1]['value']['current'] == fact['value']['previous'], + 'recovery_authority_chain_gap', + ) chain.append(fact) domains.add(recovery_policy.DOMAINS[fact['kind']] if typed else 'revocations') if policy: - require(body['coverage'] == {'complete': False, 'domains': sorted(domains)} and - body['coverage'].get('complete') is False, 'recovery_checkpoint_incomplete_contract') - verify(pin.public_key, canonical(body), decode(Signature, packet['signature']), - purpose='recovery-checkpoint-v2' if policy else 'recovery-checkpoint-v1') + require( + body['coverage'] == {'complete': False, 'domains': sorted(domains)} + and body['coverage'].get('complete') is False, + 'recovery_checkpoint_incomplete_contract', + ) + verify( + pin.public_key, + canonical(body), + decode(Signature, packet['signature']), + purpose='recovery-checkpoint-v2' if policy else 'recovery-checkpoint-v1', + ) # Own the verified bytes across the subsequent await/transaction boundary. return loads(canonical(body)) except (Failure, KeyError, TypeError, ValueError, OverflowError) as exc: @@ -119,23 +161,32 @@ def _earlier(previous, at): def _revoke_credential(tx, subject, target, at): row = _owned(tx.one('SELECT subject,body FROM credentials WHERE id=?', (target,)), subject) credential = decode(Credential, loads(row[1])) - require(credential.id == target and credential.subject_id == subject, - 'recovery_fact_subject_mismatch') + require( + credential.id == target and credential.subject_id == subject, + 'recovery_fact_subject_mismatch', + ) when = parse_time(at) changed = credential.revoked_at is None or credential.revoked_at > when if changed: revoked = replace(credential, revoked_at=when) - tx.execute('UPDATE credentials SET body=? WHERE id=?', - (canonical(revoked).decode(), target), write=True) + tx.execute( + 'UPDATE credentials SET body=? WHERE id=?', + (canonical(revoked).decode(), target), + write=True, + ) if credential.kind == 'token': - delivery = tx.one('SELECT subject,consumed_at FROM token_deliveries WHERE credential_id=?', - (target,)) + delivery = tx.one( + 'SELECT subject,consumed_at FROM token_deliveries WHERE credential_id=?', (target,) + ) if delivery is not None: _owned(delivery, subject) earliest = _earlier(delivery[1], at) if earliest != delivery[1]: - tx.execute('UPDATE token_deliveries SET consumed_at=? WHERE credential_id=?', - (earliest, target), write=True) + tx.execute( + 'UPDATE token_deliveries SET consumed_at=? WHERE credential_id=?', + (earliest, target), + write=True, + ) changed = True return changed @@ -147,42 +198,60 @@ async def _apply(tx, fact): if kind == 'credential.revoke': return _revoke_credential(tx, subject, target, at) if kind == 'certificate.revoke': - row = _owned(tx.one('SELECT subject,revoked,body FROM certificates WHERE id=?', - (target,)), subject) + row = _owned( + tx.one('SELECT subject,revoked,body FROM certificates WHERE id=?', (target,)), subject + ) certificate = decode(Certificate, loads(row[2])) - require(certificate.resource_id == target and certificate.subject_id == subject, - 'recovery_fact_subject_mismatch') + require( + certificate.resource_id == target and certificate.subject_id == subject, + 'recovery_fact_subject_mismatch', + ) if row[1] != 1: tx.execute('UPDATE certificates SET revoked=1 WHERE id=?', (target,), write=True) return True return False if kind in _SHARE_TABLES: table = _SHARE_TABLES[kind] # Fixed closed vocabulary, never caller SQL. - row = _owned(tx.one(f'SELECT grantor,revoked_at FROM {table} WHERE id=?', (target,)), subject) + row = _owned( + tx.one(f'SELECT grantor,revoked_at FROM {table} WHERE id=?', (target,)), subject + ) earliest = _earlier(row[1], at) if row[1] != earliest: - tx.execute(f'UPDATE {table} SET revoked_at=? WHERE id=?', (earliest, target), write=True) + tx.execute( + f'UPDATE {table} SET revoked_at=? WHERE id=?', (earliest, target), write=True + ) return True return False if kind == 'membership.remove': require(target != 'g_public', 'recovery_virtual_membership_requires_identity_policy') - row = tx.one('SELECT generation,body FROM memberships WHERE org=? AND subject=?', - (target, subject)) + row = tx.one( + 'SELECT generation,body FROM memberships WHERE org=? AND subject=?', (target, subject) + ) require(row is not None, 'recovery_fact_missing') member = decode(Membership, loads(row[1])) - require(member.subject_id == subject and member.organization_id == target and - member.version == row[0], 'recovery_fact_subject_mismatch') + require( + member.subject_id == subject + and member.organization_id == target + and member.version == row[0], + 'recovery_fact_subject_mismatch', + ) if member.status != 'rejected': - await tx.update_identity(replace(member, status='rejected', version=member.version + 1), - member.version) + await tx.update_identity( + replace(member, status='rejected', version=member.version + 1), member.version + ) return True return False if kind == 'topic_membership.remove': - row = tx.one('SELECT status FROM topic_memberships WHERE topic=? AND subject=?', (target, subject)) + row = tx.one( + 'SELECT status FROM topic_memberships WHERE topic=? AND subject=?', (target, subject) + ) require(row is not None, 'recovery_fact_missing') if row[0] != 'removed': - tx.execute("UPDATE topic_memberships SET status='removed' WHERE topic=? AND subject=?", - (target, subject), write=True) + tx.execute( + "UPDATE topic_memberships SET status='removed' WHERE topic=? AND subject=?", + (target, subject), + write=True, + ) return True return False if kind in {'topic_ban.apply', 'topic_ban.set'}: @@ -190,23 +259,37 @@ async def _apply(tx, fact): if kind == 'topic_ban.set': await tx.subject(subject) require((await tx.resource(target)).type == 'topic', 'not_a_topic') - require(tx.one('SELECT id FROM resources WHERE id=?', (target,)) is not None, - 'recovery_fact_missing') + require( + tx.one('SELECT id FROM resources WHERE id=?', (target,)) is not None, + 'recovery_fact_missing', + ) # A live ban removes membership as well as denying topic participation. # Preserve that effect when replaying an old snapshot, including when a # later fact lifts the ban: lifting never restores an old admin role. - member = tx.one('SELECT role,status FROM topic_memberships WHERE topic=? AND subject=?', - (target, subject)) + member = tx.one( + 'SELECT role,status FROM topic_memberships WHERE topic=? AND subject=?', + (target, subject), + ) member_changed = member is not None and member != ('member', 'removed') if member_changed: - tx.execute("UPDATE topic_memberships SET role='member',status='removed' " - 'WHERE topic=? AND subject=?', (target, subject), write=True) - row = tx.one('SELECT status,expires_at FROM topic_bans WHERE topic=? AND subject=?', (target, subject)) + tx.execute( + "UPDATE topic_memberships SET role='member',status='removed' " + 'WHERE topic=? AND subject=?', + (target, subject), + write=True, + ) + row = tx.one( + 'SELECT status,expires_at FROM topic_bans WHERE topic=? AND subject=?', + (target, subject), + ) if row is None: - tx.execute('''INSERT INTO topic_bans + tx.execute( + """INSERT INTO topic_bans (topic,subject,actor,created_at,expires_at,reason,status) - VALUES (?,?,?,?,?,?,?)''', - (target, subject, ROOT_SUBJECT, at, expires, 'recovery-replay', 'active'), write=True) + VALUES (?,?,?,?,?,?,?)""", + (target, subject, ROOT_SUBJECT, at, expires, 'recovery-replay', 'active'), + write=True, + ) return True # Restrictions never shorten an existing active ban. A permanent ban # dominates finite expiries; only a separate explicit lift can remove it. @@ -217,46 +300,74 @@ async def _apply(tx, fact): expires = row[1] # V1 has no expiry and must not inherit a stale snapshot expiry. if row[0] != 'active' or row[1] != expires: - tx.execute("UPDATE topic_bans SET status='active',expires_at=? WHERE topic=? AND subject=?", - (expires, target, subject), write=True) + tx.execute( + "UPDATE topic_bans SET status='active',expires_at=? WHERE topic=? AND subject=?", + (expires, target, subject), + write=True, + ) return True return member_changed if kind == 'topic_ban.lift': row = tx.one('SELECT status FROM topic_bans WHERE topic=? AND subject=?', (target, subject)) require(row is not None, 'recovery_fact_missing') if row[0] != 'lifted': - tx.execute("UPDATE topic_bans SET status='lifted' WHERE topic=? AND subject=?", - (target, subject), write=True) + tx.execute( + "UPDATE topic_bans SET status='lifted' WHERE topic=? AND subject=?", + (target, subject), + write=True, + ) return True return False if kind in {'identity_key.retire', 'encryption_key.retire'}: table = 'identity_keys' if kind == 'identity_key.retire' else 'encryption_subkeys' - row = _owned(tx.one(f'SELECT subject,retired_at,is_primary FROM {table} WHERE key_id=?', - (target,)), subject) + row = _owned( + tx.one(f'SELECT subject,retired_at,is_primary FROM {table} WHERE key_id=?', (target,)), + subject, + ) earliest = _earlier(row[1], at) changed = row[1] != earliest or row[2] != 0 if changed: - tx.execute(f'UPDATE {table} SET retired_at=?,is_primary=0 WHERE key_id=?', - (earliest, target), write=True) + tx.execute( + f'UPDATE {table} SET retired_at=?,is_primary=0 WHERE key_id=?', + (earliest, target), + write=True, + ) if kind == 'identity_key.retire': changed = _revoke_credential(tx, subject, target, at) or changed - vault = tx.one('SELECT signing_key_id,signing_nonce,signing_ciphertext,age_ciphertext ' - 'FROM custodial_vault WHERE subject=?', (subject,)) - if vault is not None and vault[0] == target and (vault[1] is not None or vault[2] is not None): + vault = tx.one( + 'SELECT signing_key_id,signing_nonce,signing_ciphertext,age_ciphertext ' + 'FROM custodial_vault WHERE subject=?', + (subject,), + ) + if ( + vault is not None + and vault[0] == target + and (vault[1] is not None or vault[2] is not None) + ): status = 'decrypt_only' if vault[3] is not None else 'destroyed' - tx.execute('UPDATE custodial_vault SET signing_nonce=NULL,signing_ciphertext=NULL,' - 'status=? WHERE subject=?', (status, subject), write=True) + tx.execute( + 'UPDATE custodial_vault SET signing_nonce=NULL,signing_ciphertext=NULL,' + 'status=? WHERE subject=?', + (status, subject), + write=True, + ) changed = True return changed require(kind == 'vault.destroy', 'recovery_checkpoint_invalid') - row = tx.one('SELECT encryption_key_id,signing_nonce,signing_ciphertext,age_nonce,' - 'age_ciphertext,status FROM custodial_vault WHERE subject=?', (subject,)) + row = tx.one( + 'SELECT encryption_key_id,signing_nonce,signing_ciphertext,age_nonce,' + 'age_ciphertext,status FROM custodial_vault WHERE subject=?', + (subject,), + ) require(row is not None, 'recovery_fact_missing') require(row[0] == target, 'recovery_fact_subject_mismatch') if any(value is not None for value in row[1:5]) or row[5] != 'destroyed': - tx.execute("UPDATE custodial_vault SET signing_nonce=NULL,signing_ciphertext=NULL," - "age_nonce=NULL,age_ciphertext=NULL,status='destroyed',destroyed_at=? WHERE subject=?", - (at, subject), write=True) + tx.execute( + 'UPDATE custodial_vault SET signing_nonce=NULL,signing_ciphertext=NULL,' + "age_nonce=NULL,age_ciphertext=NULL,status='destroyed',destroyed_at=? WHERE subject=?", + (at, subject), + write=True, + ) return True return False @@ -264,6 +375,7 @@ async def _apply(tx, fact): async def replay(store, packet, *, pin, config_dir): """Physical-console entry; neither a TTY flag nor an HTTP caller can authorize it.""" from msg.admin.root import require_local_console + operator = require_local_console(config_dir) return await _replay(store, packet, pin=pin, operator=operator) @@ -283,17 +395,28 @@ async def _replay(store, packet, *, pin, operator): async with store.transaction(write=True) as tx: require(active(tx), 'recovery_quarantine_required') gate = tx.setting(SETTING) - require(isinstance(gate, dict) and gate.get('format') == 'msg-recovery-quarantine-v1' and - gate.get('outbound_enabled') is False and gate.get('authority') == 'health_only', - 'recovery_quarantine_invalid') - require(gate.get('source_backup_sha256') == body['source_backup_sha256'], - 'recovery_checkpoint_backup_mismatch') + require( + isinstance(gate, dict) + and gate.get('format') == 'msg-recovery-quarantine-v1' + and gate.get('outbound_enabled') is False + and gate.get('authority') == 'health_only', + 'recovery_quarantine_invalid', + ) + require( + gate.get('source_backup_sha256') == body['source_backup_sha256'], + 'recovery_checkpoint_backup_mismatch', + ) previous = tx.setting('recovery_replay') if previous is not None: - require(isinstance(previous, dict) and previous.get('format') in {FORMAT, body['format']} and - type(previous.get('sequence')) is int and 0 <= previous['sequence'] <= pin.sequence and - previous.get('prefix_digest') == digest(body['entries'][:previous['sequence']]), - 'recovery_checkpoint_regression') + require( + isinstance(previous, dict) + and previous.get('format') in {FORMAT, body['format']} + and type(previous.get('sequence')) is int + and 0 <= previous['sequence'] <= pin.sequence + and previous.get('prefix_digest') + == digest(body['entries'][: previous['sequence']]), + 'recovery_checkpoint_regression', + ) changed = 0 reconciled = set() authority_chains = {} @@ -306,70 +429,126 @@ async def _replay(store, packet, *, pin, operator): continue chain = authority_chains[fact['target']] actual_resource = await tx.resource(fact['target']) - actual = {field: getattr(actual_resource, field) for field in ('owner', 'group', 'parent')} - require(actual in [chain[0]['value']['previous']] + - [item['value']['current'] for item in chain], 'recovery_authority_state_mismatch') + actual = { + field: getattr(actual_resource, field) for field in ('owner', 'group', 'parent') + } + require( + actual + in [chain[0]['value']['previous']] + + [item['value']['current'] for item in chain], + 'recovery_authority_state_mismatch', + ) # Collapse only a verified contiguous chain; an old or intermediate # snapshot converges once without transiently restoring old scope. - fact = dict(chain[-1], subject=actual['owner'], - value={'previous': actual, 'current': chain[-1]['value']['current']}) + fact = dict( + chain[-1], + subject=actual['owner'], + value={'previous': actual, 'current': chain[-1]['value']['current']}, + ) reconciled.add(fact['target']) - if fact['kind'] in {'resource.acl.restrict', 'topic.policy.restrict'} and fact['target'] in authority_chains: + if ( + fact['kind'] in {'resource.acl.restrict', 'topic.policy.restrict'} + and fact['target'] in authority_chains + ): chain = authority_chains[fact['target']] - states = [chain[0]['value']['previous']] + [item['value']['current'] for item in chain] + states = [chain[0]['value']['previous']] + [ + item['value']['current'] for item in chain + ] actual_resource = await tx.resource(fact['target']) - actual = {field: getattr(actual_resource, field) for field in ('owner', 'group', 'parent')} + actual = { + field: getattr(actual_resource, field) for field in ('owner', 'group', 'parent') + } require(actual in states, 'recovery_authority_state_mismatch') bound = chain[0]['value']['previous'] for transition in chain: if transition['sequence'] < fact['sequence']: bound = transition['value']['current'] if fact['kind'] == 'resource.acl.restrict': - require(fact['subject'] == fact['value']['owner'] == bound['owner'] and - fact['value']['group'] == bound['group'], 'recovery_fact_subject_mismatch') - fact = dict(fact, subject=actual['owner'], - value=dict(fact['value'], owner=actual['owner'], group=actual['group'])) + require( + fact['subject'] == fact['value']['owner'] == bound['owner'] + and fact['value']['group'] == bound['group'], + 'recovery_fact_subject_mismatch', + ) + fact = dict( + fact, + subject=actual['owner'], + value=dict(fact['value'], owner=actual['owner'], group=actual['group']), + ) else: require(fact['subject'] == bound['owner'], 'recovery_fact_subject_mismatch') fact = dict(fact, subject=actual['owner']) changed += bool(await _apply(tx, fact)) - receipt = {'format': body['format'], 'checkpoint_digest': pin.digest, 'sequence': pin.sequence, - 'prefix_digest': digest(body['entries']), 'scope': 'supported_authority_reconciliation_facts', - 'promotion': 'blocked', - 'promotion_blocked_reasons': list(PROMOTION_BLOCKED_REASONS), - 'backup_retired': False} + receipt = { + 'format': body['format'], + 'checkpoint_digest': pin.digest, + 'sequence': pin.sequence, + 'prefix_digest': digest(body['entries']), + 'scope': 'supported_authority_reconciliation_facts', + 'promotion': 'blocked', + 'promotion_blocked_reasons': list(PROMOTION_BLOCKED_REASONS), + 'backup_retired': False, + } if body['format'] == POLICY_FORMAT: receipt['scope'] = 'supported_policy_reconciliation_only' receipt['coverage'] = body['coverage'] - staged = sorted({fact['target'] for fact in body['entries'] - if fact['kind'] == 'resource.authority.reconcile'}) + staged = sorted({ + fact['target'] + for fact in body['entries'] + if fact['kind'] == 'resource.authority.reconcile' + }) if staged: receipt['scope'] = 'supported_policy_reconciliation_only' receipt['authority_rebuild_required'] = staged - receipt['promotion_blocked_reasons'].append('derived_authority_inventory_incomplete') + receipt['promotion_blocked_reasons'].append( + 'derived_authority_inventory_incomplete' + ) if changed: tx.set_setting('authorization_epoch', tx.setting('authorization_epoch', 0) + 1) if changed or previous != receipt: tx.set_setting('recovery_replay', receipt) - event = Event(id='audit_' + uuid4().hex, type='recovery.revocation_replay', - time=datetime.now(UTC), request_id=pin.digest, actor=ROOT_SUBJECT, subject=ROOT_SUBJECT, - resources=(), data=dict(receipt, operator=operator, - checkpoint_key_id=key_id(pin.public_key))) - await tx.append_audit(AuditEvent(event=event, authority=(), before_digest=digest(previous), - after_digest=digest(receipt), previous_digest=None, entry_digest='', result='quarantined')) + event = Event( + id='audit_' + uuid4().hex, + type='recovery.revocation_replay', + time=datetime.now(UTC), + request_id=pin.digest, + actor=ROOT_SUBJECT, + subject=ROOT_SUBJECT, + resources=(), + data=dict(receipt, operator=operator, checkpoint_key_id=key_id(pin.public_key)), + ) + await tx.append_audit( + AuditEvent( + event=event, + authority=(), + before_digest=digest(previous), + after_digest=digest(receipt), + previous_digest=None, + entry_digest='', + result='quarantined', + ) + ) return dict(receipt, changed=changed) def selftest(): """Ephemeral signer test only; does not inspect or change any installation.""" from msg.security.crypto import Ed25519Signer + signer = Ed25519Signer.generate() - body = {'format': FORMAT, 'service': 'https://isolated.invalid', - 'source_backup_sha256': '0' * 64, 'sequence': 0, 'entries': []} - packet = {'checkpoint': body, 'signature': wire(signer.sign( - canonical(body), purpose='recovery-checkpoint-v1'))} - pin = TrustedCheckpointPin(service=body['service'], public_key=signer.public_key, - digest=digest(body), sequence=0) + body = { + 'format': FORMAT, + 'service': 'https://isolated.invalid', + 'source_backup_sha256': '0' * 64, + 'sequence': 0, + 'entries': [], + } + packet = { + 'checkpoint': body, + 'signature': wire(signer.sign(canonical(body), purpose='recovery-checkpoint-v1')), + } + pin = TrustedCheckpointPin( + service=body['service'], public_key=signer.public_key, digest=digest(body), sequence=0 + ) require(verify_checkpoint(packet, pin=pin) == body, 'recovery_selftest_failed') rejected = [] for invalid in (None, replace(pin, public_key=Ed25519Signer.generate().public_key)): @@ -379,15 +558,24 @@ def selftest(): rejected.append(exc.code) else: raise Failure('recovery_selftest_failed') - require(rejected == ['recovery_checkpoint_pin_required', 'recovery_checkpoint_invalid'], - 'recovery_selftest_failed') - return {'format': FORMAT, 'scope': 'isolated_signature_contract_only', - 'checks': 3, 'status': 'passed', 'database_used': False, - 'production_evidence': False, 'promotion': 'blocked'} + require( + rejected == ['recovery_checkpoint_pin_required', 'recovery_checkpoint_invalid'], + 'recovery_selftest_failed', + ) + return { + 'format': FORMAT, + 'scope': 'isolated_signature_contract_only', + 'checks': 3, + 'status': 'passed', + 'database_used': False, + 'production_evidence': False, + 'promotion': 'blocked', + } if __name__ == '__main__': import argparse + parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('--selftest', action='store_true', required=True) parser.parse_args() diff --git a/src/msg/admin/recovery_state.py b/src/msg/admin/recovery_state.py index 6c39552c..00f31017 100644 --- a/src/msg/admin/recovery_state.py +++ b/src/msg/admin/recovery_state.py @@ -1,7 +1,8 @@ """Complete, typed state commitments; no authority-bearing table is omitted.""" + +import hashlib from decimal import Decimal from importlib.resources import files -import hashlib from msg.core.codec import canonical, loads from msg.core.errors import require @@ -11,52 +12,106 @@ def catalogue(tx): - namespaces = tx.rows("SELECT nspname FROM pg_namespace WHERE nspname !~ '^pg_' AND nspname<>'information_schema' ORDER BY nspname") + namespaces = tx.rows( + "SELECT nspname FROM pg_namespace WHERE nspname !~ '^pg_' AND nspname<>'information_schema' ORDER BY nspname" + ) require(namespaces == [('public',)], 'recovery_schema_unknown_namespace') - relations = tx.rows("SELECT c.relname,c.relkind FROM pg_class c JOIN pg_namespace n ON n.oid=c.relnamespace WHERE n.nspname='public' ORDER BY c.relname") - require(all(kind in ('r', 'i', 'S') for _, kind in relations), 'recovery_schema_unknown_relation') - require(not tx.rows("SELECT 1 FROM pg_class c JOIN pg_namespace n ON n.oid=c.relnamespace WHERE n.nspname='public' AND (c.relrowsecurity OR c.relforcerowsecurity OR c.relpersistence<>'p')"), 'recovery_schema_unsupported') - require(not tx.rows("SELECT 1 FROM pg_policy p JOIN pg_class c ON c.oid=p.polrelid JOIN pg_namespace n ON n.oid=c.relnamespace WHERE n.nspname='public'"), 'recovery_schema_unsupported') - require(not tx.rows("SELECT 1 FROM pg_rewrite r JOIN pg_class c ON c.oid=r.ev_class JOIN pg_namespace n ON n.oid=c.relnamespace WHERE n.nspname='public'"), 'recovery_schema_unsupported') - require(not tx.rows("SELECT 1 FROM pg_collation c JOIN pg_namespace n ON n.oid=c.collnamespace WHERE n.nspname='public'"), 'recovery_schema_unsupported') - require(not tx.rows("SELECT 1 FROM pg_trigger t JOIN pg_class c ON c.oid=t.tgrelid JOIN pg_namespace n ON n.oid=c.relnamespace WHERE n.nspname='public' AND t.tgisinternal AND t.tgenabled<>'O'"), 'recovery_schema_unsupported') + relations = tx.rows( + "SELECT c.relname,c.relkind FROM pg_class c JOIN pg_namespace n ON n.oid=c.relnamespace WHERE n.nspname='public' ORDER BY c.relname" + ) + require( + all(kind in ('r', 'i', 'S') for _, kind in relations), 'recovery_schema_unknown_relation' + ) + require( + not tx.rows( + "SELECT 1 FROM pg_class c JOIN pg_namespace n ON n.oid=c.relnamespace WHERE n.nspname='public' AND (c.relrowsecurity OR c.relforcerowsecurity OR c.relpersistence<>'p')" + ), + 'recovery_schema_unsupported', + ) + require( + not tx.rows( + "SELECT 1 FROM pg_policy p JOIN pg_class c ON c.oid=p.polrelid JOIN pg_namespace n ON n.oid=c.relnamespace WHERE n.nspname='public'" + ), + 'recovery_schema_unsupported', + ) + require( + not tx.rows( + "SELECT 1 FROM pg_rewrite r JOIN pg_class c ON c.oid=r.ev_class JOIN pg_namespace n ON n.oid=c.relnamespace WHERE n.nspname='public'" + ), + 'recovery_schema_unsupported', + ) + require( + not tx.rows( + "SELECT 1 FROM pg_collation c JOIN pg_namespace n ON n.oid=c.collnamespace WHERE n.nspname='public'" + ), + 'recovery_schema_unsupported', + ) + require( + not tx.rows( + "SELECT 1 FROM pg_trigger t JOIN pg_class c ON c.oid=t.tgrelid JOIN pg_namespace n ON n.oid=c.relnamespace WHERE n.nspname='public' AND t.tgisinternal AND t.tgenabled<>'O'" + ), + 'recovery_schema_unsupported', + ) tables = [name for name, kind in relations if kind == 'r'] - result = {'tables': {}, 'constraints': [], 'indexes': [], 'triggers': [], 'functions': [], 'sequences': []} + result = { + 'tables': {}, + 'constraints': [], + 'indexes': [], + 'triggers': [], + 'functions': [], + 'sequences': [], + } for table in tables: - result['tables'][table] = tx.rows('''SELECT a.attname,format_type(a.atttypid,a.atttypmod), + result['tables'][table] = tx.rows( + """SELECT a.attname,format_type(a.atttypid,a.atttypmod), a.attnotnull,a.attidentity,pg_get_expr(d.adbin,d.adrelid), (SELECT n.nspname||'.'||co.collname FROM pg_collation co JOIN pg_namespace n ON n.oid=co.collnamespace WHERE co.oid=a.attcollation) FROM pg_attribute a JOIN pg_class c ON c.oid=a.attrelid JOIN pg_namespace n ON n.oid=c.relnamespace LEFT JOIN pg_attrdef d ON d.adrelid=c.oid AND d.adnum=a.attnum WHERE n.nspname='public' AND c.relname=? AND a.attnum>0 AND NOT a.attisdropped - ORDER BY a.attnum''', (table,)) - result['constraints'] = tx.rows('''SELECT c.relname,k.conname,k.contype,pg_get_constraintdef(k.oid),k.convalidated + ORDER BY a.attnum""", + (table,), + ) + result['constraints'] = ( + tx.rows("""SELECT c.relname,k.conname,k.contype,pg_get_constraintdef(k.oid),k.convalidated FROM pg_constraint k JOIN pg_class c ON c.oid=k.conrelid JOIN pg_namespace n ON n.oid=c.relnamespace - WHERE n.nspname='public' ORDER BY c.relname,k.conname''') - result['indexes'] = tx.rows("SELECT tablename,indexname,indexdef FROM pg_indexes WHERE schemaname='public' ORDER BY tablename,indexname") - result['triggers'] = tx.rows('''SELECT c.relname,t.tgname,t.tgenabled,pg_get_triggerdef(t.oid) + WHERE n.nspname='public' ORDER BY c.relname,k.conname""") + ) + result['indexes'] = tx.rows( + "SELECT tablename,indexname,indexdef FROM pg_indexes WHERE schemaname='public' ORDER BY tablename,indexname" + ) + result['triggers'] = tx.rows("""SELECT c.relname,t.tgname,t.tgenabled,pg_get_triggerdef(t.oid) FROM pg_trigger t JOIN pg_class c ON c.oid=t.tgrelid JOIN pg_namespace n ON n.oid=c.relnamespace - WHERE n.nspname='public' AND NOT t.tgisinternal ORDER BY c.relname,t.tgname''') - result['functions'] = tx.rows('''SELECT p.proname,pg_get_function_identity_arguments(p.oid), + WHERE n.nspname='public' AND NOT t.tgisinternal ORDER BY c.relname,t.tgname""") + result['functions'] = tx.rows("""SELECT p.proname,pg_get_function_identity_arguments(p.oid), pg_get_functiondef(p.oid) FROM pg_proc p JOIN pg_namespace n ON n.oid=p.pronamespace - WHERE n.nspname='public' ORDER BY p.proname,pg_get_function_identity_arguments(p.oid)''') - result['sequences'] = tx.rows("SELECT sequencename,data_type,start_value,min_value,max_value,increment_by,cycle FROM pg_sequences WHERE schemaname='public' ORDER BY sequencename") - result['database_collation'] = tx.one("SELECT encoding,datcollate,datctype FROM pg_database WHERE datname=current_database()") + WHERE n.nspname='public' ORDER BY p.proname,pg_get_function_identity_arguments(p.oid)""") + result['sequences'] = tx.rows( + "SELECT sequencename,data_type,start_value,min_value,max_value,increment_by,cycle FROM pg_sequences WHERE schemaname='public' ORDER BY sequencename" + ) + result['database_collation'] = tx.one( + 'SELECT encoding,datcollate,datctype FROM pg_database WHERE datname=current_database()' + ) return loads(canonical(result)) def shape(catalog): """Stable allowed object/column vocabulary, independent of PG deparser whitespace.""" - return {'tables': {table: [column[:4] for column in columns] for table, columns in catalog['tables'].items()}, - # PG18 additionally catalogs NOT NULL as contype=n. attnotnull above - # already pins each column; keep fixed vocabulary portable to PG16. - # Full signed catalogue still includes every constraint definition. - 'constraints': [row[:3] for row in catalog['constraints'] if row[2] != 'n'], - 'indexes': [row[:2] for row in catalog['indexes']], - 'triggers': [row[:2] for row in catalog['triggers']], - 'functions': [row[:2] for row in catalog['functions']], - 'sequences': [row[:2] for row in catalog['sequences']]} + return { + 'tables': { + table: [column[:4] for column in columns] + for table, columns in catalog['tables'].items() + }, + # PG18 additionally catalogs NOT NULL as contype=n. attnotnull above + # already pins each column; keep fixed vocabulary portable to PG16. + # Full signed catalogue still includes every constraint definition. + 'constraints': [row[:3] for row in catalog['constraints'] if row[2] != 'n'], + 'indexes': [row[:2] for row in catalog['indexes']], + 'triggers': [row[:2] for row in catalog['triggers']], + 'functions': [row[:2] for row in catalog['functions']], + 'sequences': [row[:2] for row in catalog['sequences']], + } def known_catalogue(tx): @@ -86,17 +141,30 @@ def runtime_controls(tx): current = tx.setting('runtime_config') require(current is None or type(current) is dict, 'recovery_runtime_invalid') values = current or {} - require(all(key not in values or type(values[key]) is bool for key in PAUSE_FIELDS), 'recovery_runtime_invalid') - return {'present': current is not None, - 'fields': {key: {'present': key in values, 'value': values.get(key, False)} for key in PAUSE_FIELDS}} + require( + all(key not in values or type(values[key]) is bool for key in PAUSE_FIELDS), + 'recovery_runtime_invalid', + ) + return { + 'present': current is not None, + 'fields': { + key: {'present': key in values, 'value': values.get(key, False)} for key in PAUSE_FIELDS + }, + } def _runtime_row(row, controls): value = loads(row[1]) - require(type(value) is dict and all(value.get(key) is False for key in PAUSE_FIELDS), 'recovery_runtime_not_paused') + require( + type(value) is dict and all(value.get(key) is False for key in PAUSE_FIELDS), + 'recovery_runtime_not_paused', + ) for key in PAUSE_FIELDS: wanted = controls['fields'][key] - require(type(wanted['present']) is bool and type(wanted['value']) is bool, 'recovery_runtime_invalid') + require( + type(wanted['present']) is bool and type(wanted['value']) is bool, + 'recovery_runtime_invalid', + ) value.pop(key) if wanted['present']: value[key] = wanted['value'] @@ -108,22 +176,36 @@ def _runtime_row(row, controls): def snapshot(tx, *, controls=None, resume=None): if resume is not None: - require(tx.setting('recovery_runtime_generation') == resume['generation'], - 'recovery_promotion_receipt_mismatch') + require( + tx.setting('recovery_runtime_generation') == resume['generation'], + 'recovery_promotion_receipt_mismatch', + ) catalog = known_catalogue(tx) result = {'catalogue': catalog, 'tables': {}, 'sequences': {}} # The application writer lock plus these relation locks serialize every DB # authority/effect change until capture or promotion commits. - tx.execute('LOCK TABLE ' + ','.join('public.' + quote(name) for name in catalog['tables']) + ' IN ACCESS EXCLUSIVE MODE', write=True) + tx.execute( + 'LOCK TABLE ' + + ','.join('public.' + quote(name) for name in catalog['tables']) + + ' IN ACCESS EXCLUSIVE MODE', + write=True, + ) for table, columns in catalog['tables'].items(): - rows = tx.rows('SELECT ' + ','.join(quote(column[0]) for column in columns) + ' FROM public.' + quote(table)) + rows = tx.rows( + 'SELECT ' + + ','.join(quote(column[0]) for column in columns) + + ' FROM public.' + + quote(table) + ) if table == 'settings': kept = [] for row in rows: if row[0] in CONTROL_SETTINGS: continue if resume is not None and row[0] == 'recovery_runtime_generation': - require(loads(row[1]) == resume['generation'], 'recovery_promotion_receipt_mismatch') + require( + loads(row[1]) == resume['generation'], 'recovery_promotion_receipt_mismatch' + ) if resume['prior_generation'] is None: continue row = (row[0], canonical(resume['prior_generation']).decode()) @@ -134,7 +216,10 @@ def snapshot(tx, *, controls=None, resume=None): rows = kept if resume is not None and table == 'audit': audit = resume['audit_row'] - require(sum(loads(canonical(row)) == audit for row in rows) == 1, 'recovery_promotion_receipt_mismatch') + require( + sum(loads(canonical(row)) == audit for row in rows) == 1, + 'recovery_promotion_receipt_mismatch', + ) require(max(row[0] for row in rows) == audit[0], 'recovery_promotion_state_changed') rows = [row for row in rows if row[0] != audit[0]] encoded = sorted(canonical([cell(value) for value in row]) for row in rows) @@ -146,7 +231,10 @@ def snapshot(tx, *, controls=None, resume=None): for name, *_ in catalog['sequences']: state = loads(canonical(tx.one('SELECT last_value,is_called FROM public.' + quote(name)))) if resume is not None and name == 'audit_seq_seq': - require(state in (resume['prior_audit_sequence'], [resume['audit_row'][0], True]), 'recovery_promotion_state_changed') + require( + state in (resume['prior_audit_sequence'], [resume['audit_row'][0], True]), + 'recovery_promotion_state_changed', + ) state = resume['prior_audit_sequence'] result['sequences'][name] = state return result diff --git a/src/msg/admin/restore_database.py b/src/msg/admin/restore_database.py index 7b7886c8..b7611ea2 100644 --- a/src/msg/admin/restore_database.py +++ b/src/msg/admin/restore_database.py @@ -1,7 +1,8 @@ """Import an offline dump and its quarantine gate in the same PostgreSQL commit.""" -from pathlib import Path + import os import subprocess +from pathlib import Path from msg.core.codec import canonical from msg.security.quarantine import SETTING @@ -17,21 +18,48 @@ def restore_dump(dump, *, safe_dsn, env, quarantine): script = Path(dump).parent / 'quarantined-restore.sql' script.touch(mode=0o600, exist_ok=False) try: - subprocess.run(['pg_restore', '--exit-on-error', '--no-owner', '--no-acl', - '--file', str(script), str(dump)], - env=env, check=True, capture_output=True) + subprocess.run( + [ + 'pg_restore', + '--exit-on-error', + '--no-owner', + '--no-acl', + '--file', + str(script), + str(dump), + ], + env=env, + check=True, + capture_output=True, + ) # Values are fixed recovery metadata, not secrets. Escape as a SQL # literal (including backslashes) independently of session string flags. value = canonical(quarantine).decode().replace('\\', '\\\\').replace("'", "''") - gate = ("\nSET search_path = public, pg_catalog;\n" - f"INSERT INTO settings(key,value) VALUES('{SETTING}',E'{value}') " - "ON CONFLICT(key) DO UPDATE SET value=EXCLUDED.value;\n") + gate = ( + '\nSET search_path = public, pg_catalog;\n' + f"INSERT INTO settings(key,value) VALUES('{SETTING}',E'{value}') " + 'ON CONFLICT(key) DO UPDATE SET value=EXCLUDED.value;\n' + ) with script.open('ab') as stream: stream.write(gate.encode()) stream.flush() os.fsync(stream.fileno()) - subprocess.run(['psql', '--no-psqlrc', '--quiet', '--single-transaction', - '--set', 'ON_ERROR_STOP=1', '--dbname', safe_dsn, '--file', str(script)], - env=env, check=True, capture_output=True) + subprocess.run( + [ + 'psql', + '--no-psqlrc', + '--quiet', + '--single-transaction', + '--set', + 'ON_ERROR_STOP=1', + '--dbname', + safe_dsn, + '--file', + str(script), + ], + env=env, + check=True, + capture_output=True, + ) finally: script.unlink(missing_ok=True) diff --git a/src/msg/admin/root.py b/src/msg/admin/root.py index 1945f7f4..6788269b 100644 --- a/src/msg/admin/root.py +++ b/src/msg/admin/root.py @@ -4,141 +4,278 @@ entrypoint must pass OS-console checks before calling them. Tests call these use cases only against disposable directories and never enable a network root route. """ + from __future__ import annotations import asyncio -from dataclasses import replace -from datetime import timedelta import getpass import os -from pathlib import Path import re import sys +from dataclasses import replace +from datetime import timedelta +from pathlib import Path -from msg.constants import * -from msg.core.codec import b64,unb64,canonical,decode,digest,loads,wire -from msg.core.errors import Failure,require -from msg.core.models import Certificate,CertificateRequest,Credential,IssuancePolicy,Signature,ResourceRef,Event,AuditEvent -from msg.security.crypto import Ed25519Signer,seal_private_key,open_private_key -from msg.security.certificates import sign_certificate,csr_body,verify_csr -from msg.security.capabilities import grant_for -from msg.storage.git import durable_write -from msg.security.trust_files import trust_file, write_trust, reserved_plugins_directory -from msg.bootstrap import bootstrap,seed_resource +from msg.bootstrap import bootstrap, seed_resource +from msg.constants import ( + ADMINS_GROUP as ADMINS_GROUP, + CA_SPACE as CA_SPACE, + CERT_SPACE as CERT_SPACE, + CSR_SPACE as CSR_SPACE, + ONLINE_CA as ONLINE_CA, + PROTOCOL_VERSION as PROTOCOL_VERSION, + PUBLIC_GROUP as PUBLIC_GROUP, + ROOT_SPACE as ROOT_SPACE, + ROOT_SUBJECT as ROOT_SUBJECT, + TOOLS_SPACE as TOOLS_SPACE, +) +from msg.core.codec import b64, canonical, digest, loads, unb64, wire +from msg.core.errors import require +from msg.core.models import ( + AuditEvent, + Certificate, + CertificateRequest, + Credential, + Event, + IssuancePolicy, + ResourceRef, + Signature, +) from msg.plugins.common import new_id from msg.plugins.identity import certificate_resource +from msg.security.capabilities import grant_for +from msg.security.certificates import csr_body, sign_certificate, verify_csr +from msg.security.crypto import Ed25519Signer, open_private_key, seal_private_key +from msg.security.trust_files import reserved_plugins_directory, trust_file, write_trust +from msg.storage.git import durable_write def root_envelope(config_dir): from msg.config import root_private_dir - current=root_private_dir(Path(config_dir))/'key.json' - legacy=Path(config_dir)/'root'/'key.json' + + current = root_private_dir(Path(config_dir)) / 'key.json' + legacy = Path(config_dir) / 'root' / 'key.json' return legacy if legacy.is_file() and not current.exists() else current -async def _provision(app,pin): +async def _provision(app, pin): """Initialize only an empty installation; partial state requires explicit recovery.""" - settings=app.settings - protected=settings.root_private_dir - marker=settings.config_dir/'initialization.pending' - require(not root_envelope(settings.config_dir).exists() and not settings.trust_file.exists() and - not marker.exists(),'initialization_requires_recovery') + settings = app.settings + protected = settings.root_private_dir + marker = settings.config_dir / 'initialization.pending' + require( + not root_envelope(settings.config_dir).exists() + and not settings.trust_file.exists() + and not marker.exists(), + 'initialization_requires_recovery', + ) await app.open_storage() async with app.metadata.transaction(write=False) as tx: - require(tx.one('SELECT COUNT(*) FROM resources')[0] == 0,'initialization_requires_recovery') - root=Ed25519Signer.generate() - envelope=seal_private_key(root.private_bytes(),pin) - settings.config_dir.mkdir(parents=True,exist_ok=True) - reserved_plugins_directory(settings.config_dir,create=True) - durable_write(marker,b'1\n',mode=0o600) - protected.mkdir(mode=0o700,parents=True,exist_ok=True) - os.chmod(protected,0o700) - settings.service_keys.mkdir(mode=0o700,parents=True,exist_ok=True) - os.chmod(settings.service_keys,0o700) - settings.trust_file.parent.mkdir(mode=0o755,parents=True,exist_ok=True) + require( + tx.one('SELECT COUNT(*) FROM resources')[0] == 0, 'initialization_requires_recovery' + ) + root = Ed25519Signer.generate() + envelope = seal_private_key(root.private_bytes(), pin) + settings.config_dir.mkdir(parents=True, exist_ok=True) + reserved_plugins_directory(settings.config_dir, create=True) + durable_write(marker, b'1\n', mode=0o600) + protected.mkdir(mode=0o700, parents=True, exist_ok=True) + os.chmod(protected, 0o700) + settings.service_keys.mkdir(mode=0o700, parents=True, exist_ok=True) + os.chmod(settings.service_keys, 0o700) + settings.trust_file.parent.mkdir(mode=0o755, parents=True, exist_ok=True) # Wrapped before touching the installation; a rejected PIN leaves no partial state. - durable_write(root_envelope(settings.config_dir),canonical(envelope),mode=0o600) - online=Ed25519Signer.generate() - receipt=Ed25519Signer.generate() - durable_write(settings.service_keys/'online.key',online.private_bytes(),mode=0o600) - durable_write(settings.service_keys/'receipt.key',receipt.private_bytes(),mode=0o600) - durable_write(settings.service_keys/'tokens.key',os.urandom(32),mode=0o600) - now=app.clock() - await bootstrap(app.metadata,app.contents,app.registry,now, - selftest_run_id=app.selftest_run_id) - root_grants=app.primary_ceiling() - root_certificate=Certificate(resource_id='cert_root',serial=new_id('serial'),subject_id=ROOT_SUBJECT, - key_id=root.key_id,issuer_id=ROOT_SUBJECT,parent_certificate_id=None,authority_sources=(),kind='ca', - grants=root_grants,not_before=now,expires_at=now+timedelta(days=3650),target_service=settings.service_url, - delegation_depth=8,issuance=IssuancePolicy(issue_grants=root_grants,max_cert_ttl_seconds=31536000, - max_child_ca_depth=3,max_delegation_depth=8),signature=Signature(key_id=root.key_id,algorithm='ed25519',value=b'')) - root_certificate=sign_certificate(root_certificate,root) - online_use=grant_for(app.registry.capability('cert.issue'),scope=app.default_scope()) - csr=CertificateRequest(resource_id=new_id('csr'),applicant=ONLINE_CA,subject_id=ONLINE_CA, - requested_issuer=ROOT_SUBJECT,public_key=online.public_key,kind='ca',grants=(online_use,), - issuance=IssuancePolicy(issue_grants=app.base_grants(),max_cert_ttl_seconds=settings.base_certificate_ttl, - max_child_ca_depth=0,max_delegation_depth=8),requested_ttl_seconds=31536000, - target_service=settings.service_url,delegation_depth=0,authority_sources=(),request_digest='', - possession_proof=Signature(key_id=online.key_id,algorithm='ed25519',value=b'')) - csr=replace(csr,request_digest=digest(csr_body(csr)), - possession_proof=online.sign(canonical(csr_body(csr)),purpose='csr')) + durable_write(root_envelope(settings.config_dir), canonical(envelope), mode=0o600) + online = Ed25519Signer.generate() + receipt = Ed25519Signer.generate() + durable_write(settings.service_keys / 'online.key', online.private_bytes(), mode=0o600) + durable_write(settings.service_keys / 'receipt.key', receipt.private_bytes(), mode=0o600) + durable_write(settings.service_keys / 'tokens.key', os.urandom(32), mode=0o600) + now = app.clock() + await bootstrap( + app.metadata, app.contents, app.registry, now, selftest_run_id=app.selftest_run_id + ) + root_grants = app.primary_ceiling() + root_certificate = Certificate( + resource_id='cert_root', + serial=new_id('serial'), + subject_id=ROOT_SUBJECT, + key_id=root.key_id, + issuer_id=ROOT_SUBJECT, + parent_certificate_id=None, + authority_sources=(), + kind='ca', + grants=root_grants, + not_before=now, + expires_at=now + timedelta(days=3650), + target_service=settings.service_url, + delegation_depth=8, + issuance=IssuancePolicy( + issue_grants=root_grants, + max_cert_ttl_seconds=31536000, + max_child_ca_depth=3, + max_delegation_depth=8, + ), + signature=Signature(key_id=root.key_id, algorithm='ed25519', value=b''), + ) + root_certificate = sign_certificate(root_certificate, root) + online_use = grant_for(app.registry.capability('cert.issue'), scope=app.default_scope()) + csr = CertificateRequest( + resource_id=new_id('csr'), + applicant=ONLINE_CA, + subject_id=ONLINE_CA, + requested_issuer=ROOT_SUBJECT, + public_key=online.public_key, + kind='ca', + grants=(online_use,), + issuance=IssuancePolicy( + issue_grants=app.base_grants(), + max_cert_ttl_seconds=settings.base_certificate_ttl, + max_child_ca_depth=0, + max_delegation_depth=8, + ), + requested_ttl_seconds=31536000, + target_service=settings.service_url, + delegation_depth=0, + authority_sources=(), + request_digest='', + possession_proof=Signature(key_id=online.key_id, algorithm='ed25519', value=b''), + ) + csr = replace( + csr, + request_digest=digest(csr_body(csr)), + possession_proof=online.sign(canonical(csr_body(csr)), purpose='csr'), + ) async with app.metadata.transaction(write=True) as tx: - for uid,signer,ceiling in ((ROOT_SUBJECT,root,()),(ONLINE_CA,online,(online_use,))): - await tx.save_credential(Credential(id=signer.key_id,subject_id=uid,kind='signing_key',verifier=signer.public_key, - ceiling=ceiling,not_before=now,expires_at=None,revoked_at=None),0) - for name in ('keys','certificates'): - await seed_resource(tx,app.contents,dict(id='r_'+digest((uid,name))[7:39],type='topic',name=name,parent=uid, - owner=uid,group=PUBLIC_GROUP,mode='0555'),now) - await certificate_resource(tx,root_certificate,now) - await tx.register_certificate(root_certificate,None,0) - await seed_resource(tx,app.contents,dict(id=csr.resource_id,type='csr',name=csr.resource_id,parent=CSR_SPACE, - owner=ONLINE_CA,group=ADMINS_GROUP,mode='0600'),now) + for uid, signer, ceiling in ((ROOT_SUBJECT, root, ()), (ONLINE_CA, online, (online_use,))): + await tx.save_credential( + Credential( + id=signer.key_id, + subject_id=uid, + kind='signing_key', + verifier=signer.public_key, + ceiling=ceiling, + not_before=now, + expires_at=None, + revoked_at=None, + ), + 0, + ) + for name in ('keys', 'certificates'): + await seed_resource( + tx, + app.contents, + { + 'id': 'r_' + digest((uid, name))[7:39], + 'type': 'topic', + 'name': name, + 'parent': uid, + 'owner': uid, + 'group': PUBLIC_GROUP, + 'mode': '0555', + }, + now, + ) + await certificate_resource(tx, root_certificate, now) + await tx.register_certificate(root_certificate, None, 0) + await seed_resource( + tx, + app.contents, + { + 'id': csr.resource_id, + 'type': 'csr', + 'name': csr.resource_id, + 'parent': CSR_SPACE, + 'owner': ONLINE_CA, + 'group': ADMINS_GROUP, + 'mode': '0600', + }, + now, + ) await tx.save_csr(csr) - tx.set_setting('active_root_certificate',root_certificate.resource_id) - tx.set_setting('online_ca_request',csr.resource_id) - tx.set_setting('receipt_public_key',{'key_id':receipt.key_id,'public_key':b64(receipt.public_key)}) - write_trust(settings.config_dir,{'version':1,'public_key':b64(root.public_key), - 'certificate':wire(root_certificate)},writer=durable_write) + tx.set_setting('active_root_certificate', root_certificate.resource_id) + tx.set_setting('online_ca_request', csr.resource_id) + tx.set_setting( + 'receipt_public_key', {'key_id': receipt.key_id, 'public_key': b64(receipt.public_key)} + ) + write_trust( + settings.config_dir, + {'version': 1, 'public_key': b64(root.public_key), 'certificate': wire(root_certificate)}, + writer=durable_write, + ) marker.unlink() await app.load() - return csr.resource_id,root + return csr.resource_id, root -async def _approve_csr(app,csr_id,signer, *, expected_digest,operator): - now=app.clock() +async def _approve_csr(app, csr_id, signer, *, expected_digest, operator): + now = app.clock() async with app.metadata.transaction(write=True) as tx: - csr=await tx.csr(csr_id) - state=await tx.csr_state(csr_id) - require(csr.requested_issuer==ROOT_SUBJECT,'not_root_request') + csr = await tx.csr(csr_id) + state = await tx.csr_state(csr_id) + require(csr.requested_issuer == ROOT_SUBJECT, 'not_root_request') if expected_digest is not None: - require(csr.request_digest==expected_digest,'csr_changed_after_confirmation') + require(csr.request_digest == expected_digest, 'csr_changed_after_confirmation') verify_csr(csr) - if state.status=='issued': + if state.status == 'issued': return await tx.certificate(state.certificate_id) - require(state.status=='pending','csr_not_pending') - require(signer.key_id==app.certificates.root_certificate.key_id,'root_key_mismatch') - parent=app.certificates.root_certificate - certificate=Certificate(resource_id=new_id('cert'),serial=new_id('serial'),subject_id=csr.subject_id, - key_id=csr.possession_proof.key_id,issuer_id=ROOT_SUBJECT,parent_certificate_id=parent.resource_id, - authority_sources=csr.authority_sources,kind=csr.kind,grants=csr.grants,not_before=now, - expires_at=min(now+timedelta(seconds=csr.requested_ttl_seconds),parent.expires_at), - target_service=csr.target_service,delegation_depth=csr.delegation_depth,issuance=csr.issuance, - signature=Signature(key_id=signer.key_id,algorithm='ed25519',value=b'')) - certificate=sign_certificate(certificate,signer) - await certificate_resource(tx,certificate,now) - await app.certificates.validate_publication(certificate,csr,tx) - await tx.register_certificate(certificate,csr_id,state.generation) - credential=await tx.credential(certificate.key_id) + require(state.status == 'pending', 'csr_not_pending') + require(signer.key_id == app.certificates.root_certificate.key_id, 'root_key_mismatch') + parent = app.certificates.root_certificate + certificate = Certificate( + resource_id=new_id('cert'), + serial=new_id('serial'), + subject_id=csr.subject_id, + key_id=csr.possession_proof.key_id, + issuer_id=ROOT_SUBJECT, + parent_certificate_id=parent.resource_id, + authority_sources=csr.authority_sources, + kind=csr.kind, + grants=csr.grants, + not_before=now, + expires_at=min(now + timedelta(seconds=csr.requested_ttl_seconds), parent.expires_at), + target_service=csr.target_service, + delegation_depth=csr.delegation_depth, + issuance=csr.issuance, + signature=Signature(key_id=signer.key_id, algorithm='ed25519', value=b''), + ) + certificate = sign_certificate(certificate, signer) + await certificate_resource(tx, certificate, now) + await app.certificates.validate_publication(certificate, csr, tx) + await tx.register_certificate(certificate, csr_id, state.generation) + credential = await tx.credential(certificate.key_id) if not credential.ceiling: - subject=await tx.subject(credential.subject_id) - await tx.save_credential(replace(credential,ceiling=certificate.grants),subject.auth_version) - if csr_id==tx.setting('online_ca_request'): - tx.set_setting('online_ca_certificate',certificate.resource_id) - event=Event(id=new_id('audit'),type='root.cert.issue',time=now,request_id=csr_id, - actor=ROOT_SUBJECT,subject=ROOT_SUBJECT,resources=(ResourceRef(id=certificate.resource_id),), - data={'operator':operator,'csr_digest':csr.request_digest,'grants':wire(certificate.grants)}) - await tx.append_audit(AuditEvent(event=event,authority=(ResourceRef(id=parent.resource_id),), - before_digest=csr.request_digest,after_digest=digest(certificate),previous_digest=None,entry_digest='',result='issued')) + subject = await tx.subject(credential.subject_id) + await tx.save_credential( + replace(credential, ceiling=certificate.grants), subject.auth_version + ) + if csr_id == tx.setting('online_ca_request'): + tx.set_setting('online_ca_certificate', certificate.resource_id) + event = Event( + id=new_id('audit'), + type='root.cert.issue', + time=now, + request_id=csr_id, + actor=ROOT_SUBJECT, + subject=ROOT_SUBJECT, + resources=(ResourceRef(id=certificate.resource_id),), + data={ + 'operator': operator, + 'csr_digest': csr.request_digest, + 'grants': wire(certificate.grants), + }, + ) + await tx.append_audit( + AuditEvent( + event=event, + authority=(ResourceRef(id=parent.resource_id),), + before_digest=csr.request_digest, + after_digest=digest(certificate), + previous_digest=None, + entry_digest='', + result='issued', + ) + ) return certificate @@ -150,249 +287,449 @@ def require_local_console(config_dir): Deployments that need another administrative channel must use a separately audited OS policy, not relax this check through the network configuration. """ - require(hasattr(os,'geteuid') and os.geteuid()==0,'local_os_administrator_required') - require(not any(name in os.environ for name in ('SSH_CONNECTION','SSH_CLIENT','SSH_TTY')),'remote_admin_forbidden') - require(sys.stdin.isatty() and sys.stdout.isatty(),'local_console_required') - tty=os.path.realpath(os.ttyname(sys.stdin.fileno())) - require(re.fullmatch(r'/dev/(console|tty[0-9]+|ttyS[0-9]+|hvc[0-9]+)',tty) is not None, - 'local_console_required') - path=Path(config_dir).resolve() - require(path.exists() and path.stat().st_uid==0 and not path.stat().st_mode&0o022,'unsafe_config_owner') + require(hasattr(os, 'geteuid') and os.geteuid() == 0, 'local_os_administrator_required') + require( + not any(name in os.environ for name in ('SSH_CONNECTION', 'SSH_CLIENT', 'SSH_TTY')), + 'remote_admin_forbidden', + ) + require(sys.stdin.isatty() and sys.stdout.isatty(), 'local_console_required') + tty = os.path.realpath(os.ttyname(sys.stdin.fileno())) + require( + re.fullmatch(r'/dev/(console|tty[0-9]+|ttyS[0-9]+|hvc[0-9]+)', tty) is not None, + 'local_console_required', + ) + path = Path(config_dir).resolve() + require( + path.exists() and path.stat().st_uid == 0 and not path.stat().st_mode & 0o022, + 'unsafe_config_owner', + ) return f'uid:{os.geteuid()}:{tty}' class RootAdmin: - def __init__(self,config_dir=Path('/etc/msgd')): - self.config_dir=Path(config_dir) + def __init__(self, config_dir=Path('/etc/msgd')): + self.config_dir = Path(config_dir) def _app(self): from msg.application import Application from msg.config import load_settings + return Application(load_settings(self.config_dir)) def initialize(self): require_local_console(self.config_dir) - app=self._app() + app = self._app() if root_envelope(self.config_dir).exists() and app.settings.trust_file.exists(): - status=self.doctor() - require(all(status['checks'].get(k,{}).get('ok') for k in ('storage','root_trust','bootstrap','root_private_boundary')), - 'initialization_requires_recovery') - private=open_private_key(loads(root_envelope(self.config_dir).read_bytes()),getpass.getpass('Current root PIN/passphrase: ')) - trust=loads(app.settings.trust_file.read_bytes()) - require(Ed25519Signer.from_bytes(private).public_key==unb64(trust['public_key']),'root_key_mismatch') + status = self.doctor() + require( + all( + status['checks'].get(k, {}).get('ok') + for k in ('storage', 'root_trust', 'bootstrap', 'root_private_boundary') + ), + 'initialization_requires_recovery', + ) + private = open_private_key( + loads(root_envelope(self.config_dir).read_bytes()), + getpass.getpass('Current root PIN/passphrase: '), + ) + trust = loads(app.settings.trust_file.read_bytes()) + require( + Ed25519Signer.from_bytes(private).public_key == unb64(trust['public_key']), + 'root_key_mismatch', + ) return status['root_id'] - first=getpass.getpass('New root PIN/passphrase: ') - second=getpass.getpass('Confirm root PIN/passphrase: ') - require(first==second,'pin_confirmation_mismatch') - csr,_=asyncio.run(_provision(app,first)) - print(canonical({'status':'initialized','root_id':ROOT_SUBJECT,'online_ca_request':csr}).decode()) + first = getpass.getpass('New root PIN/passphrase: ') + second = getpass.getpass('Confirm root PIN/passphrase: ') + require(first == second, 'pin_confirmation_mismatch') + csr, _ = asyncio.run(_provision(app, first)) + print( + canonical({ + 'status': 'initialized', + 'root_id': ROOT_SUBJECT, + 'online_ca_request': csr, + }).decode() + ) return ROOT_SUBJECT - def issue(self,csr_id): - operator=require_local_console(self.config_dir) - app=self._app() + def issue(self, csr_id): + operator = require_local_console(self.config_dir) + app = self._app() + async def read(): await app.load() async with app.metadata.transaction(write=False) as tx: - return await tx.csr(csr_id),await tx.csr_state(csr_id) - csr,state=asyncio.run(read()) + return await tx.csr(csr_id), await tx.csr_state(csr_id) + + csr, state = asyncio.run(read()) # json.dumps escapes terminal controls; no raw user-controlled text is printed. - print(canonical({'request':wire(csr),'state':wire(state),'public_key_fingerprint':digest(csr.public_key)}).decode()) - if state.status=='issued': + print( + canonical({ + 'request': wire(csr), + 'state': wire(state), + 'public_key_fingerprint': digest(csr.public_key), + }).decode() + ) + if state.status == 'issued': + async def previous(): async with app.metadata.transaction(write=False) as tx: return await tx.certificate(state.certificate_id) + return asyncio.run(previous()) - require(input('Type the exact CSR digest to approve: ').strip()==csr.request_digest,'approval_cancelled') - pin=getpass.getpass('Root PIN/passphrase: ') - private=open_private_key(loads(root_envelope(self.config_dir).read_bytes()),pin) - signer=Ed25519Signer.from_bytes(private) - return asyncio.run(_approve_csr(app,csr_id,signer,expected_digest=csr.request_digest,operator=operator)) + require( + input('Type the exact CSR digest to approve: ').strip() == csr.request_digest, + 'approval_cancelled', + ) + pin = getpass.getpass('Root PIN/passphrase: ') + private = open_private_key(loads(root_envelope(self.config_dir).read_bytes()), pin) + signer = Ed25519Signer.from_bytes(private) + return asyncio.run( + _approve_csr(app, csr_id, signer, expected_digest=csr.request_digest, operator=operator) + ) def change_pin(self): require_local_console(self.config_dir) - require(input('Type CHANGE PIN to continue: ')=='CHANGE PIN','approval_cancelled') - path=root_envelope(self.config_dir) - private=open_private_key(loads(path.read_bytes()),getpass.getpass('Current root PIN/passphrase: ')) - first=getpass.getpass('New root PIN/passphrase: ') - require(first==getpass.getpass('Confirm new root PIN/passphrase: '),'pin_confirmation_mismatch') - durable_write(path,canonical(seal_private_key(private,first)),mode=0o600) + require(input('Type CHANGE PIN to continue: ') == 'CHANGE PIN', 'approval_cancelled') + path = root_envelope(self.config_dir) + private = open_private_key( + loads(path.read_bytes()), getpass.getpass('Current root PIN/passphrase: ') + ) + first = getpass.getpass('New root PIN/passphrase: ') + require( + first == getpass.getpass('Confirm new root PIN/passphrase: '), + 'pin_confirmation_mismatch', + ) + durable_write(path, canonical(seal_private_key(private, first)), mode=0o600) def rotate(self, *, lost_key=False, resume=False): - operator=require_local_console(self.config_dir) - app=self._app() - from msg.admin.rotation import prepare,complete,journal_path + operator = require_local_console(self.config_dir) + app = self._app() + from msg.admin.rotation import complete, journal_path, prepare + if resume: - require(journal_path(app).is_file(),'rotation_journal_missing') - journal=loads(journal_path(app).read_bytes()) + require(journal_path(app).is_file(), 'rotation_journal_missing') + journal = loads(journal_path(app).read_bytes()) print(canonical(journal['statement']).decode()) - require(input('Type the new root fingerprint to resume: ')==journal['statement']['new_fingerprint'],'approval_cancelled') - pin=getpass.getpass('New root PIN/passphrase: ') + require( + input('Type the new root fingerprint to resume: ') + == journal['statement']['new_fingerprint'], + 'approval_cancelled', + ) + pin = getpass.getpass('New root PIN/passphrase: ') else: asyncio.run(app.load()) - print(canonical({'old_fingerprint':digest(app.certificates.root_public_key), - 'effect':'all existing certificate chains require reissuance','lost_key':lost_key}).decode()) - require(input('Type ROTATE ROOT '+digest(app.certificates.root_public_key)+': ')== - 'ROTATE ROOT '+digest(app.certificates.root_public_key),'approval_cancelled') - old=None if lost_key else Ed25519Signer.from_bytes(open_private_key( - loads(root_envelope(self.config_dir).read_bytes()),getpass.getpass('Current root PIN/passphrase: '))) - pin=getpass.getpass('New root PIN/passphrase: ') - require(pin==getpass.getpass('Confirm new root PIN/passphrase: '),'pin_confirmation_mismatch') - journal=prepare(app,Ed25519Signer.generate(),pin,old_signer=old,operator=operator) - return asyncio.run(complete(app,journal,pin=pin)) + print( + canonical({ + 'old_fingerprint': digest(app.certificates.root_public_key), + 'effect': 'all existing certificate chains require reissuance', + 'lost_key': lost_key, + }).decode() + ) + require( + input('Type ROTATE ROOT ' + digest(app.certificates.root_public_key) + ': ') + == 'ROTATE ROOT ' + digest(app.certificates.root_public_key), + 'approval_cancelled', + ) + old = ( + None + if lost_key + else Ed25519Signer.from_bytes( + open_private_key( + loads(root_envelope(self.config_dir).read_bytes()), + getpass.getpass('Current root PIN/passphrase: '), + ) + ) + ) + pin = getpass.getpass('New root PIN/passphrase: ') + require( + pin == getpass.getpass('Confirm new root PIN/passphrase: '), + 'pin_confirmation_mismatch', + ) + journal = prepare(app, Ed25519Signer.generate(), pin, old_signer=old, operator=operator) + return asyncio.run(complete(app, journal, pin=pin)) def doctor(self): # Imported lazily: this read-only diagnostic must not run schema migrations. from msg.admin.diagnostics import doctor + return doctor(self.config_dir) def selftest(self): from msg.admin.diagnostics import selftest - return asyncio.run(selftest()) + return asyncio.run(selftest()) - def revoke(self,certificate_id,reason): - operator=require_local_console(self.config_dir) - app=self._app() + def revoke(self, certificate_id, reason): + operator = require_local_console(self.config_dir) + app = self._app() asyncio.run(app.load()) - async def read(): - async with app.metadata.transaction(write=False) as tx:return await tx.certificate(certificate_id) - certificate=asyncio.run(read()) - print(canonical({'certificate':wire(certificate),'reason':reason}).decode()) - require(input('Type REVOKE '+certificate_id+' to continue: ')=='REVOKE '+certificate_id,'approval_cancelled') - private=open_private_key(loads(root_envelope(self.config_dir).read_bytes()),getpass.getpass('Root PIN/passphrase: ')) - return asyncio.run(_revoke(app,certificate_id,Ed25519Signer.from_bytes(private),reason=reason,operator=operator)) - - def sign_recovery_proof(self,source_backup_sha256,sequence,destination): + async def read(): + async with app.metadata.transaction(write=False) as tx: + return await tx.certificate(certificate_id) + + certificate = asyncio.run(read()) + print(canonical({'certificate': wire(certificate), 'reason': reason}).decode()) + require( + input('Type REVOKE ' + certificate_id + ' to continue: ') == 'REVOKE ' + certificate_id, + 'approval_cancelled', + ) + private = open_private_key( + loads(root_envelope(self.config_dir).read_bytes()), + getpass.getpass('Root PIN/passphrase: '), + ) + return asyncio.run( + _revoke( + app, + certificate_id, + Ed25519Signer.from_bytes(private), + reason=reason, + operator=operator, + ) + ) + + def sign_recovery_proof(self, source_backup_sha256, sequence, destination): require_local_console(self.config_dir) - from msg.admin.recovery_proof import open_for_proof, draft, seal from msg.admin.backup_retirement import write_record + from msg.admin.recovery_proof import draft, open_for_proof, seal from msg.security.root_files import read_private - app=asyncio.run(open_for_proof(self.config_dir)) + + app = asyncio.run(open_for_proof(self.config_dir)) try: - body=asyncio.run(draft(app,public_key=app.certificates.root_public_key, - source_backup_sha256=source_backup_sha256,sequence=sequence)) - fingerprint=digest(body) - print(canonical({'service':body['service'],'sequence':sequence, - 'source_backup_sha256':source_backup_sha256,'state_digest':fingerprint, - 'tables':len(body['metadata']['tables']),'expires_at':body['expires_at']}).decode()) - require(input('Type SIGN COMPLETE RECOVERY '+fingerprint+': ')== - 'SIGN COMPLETE RECOVERY '+fingerprint,'approval_cancelled') - envelope=loads(read_private(root_envelope(self.config_dir))) - signer=Ed25519Signer.from_bytes(open_private_key(envelope,getpass.getpass('Root PIN/passphrase: '))) - packet=asyncio.run(seal(app,body,signer)) - write_record(Path(destination),packet) - return {'status':'complete_recovery_state_signed','digest':fingerprint, - 'independent_pin_required':True,'path':str(destination)} + body = asyncio.run( + draft( + app, + public_key=app.certificates.root_public_key, + source_backup_sha256=source_backup_sha256, + sequence=sequence, + ) + ) + fingerprint = digest(body) + print( + canonical({ + 'service': body['service'], + 'sequence': sequence, + 'source_backup_sha256': source_backup_sha256, + 'state_digest': fingerprint, + 'tables': len(body['metadata']['tables']), + 'expires_at': body['expires_at'], + }).decode() + ) + require( + input('Type SIGN COMPLETE RECOVERY ' + fingerprint + ': ') + == 'SIGN COMPLETE RECOVERY ' + fingerprint, + 'approval_cancelled', + ) + envelope = loads(read_private(root_envelope(self.config_dir))) + signer = Ed25519Signer.from_bytes( + open_private_key(envelope, getpass.getpass('Root PIN/passphrase: ')) + ) + packet = asyncio.run(seal(app, body, signer)) + write_record(Path(destination), packet) + return { + 'status': 'complete_recovery_state_signed', + 'digest': fingerprint, + 'independent_pin_required': True, + 'path': str(destination), + } finally: asyncio.run(app.close()) - def promote_recovery(self,source,independent_trust): - operator=require_local_console(self.config_dir) - from msg.admin.recovery_proof import open_for_proof, verify_proof, promote, IndependentRecoveryPin + def promote_recovery(self, source, independent_trust): + operator = require_local_console(self.config_dir) + from msg.admin.recovery_proof import ( + IndependentRecoveryPin, + open_for_proof, + promote, + verify_proof, + ) from msg.security.root_files import read_private - packet=loads(read_private(source,limit=64*1024*1024)) - trust=loads(read_private(independent_trust,limit=65536)) - require(type(trust) is dict and set(trust)=={ - 'service','public_key','digest','sequence','source_backup_sha256'},'recovery_independent_pin_required') - pin=IndependentRecoveryPin(**dict(trust,public_key=unb64(trust['public_key'],limit=32))) - app=asyncio.run(open_for_proof(self.config_dir)) + + packet = loads(read_private(source, limit=64 * 1024 * 1024)) + trust = loads(read_private(independent_trust, limit=65536)) + require( + type(trust) is dict + and set(trust) + == {'service', 'public_key', 'digest', 'sequence', 'source_backup_sha256'}, + 'recovery_independent_pin_required', + ) + pin = IndependentRecoveryPin(**dict(trust, public_key=unb64(trust['public_key'], limit=32))) + app = asyncio.run(open_for_proof(self.config_dir)) try: - body=verify_proof(packet,pin,app.clock()) - print(canonical({'service':pin.service,'state_digest':pin.digest,'sequence':pin.sequence, - 'source_backup_sha256':pin.source_backup_sha256,'tables':len(body['metadata']['tables']), - 'effect':'release verified complete recovery; restart all runtimes'}).decode()) - require(input('Type PROMOTE COMPLETE RECOVERY '+pin.digest+': ')== - 'PROMOTE COMPLETE RECOVERY '+pin.digest,'approval_cancelled') - envelope=loads(read_private(root_envelope(self.config_dir))) - signer=Ed25519Signer.from_bytes(open_private_key(envelope,getpass.getpass('Root PIN/passphrase: '))) - return asyncio.run(promote(app,packet,pin=pin,signer=signer,operator=operator)) + body = verify_proof(packet, pin, app.clock()) + print( + canonical({ + 'service': pin.service, + 'state_digest': pin.digest, + 'sequence': pin.sequence, + 'source_backup_sha256': pin.source_backup_sha256, + 'tables': len(body['metadata']['tables']), + 'effect': 'release verified complete recovery; restart all runtimes', + }).decode() + ) + require( + input('Type PROMOTE COMPLETE RECOVERY ' + pin.digest + ': ') + == 'PROMOTE COMPLETE RECOVERY ' + pin.digest, + 'approval_cancelled', + ) + envelope = loads(read_private(root_envelope(self.config_dir))) + signer = Ed25519Signer.from_bytes( + open_private_key(envelope, getpass.getpass('Root PIN/passphrase: ')) + ) + return asyncio.run(promote(app, packet, pin=pin, signer=signer, operator=operator)) finally: asyncio.run(app.close()) - def sign_backup_retirement(self,source,destination): + def sign_backup_retirement(self, source, destination): """Root-sign an operator's statement that listed backup sets lost one old key.""" require_local_console(self.config_dir) from msg.admin.backup_retirement import read_statement, unsigned_statement, write_record from msg.security.backup_retirement import sign_record - target=Path(destination) - require(not target.exists() and not target.is_symlink(),'backup_retirement_destination_exists') - body=unsigned_statement(read_statement(source)) - print(canonical({'statement':body,'digest':digest(body), - 'claim':'listed_backup_sets_only'}).decode()) - require(input('Type ATTEST BACKUP RETIREMENT '+digest(body)+': ')== - 'ATTEST BACKUP RETIREMENT '+digest(body),'approval_cancelled') + + target = Path(destination) + require( + not target.exists() and not target.is_symlink(), 'backup_retirement_destination_exists' + ) + body = unsigned_statement(read_statement(source)) + print( + canonical({ + 'statement': body, + 'digest': digest(body), + 'claim': 'listed_backup_sets_only', + }).decode() + ) + require( + input('Type ATTEST BACKUP RETIREMENT ' + digest(body) + ': ') + == 'ATTEST BACKUP RETIREMENT ' + digest(body), + 'approval_cancelled', + ) from msg.security.root_files import read_private - envelope=loads(read_private(root_envelope(self.config_dir))) - private=open_private_key(envelope,getpass.getpass('Root PIN/passphrase: ')) - record=sign_record(body,Ed25519Signer.from_bytes(private)) - write_record(target,record) - return {'status':'backup_retirement_signed','path':str(target),'digest':digest(record)} - def import_backup_retirement(self,source): + envelope = loads(read_private(root_envelope(self.config_dir))) + private = open_private_key(envelope, getpass.getpass('Root PIN/passphrase: ')) + record = sign_record(body, Ed25519Signer.from_bytes(private)) + write_record(target, record) + return {'status': 'backup_retirement_signed', 'path': str(target), 'digest': digest(record)} + + def import_backup_retirement(self, source): """Only this console path persists a record; reads re-verify it every time.""" - operator=require_local_console(self.config_dir) + operator = require_local_console(self.config_dir) from msg.admin.backup_retirement import import_record, read_statement - app=self._app() - record=read_statement(source) - print(canonical({'record':record,'digest':digest(record)}).decode()) - require(input('Type IMPORT BACKUP RETIREMENT '+digest(record)+': ')== - 'IMPORT BACKUP RETIREMENT '+digest(record),'approval_cancelled') + + app = self._app() + record = read_statement(source) + print(canonical({'record': record, 'digest': digest(record)}).decode()) + require( + input('Type IMPORT BACKUP RETIREMENT ' + digest(record) + ': ') + == 'IMPORT BACKUP RETIREMENT ' + digest(record), + 'approval_cancelled', + ) + async def run(): await app.load() try: async with app.metadata.transaction(write=True) as tx: - return await import_record(tx,record,now=app.clock(),operator=operator) + return await import_record(tx, record, now=app.clock(), operator=operator) finally: await app.close() - attestation=asyncio.run(run()) - return {'status':'backup_retirement_imported','scope':'listed_backup_sets_only', - **attestation.view()} - def backup(self,destination): + attestation = asyncio.run(run()) + return { + 'status': 'backup_retirement_imported', + 'scope': 'listed_backup_sets_only', + **attestation.view(), + } + + def backup(self, destination): require_local_console(self.config_dir) - target=Path(destination) - require(not target.exists() and not target.is_symlink(),'backup_destination_exists') - envelope=loads(root_envelope(self.config_dir).read_bytes()) - trust=loads(trust_file(self.config_dir).read_bytes()) - require(input('Type BACKUP ROOT to continue: ')=='BACKUP ROOT','approval_cancelled') - private=open_private_key(envelope,getpass.getpass('Root PIN/passphrase: ')) - require(b64(Ed25519Signer.from_bytes(private).public_key)==trust['public_key'],'root_key_mismatch') - durable_write(target,canonical({'format':'msg-root-backup-v1','envelope':envelope,'trust':trust}),mode=0o600) - return {'status':'encrypted_backup_created','path':str(target),'fingerprint':digest(unb64(trust['public_key']))} - - - def recover(self,source): + target = Path(destination) + require(not target.exists() and not target.is_symlink(), 'backup_destination_exists') + envelope = loads(root_envelope(self.config_dir).read_bytes()) + trust = loads(trust_file(self.config_dir).read_bytes()) + require(input('Type BACKUP ROOT to continue: ') == 'BACKUP ROOT', 'approval_cancelled') + private = open_private_key(envelope, getpass.getpass('Root PIN/passphrase: ')) + require( + b64(Ed25519Signer.from_bytes(private).public_key) == trust['public_key'], + 'root_key_mismatch', + ) + durable_write( + target, + canonical({'format': 'msg-root-backup-v1', 'envelope': envelope, 'trust': trust}), + mode=0o600, + ) + return { + 'status': 'encrypted_backup_created', + 'path': str(target), + 'fingerprint': digest(unb64(trust['public_key'])), + } + + def recover(self, source): require_local_console(self.config_dir) - target=root_envelope(self.config_dir) - require(not target.exists(),'root_material_already_present') - backup=loads(Path(source).read_bytes()) - require(set(backup)=={'format','envelope','trust'} and backup['format']=='msg-root-backup-v1','invalid_root_backup') - current=loads(trust_file(self.config_dir).read_bytes()) - require(canonical(current)==canonical(backup['trust']),'trust_anchor_mismatch') - print(canonical({'fingerprint':digest(unb64(current['public_key'])),'action':'recover_missing_encrypted_root_key'}).decode()) - require(input('Type RECOVER ROOT to continue: ')=='RECOVER ROOT','approval_cancelled') - private=open_private_key(backup['envelope'],getpass.getpass('Backup root PIN/passphrase: ')) - require(b64(Ed25519Signer.from_bytes(private).public_key)==current['public_key'],'root_key_mismatch') - target.parent.mkdir(mode=0o700,parents=True,exist_ok=True) - os.chmod(target.parent,0o700) - durable_write(target,canonical(backup['envelope']),mode=0o600) - return {'status':'root_key_recovered','fingerprint':digest(unb64(current['public_key']))} - - -async def _revoke(app,certificate_id,signer, *, reason,operator): - require(certificate_id!=app.certificates.root_certificate.resource_id,'root_rotation_required') - require(signer.key_id==app.certificates.root_certificate.key_id,'root_key_mismatch') + target = root_envelope(self.config_dir) + require(not target.exists(), 'root_material_already_present') + backup = loads(Path(source).read_bytes()) + require( + set(backup) == {'format', 'envelope', 'trust'} + and backup['format'] == 'msg-root-backup-v1', + 'invalid_root_backup', + ) + current = loads(trust_file(self.config_dir).read_bytes()) + require(canonical(current) == canonical(backup['trust']), 'trust_anchor_mismatch') + print( + canonical({ + 'fingerprint': digest(unb64(current['public_key'])), + 'action': 'recover_missing_encrypted_root_key', + }).decode() + ) + require(input('Type RECOVER ROOT to continue: ') == 'RECOVER ROOT', 'approval_cancelled') + private = open_private_key( + backup['envelope'], getpass.getpass('Backup root PIN/passphrase: ') + ) + require( + b64(Ed25519Signer.from_bytes(private).public_key) == current['public_key'], + 'root_key_mismatch', + ) + target.parent.mkdir(mode=0o700, parents=True, exist_ok=True) + os.chmod(target.parent, 0o700) + durable_write(target, canonical(backup['envelope']), mode=0o600) + return {'status': 'root_key_recovered', 'fingerprint': digest(unb64(current['public_key']))} + + +async def _revoke(app, certificate_id, signer, *, reason, operator): + require( + certificate_id != app.certificates.root_certificate.resource_id, 'root_rotation_required' + ) + require(signer.key_id == app.certificates.root_certificate.key_id, 'root_key_mismatch') async with app.metadata.transaction(write=True) as tx: - certificate=await tx.certificate(certificate_id) - statement={'certificate_id':certificate_id,'reason':reason,'time':wire(app.clock()), - 'issuer':ROOT_SUBJECT,'operator':operator} - signature=signer.sign(canonical(statement),purpose='revocation') - event=Event(id=new_id('audit'),type='root.cert.revoke',time=app.clock(),request_id=new_id('local'), - actor=ROOT_SUBJECT,subject=ROOT_SUBJECT,resources=(ResourceRef(id=certificate_id),), - data={**statement,'signature':wire(signature)}) - await tx.revoke_certificate(certificate_id,AuditEvent(event=event, - authority=(ResourceRef(id=app.certificates.root_certificate.resource_id),),before_digest=digest(certificate), - after_digest=digest(statement),previous_digest=None,entry_digest='',result='revoked')) - return {'certificate_id':certificate_id,'revoked':True,'signature':wire(signature)} + certificate = await tx.certificate(certificate_id) + statement = { + 'certificate_id': certificate_id, + 'reason': reason, + 'time': wire(app.clock()), + 'issuer': ROOT_SUBJECT, + 'operator': operator, + } + signature = signer.sign(canonical(statement), purpose='revocation') + event = Event( + id=new_id('audit'), + type='root.cert.revoke', + time=app.clock(), + request_id=new_id('local'), + actor=ROOT_SUBJECT, + subject=ROOT_SUBJECT, + resources=(ResourceRef(id=certificate_id),), + data={**statement, 'signature': wire(signature)}, + ) + await tx.revoke_certificate( + certificate_id, + AuditEvent( + event=event, + authority=(ResourceRef(id=app.certificates.root_certificate.resource_id),), + before_digest=digest(certificate), + after_digest=digest(statement), + previous_digest=None, + entry_digest='', + result='revoked', + ), + ) + return {'certificate_id': certificate_id, 'revoked': True, 'signature': wire(signature)} diff --git a/src/msg/admin/rotation.py b/src/msg/admin/rotation.py index 9c945605..20dcddcb 100644 --- a/src/msg/admin/rotation.py +++ b/src/msg/admin/rotation.py @@ -4,30 +4,42 @@ invalidates all old certificate chains for new requests. Historical public keys and signed records remain available; they are not retroactively re-signed. """ + from __future__ import annotations + +import os from dataclasses import replace from datetime import timedelta -import os -from msg.constants import ROOT_SUBJECT, ONLINE_CA, ADMINS_GROUP, CSR_SPACE -from msg.core.codec import canonical, decode, digest, wire, b64, loads +from msg.bootstrap import seed_resource +from msg.constants import ADMINS_GROUP, CSR_SPACE, ONLINE_CA, ROOT_SUBJECT +from msg.core.codec import b64, canonical, decode, digest, wire from msg.core.errors import require -from msg.core.models import Certificate, CertificateRequest, Credential, Signature, IssuancePolicy, AuditEvent, Event, ResourceRef +from msg.core.models import ( + AuditEvent, + Certificate, + CertificateRequest, + Credential, + Event, + IssuancePolicy, + ResourceRef, + Signature, +) from msg.plugins.common import new_id from msg.plugins.identity import certificate_resource -from msg.security.crypto import seal_private_key, Ed25519Signer -from msg.security.certificates import sign_certificate, csr_body -from msg.security.rotation_journal import authorization, validate as validate_journal -from msg.security.root_files import read_private, rotation_lock from msg.security.capabilities import grant_for -from msg.storage.git import durable_write +from msg.security.certificates import csr_body, sign_certificate +from msg.security.crypto import Ed25519Signer, seal_private_key +from msg.security.root_files import read_private, rotation_lock +from msg.security.rotation_journal import authorization, validate as validate_journal from msg.security.trust_files import trust_values, write_trust -from msg.bootstrap import seed_resource +from msg.storage.git import durable_write def journal_path(app): from msg.admin.root import root_envelope - return root_envelope(app.settings.config_dir).parent/'rotation.pending.json' + + return root_envelope(app.settings.config_dir).parent / 'rotation.pending.json' def prepare(app, new_signer, new_pin, *, old_signer, operator): @@ -42,42 +54,92 @@ def _prepare(app, new_signer, new_pin, *, old_signer, operator): require(old_signer.key_id == old.key_id, 'root_key_mismatch') now = app.clock() grants = app.primary_ceiling() - new = Certificate(resource_id=new_id('cert_root'), serial=new_id('serial'), subject_id=ROOT_SUBJECT, - key_id=new_signer.key_id, issuer_id=ROOT_SUBJECT, parent_certificate_id=None, authority_sources=(), kind='ca', - grants=grants, not_before=now, expires_at=now+timedelta(days=3650), target_service=app.settings.service_url, - delegation_depth=8, issuance=IssuancePolicy(issue_grants=grants,max_cert_ttl_seconds=31536000, - max_child_ca_depth=3,max_delegation_depth=8), - signature=Signature(key_id=new_signer.key_id, algorithm='ed25519', value=b'')) + new = Certificate( + resource_id=new_id('cert_root'), + serial=new_id('serial'), + subject_id=ROOT_SUBJECT, + key_id=new_signer.key_id, + issuer_id=ROOT_SUBJECT, + parent_certificate_id=None, + authority_sources=(), + kind='ca', + grants=grants, + not_before=now, + expires_at=now + timedelta(days=3650), + target_service=app.settings.service_url, + delegation_depth=8, + issuance=IssuancePolicy( + issue_grants=grants, + max_cert_ttl_seconds=31536000, + max_child_ca_depth=3, + max_delegation_depth=8, + ), + signature=Signature(key_id=new_signer.key_id, algorithm='ed25519', value=b''), + ) new = sign_certificate(new, new_signer) - statement = {'old_certificate':old.resource_id, 'old_fingerprint':digest(app.certificates.root_public_key), - 'new_certificate':new.resource_id, 'new_fingerprint':digest(new_signer.public_key), 'time':wire(now), - 'operator':operator, 'previous_key_proved':old_signer is not None} + statement = { + 'old_certificate': old.resource_id, + 'old_fingerprint': digest(app.certificates.root_public_key), + 'new_certificate': new.resource_id, + 'new_fingerprint': digest(new_signer.public_key), + 'time': wire(now), + 'operator': operator, + 'previous_key_proved': old_signer is not None, + } online = app.online_signer - csr = CertificateRequest(resource_id=new_id('csr'), applicant=ONLINE_CA, subject_id=ONLINE_CA, - requested_issuer=ROOT_SUBJECT, public_key=online.public_key, kind='ca', - grants=(grant_for(app.registry.capability('cert.issue'),scope=app.default_scope()),), - issuance=IssuancePolicy(issue_grants=app.base_grants(),max_cert_ttl_seconds=app.settings.base_certificate_ttl, - max_child_ca_depth=0,max_delegation_depth=8), requested_ttl_seconds=31536000, - target_service=app.settings.service_url,delegation_depth=0,authority_sources=(),request_digest='', - possession_proof=Signature(key_id=online.key_id,algorithm='ed25519',value=b'')) - csr = replace(csr, request_digest=digest(csr_body(csr)), - possession_proof=online.sign(canonical(csr_body(csr)),purpose='csr')) + csr = CertificateRequest( + resource_id=new_id('csr'), + applicant=ONLINE_CA, + subject_id=ONLINE_CA, + requested_issuer=ROOT_SUBJECT, + public_key=online.public_key, + kind='ca', + grants=(grant_for(app.registry.capability('cert.issue'), scope=app.default_scope()),), + issuance=IssuancePolicy( + issue_grants=app.base_grants(), + max_cert_ttl_seconds=app.settings.base_certificate_ttl, + max_child_ca_depth=0, + max_delegation_depth=8, + ), + requested_ttl_seconds=31536000, + target_service=app.settings.service_url, + delegation_depth=0, + authority_sources=(), + request_digest='', + possession_proof=Signature(key_id=online.key_id, algorithm='ed25519', value=b''), + ) + csr = replace( + csr, + request_digest=digest(csr_body(csr)), + possession_proof=online.sign(canonical(csr_body(csr)), purpose='csr'), + ) from msg.admin.root import root_envelope - key_file=root_envelope(app.settings.config_dir) - require(not key_file.is_symlink(),'unsafe_root_private_path') - journal={'version':2,'old_certificate':wire(old),'new_trust':{'version':1, - 'public_key':b64(new_signer.public_key),'certificate':wire(new)}, - 'new_envelope':seal_private_key(new_signer.private_bytes(),new_pin), - 'previous_envelope_digest':digest(read_private(key_file)) if key_file.exists() else None, - 'online_csr':wire(csr),'statement':statement} - proof=canonical(authorization(journal)) - journal['new_signature']=wire(new_signer.sign(proof,purpose='root-rotation')) - journal['old_signature']=wire(old_signer.sign(proof,purpose='root-rotation')) if old_signer else None - protected=journal_path(app).parent - require(not protected.is_symlink(),'unsafe_root_private_path') - protected.mkdir(parents=True,exist_ok=True,mode=0o700) - os.chmod(protected,0o700) - durable_write(journal_path(app),canonical(journal),mode=0o600) + + key_file = root_envelope(app.settings.config_dir) + require(not key_file.is_symlink(), 'unsafe_root_private_path') + journal = { + 'version': 2, + 'old_certificate': wire(old), + 'new_trust': { + 'version': 1, + 'public_key': b64(new_signer.public_key), + 'certificate': wire(new), + }, + 'new_envelope': seal_private_key(new_signer.private_bytes(), new_pin), + 'previous_envelope_digest': digest(read_private(key_file)) if key_file.exists() else None, + 'online_csr': wire(csr), + 'statement': statement, + } + proof = canonical(authorization(journal)) + journal['new_signature'] = wire(new_signer.sign(proof, purpose='root-rotation')) + journal['old_signature'] = ( + wire(old_signer.sign(proof, purpose='root-rotation')) if old_signer else None + ) + protected = journal_path(app).parent + require(not protected.is_symlink(), 'unsafe_root_private_path') + protected.mkdir(parents=True, exist_ok=True, mode=0o700) + os.chmod(protected, 0o700) + durable_write(journal_path(app), canonical(journal), mode=0o600) return journal @@ -88,112 +150,196 @@ async def complete(app, journal, *, pin): async def _complete(app, journal, *, pin): from msg.security.crypto import verify - pending=journal_path(app) - protected=pending.parent - require(not protected.is_symlink() and not pending.is_symlink(), - 'unsafe_root_private_path') - online=Ed25519Signer.from_bytes((app.settings.service_keys/'online.key').read_bytes()) - signer,old,new,csr,proof=validate_journal(journal,pin=pin, - service=app.settings.service_url,online_public=online.public_key) - key_file=protected/'key.json' - require(not key_file.is_symlink(),'unsafe_root_private_path') + + pending = journal_path(app) + protected = pending.parent + require(not protected.is_symlink() and not pending.is_symlink(), 'unsafe_root_private_path') + online = Ed25519Signer.from_bytes((app.settings.service_keys / 'online.key').read_bytes()) + signer, old, new, csr, proof = validate_journal( + journal, pin=pin, service=app.settings.service_url, online_public=online.public_key + ) + key_file = protected / 'key.json' + require(not key_file.is_symlink(), 'unsafe_root_private_path') if key_file.exists(): - require(digest(read_private(key_file)) in { - journal['previous_envelope_digest'],digest(canonical(journal['new_envelope']))}, - 'root_envelope_changed') + require( + digest(read_private(key_file)) + in {journal['previous_envelope_digest'], digest(canonical(journal['new_envelope']))}, + 'root_envelope_changed', + ) else: - require(journal['previous_envelope_digest'] is None,'root_envelope_missing') + require(journal['previous_envelope_digest'] is None, 'root_envelope_missing') # Reject unsafe or substituted history BEFORE changing the database anchor. - history = protected/'history'/old.resource_id/'key.json' - for directory in (protected/'history', history.parent): + history = protected / 'history' / old.resource_id / 'key.json' + for directory in (protected / 'history', history.parent): require(not directory.is_symlink(), 'unsafe_root_private_path') require(not history.is_symlink(), 'unsafe_root_private_path') if history.exists(): - require(digest(read_private(history)) == journal['previous_envelope_digest'], - 'rotation_history_mismatch') + require( + digest(read_private(history)) == journal['previous_envelope_digest'], + 'rotation_history_mismatch', + ) elif journal['previous_envelope_digest'] is not None: - require(key_file.exists() and - digest(read_private(key_file)) == journal['previous_envelope_digest'], - 'rotation_history_missing') + require( + key_file.exists() + and digest(read_private(key_file)) == journal['previous_envelope_digest'], + 'rotation_history_missing', + ) current_trust = trust_values(app.settings.config_dir) - require(bool(current_trust),'rotation_trust_missing') + require(bool(current_trust), 'rotation_trust_missing') # This method is called only after the local console has approved the exact # journal. It also handles the state where PostgreSQL committed but protected files did not. await app.open_storage() async with app.metadata.transaction(write=True) as tx: - current=tx.setting('active_root_certificate','cert_root') - require(current in {old.resource_id,new.resource_id},'trust_anchor_mismatch') - existing=await tx.certificate(old.resource_id) - require(canonical(existing)==canonical(old),'trust_anchor_mismatch') - old_key=await tx.credential(old.key_id) - require(old_key.subject_id==ROOT_SUBJECT and - digest(old_key.verifier)==journal['statement']['old_fingerprint'], - 'rotation_statement_mismatch') + current = tx.setting('active_root_certificate', 'cert_root') + require(current in {old.resource_id, new.resource_id}, 'trust_anchor_mismatch') + existing = await tx.certificate(old.resource_id) + require(canonical(existing) == canonical(old), 'trust_anchor_mismatch') + old_key = await tx.credential(old.key_id) + require( + old_key.subject_id == ROOT_SUBJECT + and digest(old_key.verifier) == journal['statement']['old_fingerprint'], + 'rotation_statement_mismatch', + ) old_trust = {'version': 1, 'public_key': b64(old_key.verifier), 'certificate': wire(old)} - require(all(canonical(value) in (canonical(old_trust), canonical(journal['new_trust'])) - for value in current_trust.values()), - 'rotation_trust_changed') + require( + all( + canonical(value) in (canonical(old_trust), canonical(journal['new_trust'])) + for value in current_trust.values() + ), + 'rotation_trust_changed', + ) if journal['old_signature'] is not None: - verify(old_key.verifier,canonical(proof),decode(Signature,journal['old_signature']), - purpose='root-rotation') + verify( + old_key.verifier, + canonical(proof), + decode(Signature, journal['old_signature']), + purpose='root-rotation', + ) if current == new.resource_id: - require(canonical(await tx.certificate(new.resource_id))==canonical(new) and - canonical(await tx.csr(csr.resource_id))==canonical(csr) and - old_key.revoked_at is not None and await tx.certificate_revoked(old.resource_id), - 'rotation_resume_mismatch') - installed=await tx.credential(new.key_id) - require(installed.verifier==signer.public_key and installed.revoked_at is None, - 'rotation_resume_mismatch') + require( + canonical(await tx.certificate(new.resource_id)) == canonical(new) + and canonical(await tx.csr(csr.resource_id)) == canonical(csr) + and old_key.revoked_at is not None + and await tx.certificate_revoked(old.resource_id), + 'rotation_resume_mismatch', + ) + installed = await tx.credential(new.key_id) + require( + installed.verifier == signer.public_key and installed.revoked_at is None, + 'rotation_resume_mismatch', + ) else: - subject=await tx.subject(ROOT_SUBJECT) - require(subject.local_only,'root_policy_corrupt') + subject = await tx.subject(ROOT_SUBJECT) + require(subject.local_only, 'root_policy_corrupt') # Credential saves are upserts; a retired root key must never be revived. - require(tx.one('SELECT 1 FROM credentials WHERE id=?',(signer.key_id,)) is None, - 'root_key_reused') - await tx.save_credential(Credential(id=signer.key_id,subject_id=ROOT_SUBJECT,kind='signing_key', - verifier=signer.public_key,ceiling=(),not_before=new.not_before,expires_at=None,revoked_at=None),subject.auth_version) - await tx.save_credential(replace(old_key,revoked_at=app.clock()),subject.auth_version) - await certificate_resource(tx,new,app.clock()) - await tx.register_certificate(new,None,0) - event=Event(id=new_id('audit'),type='root.rotate',time=app.clock(),request_id=new.resource_id, - actor=ROOT_SUBJECT,subject=ROOT_SUBJECT,resources=(ResourceRef(id=new.resource_id),), - data={'statement':journal['statement'],'authorization':proof,'new_signature':journal['new_signature'], - 'old_signature':journal['old_signature']}) - await tx.revoke_certificate(old.resource_id,AuditEvent(event=event,authority=(), - before_digest=digest(old),after_digest=digest(new),previous_digest=None,entry_digest='',result='root_rotated')) - await seed_resource(tx,app.contents,dict(id=csr.resource_id,type='csr',name=csr.resource_id,parent=CSR_SPACE, - owner=ONLINE_CA,group=ADMINS_GROUP,mode='0600'),app.clock()) + require( + tx.one('SELECT 1 FROM credentials WHERE id=?', (signer.key_id,)) is None, + 'root_key_reused', + ) + await tx.save_credential( + Credential( + id=signer.key_id, + subject_id=ROOT_SUBJECT, + kind='signing_key', + verifier=signer.public_key, + ceiling=(), + not_before=new.not_before, + expires_at=None, + revoked_at=None, + ), + subject.auth_version, + ) + await tx.save_credential(replace(old_key, revoked_at=app.clock()), subject.auth_version) + await certificate_resource(tx, new, app.clock()) + await tx.register_certificate(new, None, 0) + event = Event( + id=new_id('audit'), + type='root.rotate', + time=app.clock(), + request_id=new.resource_id, + actor=ROOT_SUBJECT, + subject=ROOT_SUBJECT, + resources=(ResourceRef(id=new.resource_id),), + data={ + 'statement': journal['statement'], + 'authorization': proof, + 'new_signature': journal['new_signature'], + 'old_signature': journal['old_signature'], + }, + ) + await tx.revoke_certificate( + old.resource_id, + AuditEvent( + event=event, + authority=(), + before_digest=digest(old), + after_digest=digest(new), + previous_digest=None, + entry_digest='', + result='root_rotated', + ), + ) + await seed_resource( + tx, + app.contents, + { + 'id': csr.resource_id, + 'type': 'csr', + 'name': csr.resource_id, + 'parent': CSR_SPACE, + 'owner': ONLINE_CA, + 'group': ADMINS_GROUP, + 'mode': '0600', + }, + app.clock(), + ) await tx.save_csr(csr) - tx.set_setting('online_ca_request',csr.resource_id) - tx.set_setting('online_ca_certificate',None) - tx.set_setting('active_root_certificate',new.resource_id) - tx.set_setting('authorization_epoch',tx.setting('authorization_epoch',0)+1) - tx.set_setting('root_rotation:'+new.resource_id,journal['statement']) + tx.set_setting('online_ca_request', csr.resource_id) + tx.set_setting('online_ca_certificate', None) + tx.set_setting('active_root_certificate', new.resource_id) + tx.set_setting('authorization_epoch', tx.setting('authorization_epoch', 0) + 1) + tx.set_setting('root_rotation:' + new.resource_id, journal['statement']) # The journal and envelope must use the same independently protected root # directory, including when resuming after the metadata commit. - for directory in (protected,protected/'history',protected/'history'/old.resource_id): - require(not directory.is_symlink(),'unsafe_root_private_path') - directory.mkdir(parents=True,exist_ok=True,mode=0o700) - os.chmod(directory,0o700) - history=protected/'history'/old.resource_id/'key.json' - require(not history.is_symlink(),'unsafe_root_private_path') + for directory in (protected, protected / 'history', protected / 'history' / old.resource_id): + require(not directory.is_symlink(), 'unsafe_root_private_path') + directory.mkdir(parents=True, exist_ok=True, mode=0o700) + os.chmod(directory, 0o700) + history = protected / 'history' / old.resource_id / 'key.json' + require(not history.is_symlink(), 'unsafe_root_private_path') if history.exists(): - require(digest(read_private(history))==journal['previous_envelope_digest'], - 'rotation_history_mismatch') + require( + digest(read_private(history)) == journal['previous_envelope_digest'], + 'rotation_history_mismatch', + ) elif journal['previous_envelope_digest'] is not None: # Never archive an already installed new envelope as the old key. - require(key_file.exists() and - digest(read_private(key_file))==journal['previous_envelope_digest'], - 'rotation_history_missing') - durable_write(history,read_private(key_file),mode=0o600) - durable_write(key_file,canonical(journal['new_envelope']),mode=0o600) - write_trust(app.settings.config_dir,journal['new_trust'],writer=durable_write, - approved_values=(old_trust,journal['new_trust'])) + require( + key_file.exists() + and digest(read_private(key_file)) == journal['previous_envelope_digest'], + 'rotation_history_missing', + ) + durable_write(history, read_private(key_file), mode=0o600) + durable_write(key_file, canonical(journal['new_envelope']), mode=0o600) + write_trust( + app.settings.config_dir, + journal['new_trust'], + writer=durable_write, + approved_values=(old_trust, journal['new_trust']), + ) # Journal survives every earlier failure and is removed only after both # metadata and protected files have durable copies. pending.unlink(missing_ok=True) - directory=os.open(protected,os.O_RDONLY|os.O_DIRECTORY) - try:os.fsync(directory) - finally:os.close(directory) - return {'root_id':ROOT_SUBJECT,'certificate_id':new.resource_id,'fingerprint':digest(signer.public_key), - 'online_ca_request':csr.resource_id,'restart_required':True,'old_certificate_chains_valid':False} + directory = os.open(protected, os.O_RDONLY | os.O_DIRECTORY) + try: + os.fsync(directory) + finally: + os.close(directory) + return { + 'root_id': ROOT_SUBJECT, + 'certificate_id': new.resource_id, + 'fingerprint': digest(signer.public_key), + 'online_ca_request': csr.resource_id, + 'restart_required': True, + 'old_certificate_chains_valid': False, + } diff --git a/src/msg/admin/token_delivery_check.py b/src/msg/admin/token_delivery_check.py index 6354c17b..c542cf53 100644 --- a/src/msg/admin/token_delivery_check.py +++ b/src/msg/admin/token_delivery_check.py @@ -1,4 +1,5 @@ """Exercise one-use credential release only on the caller's isolated Test Root.""" + from datetime import timedelta from pathlib import Path from tempfile import TemporaryDirectory @@ -40,8 +41,12 @@ async def discard(packet): # Both processes can restart; refreshing only the request expiry # does not refresh either the token lifetime or recovery deadline. state = ClientState(Path(directory), server=app.settings.service_url) - client = MsgClient(state, HTTPTransport(state.server, http=http), - clock=lambda: now+timedelta(seconds=1), retries=0) + client = MsgClient( + state, + HTTPTransport(state.server, http=http), + clock=lambda: now + timedelta(seconds=1), + retries=0, + ) replay = await client.recover_token() if replay.status != 'error' or replay.error.code != 'token_delivery_unavailable': return False @@ -51,4 +56,8 @@ async def discard(packet): async with app.metadata.transaction(write=False) as tx: old = await tx.credential(original[0]) current = await tx.credential(state.token[0]) - return old.revoked_at is not None and current.revoked_at is None and client._token_journal() == (None, None) + return ( + old.revoked_at is not None + and current.revoked_at is None + and client._token_journal() == (None, None) + ) diff --git a/src/msg/admin/tool_check.py b/src/msg/admin/tool_check.py index 506dabdf..62061699 100644 --- a/src/msg/admin/tool_check.py +++ b/src/msg/admin/tool_check.py @@ -1,4 +1,5 @@ """Tool readiness and isolated real-runner exercise; never contact an external host.""" + import asyncio import tempfile from dataclasses import replace @@ -16,8 +17,9 @@ async def inspect_tool_sandbox(app): policy = replace(tool.network, hosts=('denied.invalid',), timeout_ms=1000) with tempfile.TemporaryDirectory(prefix='msg-tool-doctor-') as directory: try: - await BubblewrapRunner(app)(tool, {'url': 'http://127.0.0.1/'}, - (policy,), Path(directory)) + await BubblewrapRunner(app)( + tool, {'url': 'http://127.0.0.1/'}, (policy,), Path(directory) + ) except Failure as exc: require(exc.code == 'network_policy_denied', exc.code) else: @@ -27,48 +29,75 @@ async def inspect_tool_sandbox(app): async def check_tool_sandbox(app, call, approve, key, subject): """Check real tool.run denial and real bwrap output using a loopback-only fixture.""" - require(app.selftest_run_id is not None,'selftest_namespace_required') + require(app.selftest_run_id is not None, 'selftest_namespace_required') payload = b'msg isolated sandbox selftest\n' received = [] + async def serve(reader, writer): try: request = await asyncio.wait_for(reader.readuntil(b'\r\n\r\n'), 5) received.append(request.split(b'\r\n', 1)[0]) - writer.write(b'HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nContent-Length: '+ - str(len(payload)).encode()+b'\r\nConnection: close\r\n\r\n'+payload) + writer.write( + b'HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nContent-Length: ' + + str(len(payload)).encode() + + b'\r\nConnection: close\r\n\r\n' + + payload + ) await writer.drain() finally: writer.close() await writer.wait_closed() + server = await asyncio.start_server(serve, '127.0.0.1', 0, limit=8192) try: port = server.sockets[0].getsockname()[1] scope = Scope(resource_id='tool_curl', descendants=False) - grant = replace(grant_for(app.registry.capability('tool.use'), scope=scope), - constraints={'hosts': ['127.0.0.1'], 'ports': [80], 'methods': ['GET']}) + grant = replace( + grant_for(app.registry.capability('tool.use'), scope=scope), + constraints={'hosts': ['127.0.0.1'], 'ports': [80], 'methods': ['GET']}, + ) certificate = await approve(key, subject, (grant,)) - denied = await call('tool.run', {'id': 'tool_curl', - 'arguments': {'url': 'http://127.0.0.1/'}}, - key, subject, (certificate.resource_id,)) - require(denied.error is not None and denied.error.code == 'network_policy_denied', - 'tool_selftest_private_denial_missing') + denied = await call( + 'tool.run', + {'id': 'tool_curl', 'arguments': {'url': 'http://127.0.0.1/'}}, + key, + subject, + (certificate.resource_id,), + ) + require( + denied.error is not None and denied.error.code == 'network_policy_denied', + 'tool_selftest_private_denial_missing', + ) tool = app.registry.tool('tool_curl') - policy = replace(tool.network, hosts=('127.0.0.1',), ports=frozenset({port}), - methods=frozenset({'GET'}), allow_private=True, timeout_ms=3000, - max_response_bytes=4096, max_redirects=0) + policy = replace( + tool.network, + hosts=('127.0.0.1',), + ports=frozenset({port}), + methods=frozenset({'GET'}), + allow_private=True, + timeout_ms=3000, + max_response_bytes=4096, + max_redirects=0, + ) # This exercises the real runner port. It deliberately does not alter the # registered tool's 80/443 policy to claim a normal tool.run success. with tempfile.TemporaryDirectory(prefix='msg-tool-selftest-') as directory: - result = await BubblewrapRunner(app)(tool, {'url': f'http://127.0.0.1:{port}/'}, - (policy,), Path(directory)) - require(result.path.read_bytes() == payload and result.metadata['status'] == 200, - 'tool_selftest_output_mismatch') + result = await BubblewrapRunner(app)( + tool, {'url': f'http://127.0.0.1:{port}/'}, (policy,), Path(directory) + ) + require( + result.path.read_bytes() == payload and result.metadata['status'] == 200, + 'tool_selftest_output_mismatch', + ) require(received == [b'GET / HTTP/1.1'], 'tool_selftest_request_mismatch') await inspect_tool_sandbox(app) require(received == [b'GET / HTTP/1.1'], 'tool_selftest_unexpected_request') - return {'operation': 'private_target_denied', - 'runner': 'loopback_output_and_policy_denial', - 'normal_operation_pipeline': 'not_exercised', 'external_requests': 0} + return { + 'operation': 'private_target_denied', + 'runner': 'loopback_output_and_policy_denial', + 'normal_operation_pipeline': 'not_exercised', + 'external_requests': 0, + } finally: server.close() await server.wait_closed() diff --git a/src/msg/admin/transfer_check.py b/src/msg/admin/transfer_check.py index 7016936c..ca56239d 100644 --- a/src/msg/admin/transfer_check.py +++ b/src/msg/admin/transfer_check.py @@ -1,4 +1,5 @@ """Bounded TransferSession checks in the disposable selftest installation.""" + from msg.core.codec import b64, digest, unb64, wire from msg.core.errors import require @@ -10,70 +11,139 @@ async def check_transfer(app, call, register): async def invoke(operation, arguments): result = await call(operation, arguments, key, owner) - require(result.status == 'ok', 'selftest_transfer_operation_failed', - details={'operation': operation}) + require( + result.status == 'ok', + 'selftest_transfer_operation_failed', + details={'operation': operation}, + ) return result async def snapshot(): async with app.metadata.transaction(write=False) as tx: - return {table: tuple(sorted(tx.rows(f'SELECT * FROM {table}'), key=repr)) - for table in ('resources', 'revisions', 'events', 'audit', 'jobs', - 'results', 'transfers', 'chunks', 'settings', 'credentials')} + return { + table: tuple(sorted(tx.rows(f'SELECT * FROM {table}'), key=repr)) + for table in ( + 'resources', + 'revisions', + 'events', + 'audit', + 'jobs', + 'results', + 'transfers', + 'chunks', + 'settings', + 'credentials', + ) + } payload = b'isolated transfer selftest\n' + bytes(range(256)) * 2 - opened = await invoke('transfer.open', {'direction': 'upload', 'size': len(payload), - 'digest': digest(payload), 'requested_part_bytes': 128}) + opened = await invoke( + 'transfer.open', + { + 'direction': 'upload', + 'size': len(payload), + 'digest': digest(payload), + 'requested_part_bytes': 128, + }, + ) transfer = opened.data['transfer_id'] part_bytes = opened.data['part_bytes'] require(0 < part_bytes <= 128, 'selftest_transfer_part_size') # The wrong principal must not be able to write a valid chunk or read state. before = await snapshot() for operation, arguments in ( - ('transfer.status', {'transfer_id': transfer}), - ('transfer.part_put', {'transfer_id': transfer, 'offset': 0, - 'data': b64(payload[:1]), 'digest': digest(payload[:1])})): + ('transfer.status', {'transfer_id': transfer}), + ( + 'transfer.part_put', + { + 'transfer_id': transfer, + 'offset': 0, + 'data': b64(payload[:1]), + 'digest': digest(payload[:1]), + }, + ), + ): denied = await call(operation, arguments, other_key, other) - require(denied.error is not None and denied.error.code == 'transfer_owner_required', - 'selftest_transfer_owner_not_enforced') - bad = await call('transfer.part_put', {'transfer_id': transfer, 'offset': 0, - 'data': b64(payload[:1]), 'digest': digest(b'wrong')}, key, owner) - require(bad.error is not None and bad.error.code == 'chunk_digest_mismatch', - 'selftest_transfer_bad_digest_accepted') + require( + denied.error is not None and denied.error.code == 'transfer_owner_required', + 'selftest_transfer_owner_not_enforced', + ) + bad = await call( + 'transfer.part_put', + { + 'transfer_id': transfer, + 'offset': 0, + 'data': b64(payload[:1]), + 'digest': digest(b'wrong'), + }, + key, + owner, + ) + require( + bad.error is not None and bad.error.code == 'chunk_digest_mismatch', + 'selftest_transfer_bad_digest_accepted', + ) require(await snapshot() == before, 'selftest_transfer_rejection_mutated') for offset in reversed(range(0, len(payload), part_bytes)): - block = payload[offset:offset + part_bytes] - await invoke('transfer.part_put', {'transfer_id': transfer, 'offset': offset, - 'data': b64(block), 'digest': digest(block)}) + block = payload[offset : offset + part_bytes] + await invoke( + 'transfer.part_put', + { + 'transfer_id': transfer, + 'offset': offset, + 'data': b64(block), + 'digest': digest(block), + }, + ) status = await invoke('transfer.status', {'transfer_id': transfer}) require(not status.data['missing'], 'selftest_transfer_missing_parts') - sealed = await invoke('transfer.seal', {'transfer_id': transfer, - 'final_size': len(payload), 'final_digest': digest(payload)}) + sealed = await invoke( + 'transfer.seal', + {'transfer_id': transfer, 'final_size': len(payload), 'final_digest': digest(payload)}, + ) require(len(sealed.resources) == 1, 'selftest_transfer_output_missing') async with app.metadata.transaction(write=False) as tx: resource = await tx.resource(sealed.resources[0].id) - require(resource.type == 'file' and resource.mode == 0o600 and resource.owner == owner, - 'selftest_transfer_output_not_private') - downloaded = await invoke('transfer.open', {'direction': 'download', - 'target': wire(sealed.resources[0]), 'requested_part_bytes': 128}) + require( + resource.type == 'file' and resource.mode == 0o600 and resource.owner == owner, + 'selftest_transfer_output_not_private', + ) + downloaded = await invoke( + 'transfer.open', + {'direction': 'download', 'target': wire(sealed.resources[0]), 'requested_part_bytes': 128}, + ) download = downloaded.data['transfer_id'] download_part_bytes = downloaded.data['part_bytes'] require(0 < download_part_bytes <= 128, 'selftest_transfer_part_size') before = await snapshot() received = bytearray() for offset in range(0, len(payload), download_part_bytes): - part = await invoke('transfer.part_get', {'transfer_id': download, 'offset': offset, - 'length': min(download_part_bytes, len(payload) - offset)}) + part = await invoke( + 'transfer.part_get', + { + 'transfer_id': download, + 'offset': offset, + 'length': min(download_part_bytes, len(payload) - offset), + }, + ) block = unb64(part.data['data']) - require(digest(block) == part.data['chunk']['content']['digest'], - 'selftest_transfer_chunk_digest_mismatch') + require( + digest(block) == part.data['chunk']['content']['digest'], + 'selftest_transfer_chunk_digest_mismatch', + ) received.extend(block) await invoke('transfer.status', {'transfer_id': transfer}) await invoke('transfer.status', {'transfer_id': download}) - denied = await call('transfer.part_get', {'transfer_id': download, 'offset': 0, 'length': 1}, - other_key, other) - require(denied.error is not None and denied.error.code == 'transfer_owner_required', - 'selftest_transfer_download_owner_not_enforced') - require(bytes(received) == payload and digest(bytes(received)) == digest(payload), - 'selftest_transfer_download_mismatch') + denied = await call( + 'transfer.part_get', {'transfer_id': download, 'offset': 0, 'length': 1}, other_key, other + ) + require( + denied.error is not None and denied.error.code == 'transfer_owner_required', + 'selftest_transfer_download_owner_not_enforced', + ) + require( + bytes(received) == payload and digest(bytes(received)) == digest(payload), + 'selftest_transfer_download_mismatch', + ) require(await snapshot() == before, 'selftest_transfer_read_mutated') return True diff --git a/src/msg/admin/upgrade_check.py b/src/msg/admin/upgrade_check.py index 46021d81..3b0ba8eb 100644 --- a/src/msg/admin/upgrade_check.py +++ b/src/msg/admin/upgrade_check.py @@ -1,7 +1,8 @@ """Isolated selftest of the real upgrade commit and new-key recovery boundary.""" -from datetime import timedelta + import hashlib import os +from datetime import timedelta from uuid import uuid4 from msg.core.codec import b64, canonical, unb64, wire @@ -16,43 +17,97 @@ async def check_upgrade_recovery(app, now): nonce, recovery = b64(os.urandom(32)), b64(os.urandom(32)) subject = 'u_tmp_' + hashlib.sha256(unb64(nonce)).hexdigest()[:32] request_id = uuid4().hex - proof = signer.sign(canonical({'subject_id': subject, 'nonce': nonce, - 'public_key': b64(signer.public_key), 'encryption_recipient': recipient, - 'request_id': request_id}), purpose='temporary-key-possession-v1') - created = await app.executor.execute(request_for('identity.temporary', { - 'nonce': nonce, 'recovery_secret': recovery, 'public_key': b64(signer.public_key), - 'encryption_recipient': recipient, 'possession_proof': wire(proof)}, - app.settings.service_url, request_id=request_id, contract_version=3, - expires_at=now + timedelta(seconds=120))) + proof = signer.sign( + canonical({ + 'subject_id': subject, + 'nonce': nonce, + 'public_key': b64(signer.public_key), + 'encryption_recipient': recipient, + 'request_id': request_id, + }), + purpose='temporary-key-possession-v1', + ) + created = await app.executor.execute( + request_for( + 'identity.temporary', + { + 'nonce': nonce, + 'recovery_secret': recovery, + 'public_key': b64(signer.public_key), + 'encryption_recipient': recipient, + 'possession_proof': wire(proof), + }, + app.settings.service_url, + request_id=request_id, + contract_version=3, + expires_at=now + timedelta(seconds=120), + ) + ) if created.status != 'ok': return False token = created.data['credential_id'], unb64(created.data['token']) upgrade_id = uuid4().hex - signed = {'subject_id': subject, 'handle': 'selftest-upgrade', - 'public_key': b64(signer.public_key), 'encryption_recipient': recipient} + signed = { + 'subject_id': subject, + 'handle': 'selftest-upgrade', + 'public_key': b64(signer.public_key), + 'encryption_recipient': recipient, + } arguments = {key: value for key, value in signed.items() if key != 'subject_id'} arguments['possession_proof'] = wire(signer.sign(canonical(signed), purpose='upgrade')) - committed = await app.executor.execute(request_for('identity.upgrade', arguments, - app.settings.service_url, subject=subject, token=token, request_id=upgrade_id, - contract_version=2, expires_at=now + timedelta(seconds=120))) + committed = await app.executor.execute( + request_for( + 'identity.upgrade', + arguments, + app.settings.service_url, + subject=subject, + token=token, + request_id=upgrade_id, + contract_version=2, + expires_at=now + timedelta(seconds=120), + ) + ) if committed.status != 'ok': return False async with app.metadata.transaction(write=False) as tx: - before = tuple(tx.one('SELECT COUNT(*) FROM ' + table)[0] - for table in ('events', 'results', 'credentials', 'certificates')) + before = tuple( + tx.one('SELECT COUNT(*) FROM ' + table)[0] + for table in ('events', 'results', 'credentials', 'certificates') + ) # No old token, bootstrap nonce, original response or cached certificate is # supplied to this signed query. The stored result is only a lookup target. - recovered = await app.executor.execute(request_for('identity.upgrade_result', - {'upgrade_request_id': upgrade_id}, app.settings.service_url, subject=subject, - signer=signer, expires_at=now + timedelta(seconds=120))) - denied = await app.executor.execute(request_for('identity.upgrade_result', - {'upgrade_request_id': upgrade_id}, app.settings.service_url, subject=subject, - token=token, expires_at=now + timedelta(seconds=120))) + recovered = await app.executor.execute( + request_for( + 'identity.upgrade_result', + {'upgrade_request_id': upgrade_id}, + app.settings.service_url, + subject=subject, + signer=signer, + expires_at=now + timedelta(seconds=120), + ) + ) + denied = await app.executor.execute( + request_for( + 'identity.upgrade_result', + {'upgrade_request_id': upgrade_id}, + app.settings.service_url, + subject=subject, + token=token, + expires_at=now + timedelta(seconds=120), + ) + ) async with app.metadata.transaction(write=False) as tx: - after = tuple(tx.one('SELECT COUNT(*) FROM ' + table)[0] - for table in ('events', 'results', 'credentials', 'certificates')) - return (recovered.status == 'ok' and recovered.subject == subject and - recovered.data['key_id'] == signer.key_id and recovered.data['status'] == 'completed' - and recovered.data['certificate_id'] == committed.data['certificate_id'] - and denied.status == 'error' and denied.error.code == 'credential_revoked' - and before == after) + after = tuple( + tx.one('SELECT COUNT(*) FROM ' + table)[0] + for table in ('events', 'results', 'credentials', 'certificates') + ) + return ( + recovered.status == 'ok' + and recovered.subject == subject + and recovered.data['key_id'] == signer.key_id + and recovered.data['status'] == 'completed' + and recovered.data['certificate_id'] == committed.data['certificate_id'] + and denied.status == 'error' + and denied.error.code == 'credential_revoked' + and before == after + ) diff --git a/src/msg/application.py b/src/msg/application.py index 12450b0d..5af8f78e 100644 --- a/src/msg/application.py +++ b/src/msg/application.py @@ -1,21 +1,21 @@ """Explicit composition root for network services. Contains no root signer.""" + from __future__ import annotations -from datetime import UTC, datetime -from pathlib import Path import hmac +from datetime import UTC, datetime from msg.constants import ROOT_SPACE, ROOT_SUBJECT -from msg.core.codec import loads, decode, unb64 +from msg.core.codec import decode, loads, unb64 from msg.core.cursors import CursorCodec from msg.core.errors import require from msg.core.executor import OperationExecutor from msg.core.models import Certificate, Scope from msg.core.registry import Registry +from msg.plugins import install_registry from msg.security.authentication import AuthenticationService from msg.security.authorization import AuthorizationService -from msg.security.capabilities import primary_ceiling, base_grants, temporary_ceiling -from msg.plugins import install_registry +from msg.security.capabilities import base_grants, primary_ceiling, temporary_ceiling from msg.security.certificates import CertificateValidator from msg.security.crypto import Ed25519Signer from msg.security.quarantine import RuntimeGeneration, active as quarantine_active @@ -24,157 +24,203 @@ class Application: - def __init__(self,settings, *, clock=None, selftest_run_id=None): - self.settings=settings - self.clock=clock or (lambda: datetime.now(UTC)) + def __init__(self, settings, *, clock=None, selftest_run_id=None): + self.settings = settings + self.clock = clock or (lambda: datetime.now(UTC)) # Only the local diagnostic constructs an isolated test namespace. - require(selftest_run_id is None or (isinstance(selftest_run_id,str) and - len(selftest_run_id)==32 and all(c in '0123456789abcdef' for c in selftest_run_id)), - 'invalid_selftest_run_id') - self.selftest_run_id=selftest_run_id - self.namespace_root=ROOT_SPACE if selftest_run_id is None else 't_selftest_'+selftest_run_id - self.registry=Registry() - self.metadata=None - self.contents=None - self.executor=None - self.token_delivery=None - self._loaded=False - self.runtime_generation=None + require( + selftest_run_id is None + or ( + isinstance(selftest_run_id, str) + and len(selftest_run_id) == 32 + and all(c in '0123456789abcdef' for c in selftest_run_id) + ), + 'invalid_selftest_run_id', + ) + self.selftest_run_id = selftest_run_id + self.namespace_root = ( + ROOT_SPACE if selftest_run_id is None else 't_selftest_' + selftest_run_id + ) + self.registry = Registry() + self.metadata = None + self.contents = None + self.executor = None + self.token_delivery = None + self._loaded = False + self.runtime_generation = None install_registry(self, settings.server.plugins) def primary_ceiling(self): - return primary_ceiling(self.registry,self.default_scope()) + return primary_ceiling(self.registry, self.default_scope()) def base_grants(self): - return base_grants(self.registry,self.default_scope()) + return base_grants(self.registry, self.default_scope()) def default_scope(self): - return Scope(resource_id=self.namespace_root,descendants=True) + return Scope(resource_id=self.namespace_root, descendants=True) @property def base_capability_names(self): return frozenset(g.capability for g in self.base_grants()) def temporary_ceiling(self): - return temporary_ceiling(self.registry,self.default_scope()) + return temporary_ceiling(self.registry, self.default_scope()) async def open_storage(self): if self.metadata is None: from msg.storage.postgres import PostgresMetadataStore from msg.storage.valkey_bus import ValkeyOutboxSignal - signal=(ValkeyOutboxSignal(self.settings.server.valkey_url) - if self.settings.server.valkey_url else None) - self.metadata=PostgresMetadataStore(self.settings.server.postgres_dsn,signal=signal) + + signal = ( + ValkeyOutboxSignal(self.settings.server.valkey_url) + if self.settings.server.valkey_url + else None + ) + self.metadata = PostgresMetadataStore(self.settings.server.postgres_dsn, signal=signal) if self.contents is None: - self.contents=GitContentStore(self.settings.server.content_dir, - binary_dir=self.settings.server.blob_dir, - staging_dir=self.settings.server.staging_dir, - group_read=self.settings.hosting_content_group_read) + self.contents = GitContentStore( + self.settings.server.content_dir, + binary_dir=self.settings.server.blob_dir, + staging_dir=self.settings.server.staging_dir, + group_read=self.settings.hosting_content_group_read, + ) async def load(self): - require(self.settings.trust_file.is_file(),'root_not_initialized') + require(self.settings.trust_file.is_file(), 'root_not_initialized') # Pin before reading runtime files, then recheck with validated authority. await self.open_storage() async with self.metadata.transaction(write=False) as tx: if self.runtime_generation is None: - self.runtime_generation=RuntimeGeneration.capture(tx) + self.runtime_generation = RuntimeGeneration.capture(tx) else: self.runtime_generation.require_current(tx) - trust=loads(self.settings.trust_file.read_bytes()) - require(set(trust)=={'version','public_key','certificate'} and trust['version']==1,'invalid_trust_anchor') - root_certificate=decode(Certificate,trust['certificate']) - root_public=unb64(trust['public_key'],limit=32) - keys=self.settings.service_keys - required=('online.key','receipt.key','tokens.key') - require(all((keys/name).is_file() for name in required),'service_keys_missing') - self.online_signer=Ed25519Signer.from_bytes((keys/'online.key').read_bytes()) - self.receipt_signer=Ed25519Signer.from_bytes((keys/'receipt.key').read_bytes()) - self._token_secret=(keys/'tokens.key').read_bytes() - require(len(self._token_secret)==32,'invalid_service_key') - self._vault_key=hmac.digest(self._token_secret,b'custodial-vault-aesgcm-v1','sha256') - self.cursors=CursorCodec(hmac.digest(self._token_secret,b'cursor-key-v1','sha256')) - self.token_delivery=TokenDelivery(metadata=self.metadata,token_secret=self._token_secret, + trust = loads(self.settings.trust_file.read_bytes()) + require( + set(trust) == {'version', 'public_key', 'certificate'} and trust['version'] == 1, + 'invalid_trust_anchor', + ) + root_certificate = decode(Certificate, trust['certificate']) + root_public = unb64(trust['public_key'], limit=32) + keys = self.settings.service_keys + required = ('online.key', 'receipt.key', 'tokens.key') + require(all((keys / name).is_file() for name in required), 'service_keys_missing') + self.online_signer = Ed25519Signer.from_bytes((keys / 'online.key').read_bytes()) + self.receipt_signer = Ed25519Signer.from_bytes((keys / 'receipt.key').read_bytes()) + self._token_secret = (keys / 'tokens.key').read_bytes() + require(len(self._token_secret) == 32, 'invalid_service_key') + self._vault_key = hmac.digest(self._token_secret, b'custodial-vault-aesgcm-v1', 'sha256') + self.cursors = CursorCodec(hmac.digest(self._token_secret, b'cursor-key-v1', 'sha256')) + self.token_delivery = TokenDelivery( + metadata=self.metadata, + token_secret=self._token_secret, clock=lambda: self.clock(), - recovery_window=lambda: self.settings.credential_delivery_recovery_window) - self.certificates=CertificateValidator(self.registry,root_certificate,root_public,self.settings.service_url,self.clock) + recovery_window=lambda: self.settings.credential_delivery_recovery_window, + ) + self.certificates = CertificateValidator( + self.registry, root_certificate, root_public, self.settings.service_url, self.clock + ) async with self.metadata.transaction(write=False) as tx: self.runtime_generation.require_current(tx) - self.token_delivery.runtime_generation=self.runtime_generation - quarantined=quarantine_active(tx) - await self.certificates.validate(root_certificate.resource_id,tx) - root=await tx.subject(ROOT_SUBJECT) - require(root.local_only,'root_policy_corrupt') + self.token_delivery.runtime_generation = self.runtime_generation + quarantined = quarantine_active(tx) + await self.certificates.validate(root_certificate.resource_id, tx) + root = await tx.subject(ROOT_SUBJECT) + require(root.local_only, 'root_policy_corrupt') # Only a verified installation may import immutable release-owned rules. # This is idempotent per source digest/version and rejects source deletion. from msg.bootstrap import sync_system_sources - marker=self.settings.config_dir/'recovery-drill.json' - quarantined=quarantined or marker.exists() or marker.is_symlink() + + marker = self.settings.config_dir / 'recovery-drill.json' + quarantined = quarantined or marker.exists() or marker.is_symlink() if not quarantined: async with self.metadata.transaction(write=True) as tx: # Recheck under the writer lock; loading a restore must not # mutate release resources before recovery has been accepted. self.runtime_generation.require_current(tx) if not quarantine_active(tx): - await sync_system_sources(tx,self.contents,self.clock(),namespace_root=self.namespace_root, - registry=self.registry) + await sync_system_sources( + tx, + self.contents, + self.clock(), + namespace_root=self.namespace_root, + registry=self.registry, + ) else: - quarantined=True - self.authenticator=AuthenticationService(self.registry,self.certificates,self.settings.service_url,self.clock, - self.primary_ceiling,self.temporary_ceiling) - self.authorizer=AuthorizationService(self.registry,self.certificates) - self.authenticator.runtime_generation=self.runtime_generation - self.authorizer.runtime_generation=self.runtime_generation - self.executor=self.new_executor(self.authenticator) - self.executor.recovery_drill_marker=marker - self.executor.recovery_quarantined=quarantined - self._loaded=True + quarantined = True + self.authenticator = AuthenticationService( + self.registry, + self.certificates, + self.settings.service_url, + self.clock, + self.primary_ceiling, + self.temporary_ceiling, + ) + self.authorizer = AuthorizationService(self.registry, self.certificates) + self.authenticator.runtime_generation = self.runtime_generation + self.authorizer.runtime_generation = self.runtime_generation + self.executor = self.new_executor(self.authenticator) + self.executor.recovery_drill_marker = marker + self.executor.recovery_quarantined = quarantined + self._loaded = True return self def new_executor(self, authenticator=None): """Compose identical limits/ports for HTTP and the restricted SSH entry.""" - executor=OperationExecutor(self.registry,self.metadata,self.contents, + executor = OperationExecutor( + self.registry, + self.metadata, + self.contents, self.authenticator if authenticator is None else authenticator, - self.authorizer,self.clock,self.receipt_signer, + self.authorizer, + self.clock, + self.receipt_signer, max_request_bytes=self.settings.server.limits.max_request_bytes, result_projection=self._result_projection, - event_notifications=self._event_notifications) - executor.runtime_generation=self.runtime_generation - executor.response_hook=self.token_delivery.release - executor.recovery_drill_marker=self.settings.config_dir/'recovery-drill.json' + event_notifications=self._event_notifications, + ) + executor.runtime_generation = self.runtime_generation + executor.response_hook = self.token_delivery.release + executor.recovery_drill_marker = self.settings.config_dir / 'recovery-drill.json' if self.executor is not None: - executor.recovery_quarantined=self.executor.recovery_quarantined + executor.recovery_quarantined = self.executor.recovery_quarantined return executor async def _result_projection(self, context, request, session, resource, *, fields): from msg.plugins.discovery import read_projection - return await read_projection(self,context,request,session,resource.id, - revision=resource.revision,fields=fields) + + return await read_projection( + self, context, request, session, resource.id, revision=resource.revision, fields=fields + ) async def _event_notifications(self, session, event): if 'communication' not in self.settings.server.plugins: return from msg.plugins.communication import enqueue_domain_webhooks - await enqueue_domain_webhooks(self,session,event) + + await enqueue_domain_webhooks(self, session, event) from msg.plugins.watches import enqueue - await enqueue(self,session,event) - async def online_issuer(self,tx): - issuer=tx.setting('online_ca_certificate') - require(issuer is not None,'issuer_not_ready') - cert=await self.certificates.validate(issuer,tx) - require(cert.kind=='ca' and cert.key_id==self.online_signer.key_id,'issuer_not_ready') + await enqueue(self, session, event) + + async def online_issuer(self, tx): + issuer = tx.setting('online_ca_certificate') + require(issuer is not None, 'issuer_not_ready') + cert = await self.certificates.validate(issuer, tx) + require(cert.kind == 'ca' and cert.key_id == self.online_signer.key_id, 'issuer_not_ready') return cert - def issued_token(self,request,subject): - return self.token_delivery.issued_token(request,subject) + def issued_token(self, request, subject): + return self.token_delivery.issued_token(request, subject) - recovery_verifier=staticmethod(recovery_verifier) + recovery_verifier = staticmethod(recovery_verifier) - def record_token_delivery(self,tx,request,credential,now,*,recovery_deadline=None): - return self.token_delivery.record_token_delivery(tx,request,credential,now,recovery_deadline=recovery_deadline) + def record_token_delivery(self, tx, request, credential, now, *, recovery_deadline=None): + return self.token_delivery.record_token_delivery( + tx, request, credential, now, recovery_deadline=recovery_deadline + ) - async def _secrets_for_caller(self,request,result): - return await self.token_delivery.release(request,result) + async def _secrets_for_caller(self, request, result): + return await self.token_delivery.release(request, result) async def close(self): if self.metadata is not None: diff --git a/src/msg/atomic_file.py b/src/msg/atomic_file.py index beaf7034..980510fd 100644 --- a/src/msg/atomic_file.py +++ b/src/msg/atomic_file.py @@ -1,22 +1,23 @@ """Atomic local-file replacement with caller-selected mode and durable syncs.""" + from __future__ import annotations import os -from pathlib import Path import tempfile +from pathlib import Path def durable_write(path: Path, data: bytes, mode: int = 0o600): - path.parent.mkdir(parents=True,exist_ok=True) - fd,temporary=tempfile.mkstemp(prefix='.pending-',dir=path.parent) + path.parent.mkdir(parents=True, exist_ok=True) + fd, temporary = tempfile.mkstemp(prefix='.pending-', dir=path.parent) try: - os.fchmod(fd,mode) - with os.fdopen(fd,'wb') as stream: + os.fchmod(fd, mode) + with os.fdopen(fd, 'wb') as stream: stream.write(data) stream.flush() os.fsync(stream.fileno()) - os.replace(temporary,path) - directory=os.open(path.parent,os.O_RDONLY|os.O_DIRECTORY) + os.replace(temporary, path) + directory = os.open(path.parent, os.O_RDONLY | os.O_DIRECTORY) try: os.fsync(directory) finally: diff --git a/src/msg/bootstrap.py b/src/msg/bootstrap.py index 74e2caa3..241a9c9d 100644 --- a/src/msg/bootstrap.py +++ b/src/msg/bootstrap.py @@ -1,235 +1,442 @@ """Versioned public bootstrap. Private-key provisioning lives in admin only.""" + from __future__ import annotations + +import re from dataclasses import replace from importlib.resources import files from pathlib import Path -import re -from msg.constants import * -from msg.core.codec import loads,wire,canonical,digest,decode -from msg.core.errors import require -from msg.core.models import Resource,ResourceRef,Revision,Subject,Organization,Membership +from msg.constants import ( + ADMINS_GROUP as ADMINS_GROUP, + CA_SPACE as CA_SPACE, + CERT_SPACE as CERT_SPACE, + CSR_SPACE as CSR_SPACE, + ONLINE_CA as ONLINE_CA, + PROTOCOL_VERSION as PROTOCOL_VERSION, + PUBLIC_GROUP as PUBLIC_GROUP, + ROOT_SPACE as ROOT_SPACE, + ROOT_SUBJECT as ROOT_SUBJECT, + TOOLS_SPACE as TOOLS_SPACE, +) +from msg.core.codec import canonical, decode, digest, loads, wire +from msg.core.errors import require +from msg.core.models import Organization, Resource, Revision, Subject -RULE_NAMES=('identity','read-write','auth','topics','files','recovery','security','protocol') -SOURCE_HEADER=re.compile(r'\n') -REQUIRES_HEADER=re.compile(r'') -ROOT_WEB_LOGO=files('msg.data').joinpath('logo.svg').read_text(encoding='utf-8') -ROOT_WEB_SAMPLE=('' +RULE_NAMES = ( + 'identity', + 'read-write', + 'auth', + 'topics', + 'files', + 'recovery', + 'security', + 'protocol', +) +SOURCE_HEADER = re.compile(r'\n') +REQUIRES_HEADER = re.compile(r'') +ROOT_WEB_LOGO = files('msg.data').joinpath('logo.svg').read_text(encoding='utf-8') +ROOT_WEB_SAMPLE = ( + '' '' 'msg.lmm.best' - '
'+ROOT_WEB_LOGO+'

msg.lmm.best

' + '
' + ROOT_WEB_LOGO + '

msg.lmm.best

' '

让 Agent 和人清楚地交流、分享与继续工作。

' '

发布、回复、私聊、交换文件；公开和私密由你决定。

' - '

Agent 入口

\n').encode('utf-8') + '

Agent 入口

\n' +).encode('utf-8') # Public identity and location are deliberately independent of the release file. # A release can change a source path only with an explicit migration declaration. -RULE_SPECS=(('msg.bootstrap','r_agents',ROOT_SPACE,'AGENTS.md','AGENTS.md',1000), - ('msg.rules.index','r_rule_index','r_rules','_index.md','rules/_index.md',2400), - *((f'msg.{name}',f'r_rule_{name.replace("-","_")}','r_rules', - name,f'rules/{name}.md',4000) for name in RULE_NAMES)) -RULE_PATHS={rule_id:('/AGENTS.md' if rid=='r_agents' else - '/_rules/_index.md' if rid=='r_rule_index' else '/_rules/'+name) - for rule_id,rid,_,name,_,_ in RULE_SPECS} +RULE_SPECS = ( + ('msg.bootstrap', 'r_agents', ROOT_SPACE, 'AGENTS.md', 'AGENTS.md', 1000), + ('msg.rules.index', 'r_rule_index', 'r_rules', '_index.md', 'rules/_index.md', 2400), + *( + ( + f'msg.{name}', + f'r_rule_{name.replace("-", "_")}', + 'r_rules', + name, + f'rules/{name}.md', + 4000, + ) + for name in RULE_NAMES + ), +) +RULE_PATHS = { + rule_id: ( + '/AGENTS.md' + if rid == 'r_agents' + else '/_rules/_index.md' + if rid == 'r_rule_index' + else '/_rules/' + name + ) + for rule_id, rid, _, name, _, _ in RULE_SPECS +} # Release maintainers declare a relocation in both maps in the same change. -SOURCE_PATH_OVERRIDES={} -SOURCE_MIGRATIONS={} +SOURCE_PATH_OVERRIDES = {} +SOURCE_MIGRATIONS = {} # Release-only pins: rule_id -> {source_path: relative .md path, version: int, # digest: sha256:}. Remove active links/operation dependencies in the same # release; keep RULE_SPECS identity and this pin so upgrades preserve history. # Retirement archives existing resources; fresh installs invent no old revision. # No current rule is retired. A pin mismatch requires an explicit staged upgrade. -SOURCE_RETIREMENTS={} +SOURCE_RETIREMENTS = {} def system_source_root(): - packaged=files('msg.data').joinpath('system') - require(packaged.is_dir(),'system_source_missing') + packaged = files('msg.data').joinpath('system') + require(packaged.is_dir(), 'system_source_missing') return packaged -async def sync_system_sources(tx,contents,now, *, source_root=None,namespace_root=ROOT_SPACE, - source_paths=None,migrations=None,retirements=None,registry=None): +async def sync_system_sources( + tx, + contents, + now, + *, + source_root=None, + namespace_root=ROOT_SPACE, + source_paths=None, + migrations=None, + retirements=None, + registry=None, +): """Sync each release-owned document independently; never delete a source implicitly.""" - root=Path(source_root or system_source_root()) - source_paths=dict(SOURCE_PATH_OVERRIDES if source_paths is None else source_paths) - migrations=dict(SOURCE_MIGRATIONS if migrations is None else migrations) - require(set(source_paths)<=RULE_PATHS.keys() and set(migrations)<=RULE_PATHS.keys(), - 'unknown_rule_id') - retirements=dict(SOURCE_RETIREMENTS if retirements is None else retirements) - require(set(retirements)<=RULE_PATHS.keys()-{'msg.bootstrap','msg.rules.index'}, - 'system_source_invalid_retirement') - require(not set(retirements)&(source_paths.keys()|migrations.keys()), - 'system_source_invalid_retirement') - for rule_id,pin in retirements.items(): - require(isinstance(pin,dict) and set(pin)=={'source_path','version','digest'} and - isinstance(pin['source_path'],str) and - pin['source_path']==Path(pin['source_path']).as_posix() and - not Path(pin['source_path']).is_absolute() and - '..' not in Path(pin['source_path']).parts and - pin['source_path'].endswith('.md') and - type(pin['version']) is int and pin['version']>0 and - isinstance(pin['digest'],str) and - re.fullmatch(r'sha256:[0-9a-f]{64}',pin['digest']) is not None, - 'system_source_invalid_retirement',rule_id) + root = Path(source_root or system_source_root()) + source_paths = dict(SOURCE_PATH_OVERRIDES if source_paths is None else source_paths) + migrations = dict(SOURCE_MIGRATIONS if migrations is None else migrations) + require( + set(source_paths) <= RULE_PATHS.keys() and set(migrations) <= RULE_PATHS.keys(), + 'unknown_rule_id', + ) + retirements = dict(SOURCE_RETIREMENTS if retirements is None else retirements) + require( + set(retirements) <= RULE_PATHS.keys() - {'msg.bootstrap', 'msg.rules.index'}, + 'system_source_invalid_retirement', + ) + require( + not set(retirements) & (source_paths.keys() | migrations.keys()), + 'system_source_invalid_retirement', + ) + for rule_id, pin in retirements.items(): + require( + isinstance(pin, dict) + and set(pin) == {'source_path', 'version', 'digest'} + and isinstance(pin['source_path'], str) + and pin['source_path'] == Path(pin['source_path']).as_posix() + and not Path(pin['source_path']).is_absolute() + and '..' not in Path(pin['source_path']).parts + and pin['source_path'].endswith('.md') + and type(pin['version']) is int + and pin['version'] > 0 + and isinstance(pin['digest'], str) + and re.fullmatch(r'sha256:[0-9a-f]{64}', pin['digest']) is not None, + 'system_source_invalid_retirement', + rule_id, + ) if retirements: - require(registry is not None and all(not set(spec.requires_rules)&retirements.keys() - for spec in registry.operations()),'system_source_retirement_referenced') - specs=[] - for rule_id,rid,parent,name,default_path,max_bytes in RULE_SPECS: + require( + registry is not None + and all( + not set(spec.requires_rules) & retirements.keys() for spec in registry.operations() + ), + 'system_source_retirement_referenced', + ) + specs = [] + for rule_id, rid, parent, name, default_path, max_bytes in RULE_SPECS: if rule_id in retirements: continue - relative=source_paths.get(rule_id,default_path) - require(isinstance(relative,str) and relative.endswith('.md') and - relative==Path(relative).as_posix() and - not Path(relative).is_absolute() and '..' not in Path(relative).parts, - 'system_source_invalid_path',rule_id) - specs.append((relative,rid,namespace_root if parent==ROOT_SPACE else parent, - name,rule_id,max_bytes)) - require(len({s[0] for s in specs})==len(specs),'duplicate_system_source_path') - expected={s[0] for s in specs} - actual={p.relative_to(root).as_posix() for p in root.rglob('*.md')} + relative = source_paths.get(rule_id, default_path) + require( + isinstance(relative, str) + and relative.endswith('.md') + and relative == Path(relative).as_posix() + and not Path(relative).is_absolute() + and '..' not in Path(relative).parts, + 'system_source_invalid_path', + rule_id, + ) + specs.append(( + relative, + rid, + namespace_root if parent == ROOT_SPACE else parent, + name, + rule_id, + max_bytes, + )) + require(len({s[0] for s in specs}) == len(specs), 'duplicate_system_source_path') + expected = {s[0] for s in specs} + actual = {p.relative_to(root).as_posix() for p in root.rglob('*.md')} for relative in sorted(actual): - candidate=root/relative - require(not candidate.is_symlink() and candidate.resolve().is_relative_to(root.resolve()), - 'system_source_invalid_path',relative) - observed={} + candidate = root / relative + require( + not candidate.is_symlink() and candidate.resolve().is_relative_to(root.resolve()), + 'system_source_invalid_path', + relative, + ) + observed = {} for relative in sorted(actual): - match=SOURCE_HEADER.match((root/relative).read_text(encoding='utf-8')) - require(match is not None,'system_source_invalid_header',relative) - rule_id=match.group(1) - require(rule_id in RULE_PATHS,'unknown_rule_id',relative) - require(rule_id not in observed,'duplicate_rule_id',rule_id) - require(rule_id not in retirements,'system_source_retirement_source_present',rule_id) - observed[rule_id]=relative - require(not expected-actual,'system_source_deleted_requires_migration', - details={'paths':sorted(expected-actual)}) - require(not actual-expected,'system_source_inventory_mismatch', - details={'unknown':sorted(actual-expected)}) - prepared=[] - rule_ids=set() - for relative,rid,parent,name,expected_rule_id,max_bytes in specs: - path=root/relative - require(path.is_file() and not path.is_symlink(),'system_source_missing',relative) - raw=path.read_bytes() - require(len(raw)<=max_bytes,'system_source_too_large',relative) + match = SOURCE_HEADER.match((root / relative).read_text(encoding='utf-8')) + require(match is not None, 'system_source_invalid_header', relative) + rule_id = match.group(1) + require(rule_id in RULE_PATHS, 'unknown_rule_id', relative) + require(rule_id not in observed, 'duplicate_rule_id', rule_id) + require(rule_id not in retirements, 'system_source_retirement_source_present', rule_id) + observed[rule_id] = relative + require( + not expected - actual, + 'system_source_deleted_requires_migration', + details={'paths': sorted(expected - actual)}, + ) + require( + not actual - expected, + 'system_source_inventory_mismatch', + details={'unknown': sorted(actual - expected)}, + ) + prepared = [] + rule_ids = set() + for relative, rid, parent, name, expected_rule_id, max_bytes in specs: + path = root / relative + require(path.is_file() and not path.is_symlink(), 'system_source_missing', relative) + raw = path.read_bytes() + require(len(raw) <= max_bytes, 'system_source_too_large', relative) try: - source_text=raw.decode('utf-8') + source_text = raw.decode('utf-8') except UnicodeError as exc: raise ValueError('system_source_invalid_utf8') from exc - header=SOURCE_HEADER.match(source_text) - require(header is not None,'system_source_invalid_header',relative) - rule_id,version=header.group(1),int(header.group(2)) - require(rule_id in RULE_PATHS and rule_id==expected_rule_id,'unknown_rule_id',relative) - require(rule_id not in rule_ids,'duplicate_rule_id',rule_id) + header = SOURCE_HEADER.match(source_text) + require(header is not None, 'system_source_invalid_header', relative) + rule_id, version = header.group(1), int(header.group(2)) + require(rule_id in RULE_PATHS and rule_id == expected_rule_id, 'unknown_rule_id', relative) + require(rule_id not in rule_ids, 'duplicate_rule_id', rule_id) rule_ids.add(rule_id) - note_match=re.search(r'',source_text) - change_note=note_match.group(1) if note_match else None - required=REQUIRES_HEADER.findall(source_text) - require(len(required)==source_text.count('', source_text) + change_note = note_match.group(1) if note_match else None + required = REQUIRES_HEADER.findall(source_text) + require( + len(required) == source_text.count('\nA new sentence.\n') + first = (await tx.resource(rid)).revision + source = await tx.revision( + __import__('msg.core.models', fromlist=['ResourceRef']).ResourceRef(id=rid) + ) + assert (source.source_kind, source.source_version, source.source_digest) == ( + 'release', + 1, + source.content.digest, + ) + copied = tmp_path / 'system' + shutil.copytree(system_source_root(), copied) + path = copied / 'rules' / 'identity.md' + path.write_text( + path.read_text().replace('version: 1', 'version: 2') + + '\n\nA new sentence.\n' + ) async with app.metadata.transaction(write=True) as tx: - await sync_system_sources(tx,app.contents,NOW,source_root=copied) - history=await call(app,'discovery.get',{'id':rid,'view':'history','limit':1}) - assert history.status=='ok' and history.data['cursor'] - older=await call(app,'discovery.get',{'id':rid,'view':'history','limit':1, - 'cursor':history.data['cursor']}) - records=(*history.data['revisions'],*older.data['revisions']) - assert {item['source_version'] for item in records}=={1,2} - assert any(item.get('change_note')=='Clarified key ownership.' for item in records) - current=(await call(app,'discovery.get',{'id':rid})).data['revision'] - diff=await call(app,'discovery.diff_view',{'id':rid,'known_revision':first}) - assert diff.status=='ok' and diff.data['from']['revision']==first - assert diff.data['to']['revision']==current and '+A new sentence.' in diff.data['diff'] - assert diff.data['to_source']['source_version']==2 + await sync_system_sources(tx, app.contents, NOW, source_root=copied) + history = await call(app, 'discovery.get', {'id': rid, 'view': 'history', 'limit': 1}) + assert history.status == 'ok' and history.data['cursor'] + older = await call( + app, + 'discovery.get', + {'id': rid, 'view': 'history', 'limit': 1, 'cursor': history.data['cursor']}, + ) + records = (*history.data['revisions'], *older.data['revisions']) + assert {item['source_version'] for item in records} == {1, 2} + assert any(item.get('change_note') == 'Clarified key ownership.' for item in records) + current = (await call(app, 'discovery.get', {'id': rid})).data['revision'] + diff = await call(app, 'discovery.diff_view', {'id': rid, 'known_revision': first}) + assert diff.status == 'ok' and diff.data['from']['revision'] == first + assert diff.data['to']['revision'] == current and '+A new sentence.' in diff.data['diff'] + assert diff.data['to_source']['source_version'] == 2 @pytest.mark.asyncio async def test_user_cannot_forge_release_source_and_rule_links_resolve(installed): app, _ = installed - key,subject,_=await register(app,'source-user') - forged=await call(app,'content.post_create',{'parent':'/main','body':'a post', - 'source_kind':'release','source_version':999},key=key,subject=subject) - assert forged.status=='error' and forged.error.code=='schema_validation' + key, subject, _ = await register(app, 'source-user') + forged = await call( + app, + 'content.post_create', + {'parent': '/main', 'body': 'a post', 'source_kind': 'release', 'source_version': 999}, + key=key, + subject=subject, + ) + assert forged.status == 'error' and forged.error.code == 'schema_validation' for operation in app.registry.operations('network'): - described=app.registry.describe(operation) - links=described['requires_rules'] + described = app.registry.describe(operation) + links = described['requires_rules'] assert links for link in links: assert link['rule_id'].startswith('msg.') and link['path'].startswith('/_rules/') - rule=await call(app,'discovery.get',{'id':link['path']}) - assert rule.status=='ok' and link['rule_id'] in rule.data['content'] - schema=await call(app,'discovery.schema',{'operation':'identity.register'}) - assert schema.status=='ok' and schema.data['operation']['requires_rules'] + rule = await call(app, 'discovery.get', {'id': link['path']}) + assert rule.status == 'ok' and link['rule_id'] in rule.data['content'] + schema = await call(app, 'discovery.schema', {'operation': 'identity.register'}) + assert schema.status == 'ok' and schema.data['operation']['requires_rules'] @pytest.mark.asyncio async def test_history_cursor_rechecks_principal_and_current_visibility(installed): app, _ = installed - key,subject,_=await register(app,'history-owner') - created=await call(app,'content.post_create',{'parent':'/main','body':'first'}, - key=key,subject=subject) - rid=created.resources[0].id - edited=await call(app,'content.post_edit',{'id':rid, - 'expected_revision':created.resources[0].revision,'body':'second'}, - key=key,subject=subject,expected=((rid,created.data['generation']),)) - assert edited.status=='ok' - page=await call(app,'discovery.get',{'id':rid,'view':'history','limit':1}) - assert page.status=='ok' and page.data['cursor'] - wrong=await call(app,'discovery.get',{'id':rid,'view':'history','limit':1, - 'cursor':page.data['cursor']},key=key,subject=subject) - assert wrong.status=='error' and wrong.error.code=='cursor_principal_mismatch' - hidden=await call(app,'content.chmod',{'id':rid,'mode':'0600'},key=key,subject=subject, - expected=((rid,edited.data['generation']),)) - assert hidden.status=='ok' - stale=await call(app,'discovery.get',{'id':rid,'view':'history','limit':1, - 'cursor':page.data['cursor']}) - assert stale.status=='error' + key, subject, _ = await register(app, 'history-owner') + created = await call( + app, 'content.post_create', {'parent': '/main', 'body': 'first'}, key=key, subject=subject + ) + rid = created.resources[0].id + edited = await call( + app, + 'content.post_edit', + {'id': rid, 'expected_revision': created.resources[0].revision, 'body': 'second'}, + key=key, + subject=subject, + expected=((rid, created.data['generation']),), + ) + assert edited.status == 'ok' + page = await call(app, 'discovery.get', {'id': rid, 'view': 'history', 'limit': 1}) + assert page.status == 'ok' and page.data['cursor'] + wrong = await call( + app, + 'discovery.get', + {'id': rid, 'view': 'history', 'limit': 1, 'cursor': page.data['cursor']}, + key=key, + subject=subject, + ) + assert wrong.status == 'error' and wrong.error.code == 'cursor_principal_mismatch' + hidden = await call( + app, + 'content.chmod', + {'id': rid, 'mode': '0600'}, + key=key, + subject=subject, + expected=((rid, edited.data['generation']),), + ) + assert hidden.status == 'ok' + stale = await call( + app, + 'discovery.get', + {'id': rid, 'view': 'history', 'limit': 1, 'cursor': page.data['cursor']}, + ) + assert stale.status == 'error' diff --git a/tests/test_root_rotation.py b/tests/test_root_rotation.py index ac15e34b..ddd88f31 100644 --- a/tests/test_root_rotation.py +++ b/tests/test_root_rotation.py @@ -1,37 +1,49 @@ import pytest -from msg.admin.rotation import prepare,complete,journal_path -from msg.security.crypto import Ed25519Signer,open_private_key -from msg.core.codec import loads,wire +from test_service import NOW, call, register + +from msg.admin.root import _approve_csr, root_envelope +from msg.admin.rotation import complete, journal_path, prepare from msg.application import Application -from msg.admin.root import _approve_csr,root_envelope -from test_service import call,register,NOW +from msg.core.codec import loads, wire +from msg.security.crypto import Ed25519Signer, open_private_key + @pytest.mark.asyncio async def test_root_rotation_retires_old_chain_and_requires_explicit_online_ca_approval(installed): - app,root=installed - key,uid,cert=await register(app,'rotation-agent') - next_key=Ed25519Signer.generate() - pin='new-root-long-passphrase' - journal=prepare(app,next_key,pin,old_signer=root,operator='isolated-test-console') - assert journal['new_trust']['certificate']['issuance']['max_child_ca_depth']==3 - assert journal['online_csr']['issuance']['max_child_ca_depth']==0 + app, root = installed + key, uid, cert = await register(app, 'rotation-agent') + next_key = Ed25519Signer.generate() + pin = 'new-root-long-passphrase' + journal = prepare(app, next_key, pin, old_signer=root, operator='isolated-test-console') + assert journal['new_trust']['certificate']['issuance']['max_child_ca_depth'] == 3 + assert journal['online_csr']['issuance']['max_child_ca_depth'] == 0 assert journal_path(app).exists() assert next_key.private_bytes() not in journal_path(app).read_bytes() - outcome=await complete(app,journal,pin=pin) + outcome = await complete(app, journal, pin=pin) assert outcome['restart_required'] and not journal_path(app).exists() - stale=await call(app,'discovery.get',{'id':'/main'}) - assert stale.error.code=='service_restart_required' - fresh=Application(app.settings,clock=lambda:NOW);await fresh.load() + stale = await call(app, 'discovery.get', {'id': '/main'}) + assert stale.error.code == 'service_restart_required' + fresh = Application(app.settings, clock=lambda: NOW) + await fresh.load() try: - denied=await call(fresh,'discovery.get',{'id':'/main'},key=key,subject=uid,certs=(cert,)) - assert denied.error.code=='certificate_revoked' - renewal=await call(fresh,'identity.certificate_renew',{},key=key,subject=uid) - assert renewal.error.code=='issuer_not_ready' - await _approve_csr(fresh,outcome['online_ca_request'],next_key,expected_digest=None,operator='isolated-test-console') - renewal=await call(fresh,'identity.certificate_renew',{},key=key,subject=uid) - assert renewal.error.code=='renewal_source_required',wire(renewal) - assert fresh.certificates.root_public_key==next_key.public_key - assert not (app.settings.config_dir/'root').exists() - envelope=loads(root_envelope(app.settings.config_dir).read_bytes()) - assert open_private_key(envelope,pin)==next_key.private_bytes() - finally:await fresh.close() + denied = await call( + fresh, 'discovery.get', {'id': '/main'}, key=key, subject=uid, certs=(cert,) + ) + assert denied.error.code == 'certificate_revoked' + renewal = await call(fresh, 'identity.certificate_renew', {}, key=key, subject=uid) + assert renewal.error.code == 'issuer_not_ready' + await _approve_csr( + fresh, + outcome['online_ca_request'], + next_key, + expected_digest=None, + operator='isolated-test-console', + ) + renewal = await call(fresh, 'identity.certificate_renew', {}, key=key, subject=uid) + assert renewal.error.code == 'renewal_source_required', wire(renewal) + assert fresh.certificates.root_public_key == next_key.public_key + assert not (app.settings.config_dir / 'root').exists() + envelope = loads(root_envelope(app.settings.config_dir).read_bytes()) + assert open_private_key(envelope, pin) == next_key.private_bytes() + finally: + await fresh.close() diff --git a/tests/test_root_rotation_resume.py b/tests/test_root_rotation_resume.py index e7f6e68f..02904c20 100644 --- a/tests/test_root_rotation_resume.py +++ b/tests/test_root_rotation_resume.py @@ -1,16 +1,17 @@ """Use real isolated PostgreSQL; local use cases do not simulate a physical console.""" + from copy import deepcopy import pytest +from test_service import NOW from msg.admin.root import root_envelope -from msg.admin.rotation import prepare, complete, journal_path +from msg.admin.rotation import complete, journal_path, prepare from msg.application import Application from msg.core.codec import canonical, loads from msg.core.errors import Failure -from msg.security.crypto import Ed25519Signer, seal_private_key, open_private_key +from msg.security.crypto import Ed25519Signer, open_private_key, seal_private_key from msg.storage.git import durable_write -from test_service import NOW OLD_PIN = 'old-protected-envelope-test-passphrase' NEW_PIN = 'new-protected-envelope-test-passphrase' @@ -25,8 +26,11 @@ def old_envelope(app, root): @pytest.mark.asyncio @pytest.mark.parametrize('failure_at', ['key', 'trust']) -async def test_rotation_resumes_without_archiving_new_key_as_old(installed, monkeypatch, failure_at): +async def test_rotation_resumes_without_archiving_new_key_as_old( + installed, monkeypatch, failure_at +): from msg.admin import rotation + app, root = installed path, original_key = old_envelope(app, root) successor = Ed25519Signer.generate() @@ -35,32 +39,37 @@ async def test_rotation_resumes_without_archiving_new_key_as_old(installed, monk assert journal['version'] == 2 and pending.parent.stat().st_mode & 0o777 == 0o700 fail_path = path if failure_at == 'key' else app.settings.trust_file real_write = rotation.durable_write + def interrupted(target, data, **kwargs): if target == fail_path: raise OSError('injected protected-file interruption') return real_write(target, data, **kwargs) + with monkeypatch.context() as patch: patch.setattr(rotation, 'durable_write', interrupted) with pytest.raises(OSError, match='injected'): await complete(app, journal, pin=NEW_PIN) assert pending.exists() async with app.metadata.transaction(write=False) as tx: - assert tx.setting('active_root_certificate') == journal['new_trust']['certificate']['resource_id'] + assert ( + tx.setting('active_root_certificate') + == journal['new_trust']['certificate']['resource_id'] + ) assert await tx.certificate_revoked(journal['old_certificate']['resource_id']) before = tx.one('SELECT COUNT(*) FROM audit')[0] # Resume must work before Application.load can validate partially switched trust files. - fresh = Application(app.settings, clock=lambda:NOW) + fresh = Application(app.settings, clock=lambda: NOW) try: result = await complete(fresh, loads(pending.read_bytes()), pin=NEW_PIN) assert result['restart_required'] and not pending.exists() assert root_envelope(app.settings.config_dir) == path assert open_private_key(loads(path.read_bytes()), NEW_PIN) == successor.private_bytes() - history = path.parent/'history'/journal['old_certificate']['resource_id']/'key.json' + history = path.parent / 'history' / journal['old_certificate']['resource_id'] / 'key.json' assert history.read_bytes() == original_key assert open_private_key(loads(history.read_bytes()), OLD_PIN) == root.private_bytes() assert path.stat().st_mode & 0o777 == history.stat().st_mode & 0o777 == 0o600 assert history.parent.stat().st_mode & 0o777 == 0o700 - assert not (app.settings.config_dir/'root').exists() + assert not (app.settings.config_dir / 'root').exists() async with fresh.metadata.transaction(write=False) as tx: assert tx.one('SELECT COUNT(*) FROM audit')[0] == before await fresh.load() @@ -73,7 +82,9 @@ def interrupted(target, data, **kwargs): async def test_replacement_csr_is_rejected_before_any_commit(installed): app, root = installed path, original = old_envelope(app, root) - journal = prepare(app, Ed25519Signer.generate(), NEW_PIN, old_signer=root, operator='isolated-test') + journal = prepare( + app, Ed25519Signer.generate(), NEW_PIN, old_signer=root, operator='isolated-test' + ) changed = deepcopy(journal) changed['online_csr']['resource_id'] = 'csr_replaced_after_confirmation' with pytest.raises(Failure, match='invalid_signature'): @@ -93,7 +104,9 @@ async def test_explicit_lost_root_key_rotation_uses_bound_journal(installed): assert journal['statement']['previous_key_proved'] is False and journal['old_signature'] is None await complete(app, journal, pin=NEW_PIN) assert open_private_key(loads(path.read_bytes()), NEW_PIN) == successor.private_bytes() - assert (path.parent/'history'/journal['old_certificate']['resource_id']/'key.json').read_bytes() == original + assert ( + path.parent / 'history' / journal['old_certificate']['resource_id'] / 'key.json' + ).read_bytes() == original @pytest.mark.asyncio @@ -101,9 +114,12 @@ async def test_rotation_back_to_retired_root_key_is_rejected_before_commit(insta app, root = installed old_envelope(app, root) successor = Ed25519Signer.generate() - await complete(app, prepare(app, successor, NEW_PIN, old_signer=root, operator='isolated-test'), - pin=NEW_PIN) - fresh = Application(app.settings, clock=lambda:NOW) + await complete( + app, + prepare(app, successor, NEW_PIN, old_signer=root, operator='isolated-test'), + pin=NEW_PIN, + ) + fresh = Application(app.settings, clock=lambda: NOW) await fresh.load() try: path = root_envelope(app.settings.config_dir) @@ -114,7 +130,9 @@ async def test_rotation_back_to_retired_root_key_is_rejected_before_commit(insta assert journal_path(app).exists() assert path.read_bytes() == installed_key and app.settings.trust_file.read_bytes() == trust async with fresh.metadata.transaction(write=False) as tx: - assert tx.setting('active_root_certificate') == journal['old_certificate']['resource_id'] + assert ( + tx.setting('active_root_certificate') == journal['old_certificate']['resource_id'] + ) assert (await tx.credential(root.key_id)).revoked_at is not None assert (await tx.credential(successor.key_id)).revoked_at is None finally: diff --git a/tests/test_route_effect_matrix.py b/tests/test_route_effect_matrix.py index 3493e43d..6e4fefd2 100644 --- a/tests/test_route_effect_matrix.py +++ b/tests/test_route_effect_matrix.py @@ -1,119 +1,166 @@ """Executable ordinary-route matrix: reads and rejected writes preserve state.""" + from datetime import timedelta import httpx import pytest +from test_service import NOW, call, register from msg.core.codec import b64, canonical, wire from msg.core.requests import request_for from msg.transports.http import RouteEffect, classify_route, create_app -from test_service import NOW, call, register async def database_snapshot(app): """Compare complete rows, so a read counter or an updated ACK cannot hide.""" async with app.metadata.transaction(write=False) as tx: - tables=[row[0] for row in tx.rows( - "SELECT tablename FROM pg_tables WHERE schemaname='public' " - "AND tablename<>'schema_version' ORDER BY tablename")] - return {table:sorted(repr(tuple(row)) for row in tx.rows( - f'SELECT * FROM {table}')) for table in tables} + tables = [ + row[0] + for row in tx.rows( + "SELECT tablename FROM pg_tables WHERE schemaname='public' " + "AND tablename<>'schema_version' ORDER BY tablename" + ) + ] + return { + table: sorted(repr(tuple(row)) for row in tx.rows(f'SELECT * FROM {table}')) + for table in tables + } @pytest.mark.asyncio async def test_ordinary_read_and_failed_route_matrix_preserves_all_authority_rows(installed): - app,_=installed - key,subject,_=await register(app,'route-effect-owner') - post=await call(app,'content.post_create', - {'parent':'/main','name':'effect-probe','body':'route effect marker'}, - key=key,subject=subject) - repo=await call(app,'git.create',{'parent':'/@route-effect-owner','name':'code.git'}, - key=key,subject=subject) - assert post.status==repo.status=='ok' - rid=post.resources[0].id - repo_id=repo.resources[0].id - read_repo=repo.data['read_url'].removeprefix(app.settings.service_url.rstrip('/')) - write=request_for('content.post_create', - {'parent':'/main','body':'must never be written by an ordinary path'}, - app.settings.service_url,signer=key,subject=subject, - request_id='ordinary-route-matrix',expires_at=NOW+timedelta(seconds=90)) - encoded=b64(canonical(write)) - before=await database_snapshot(app) - async with httpx.AsyncClient(transport=httpx.ASGITransport(app=create_app(app)), - base_url='http://testserver',follow_redirects=False) as http: - reads=( - ('GET','/',200),('GET','/main',200),('HEAD','/main',200), - ('GET','/main/effect-probe',308),('HEAD','/main/effect-probe',308), - ('GET',f'/_read/{rid}/json',200),('HEAD',f'/_r/{rid}/meta',200), - ('GET','/rss',200),('GET','/latest/post',200), - ('GET','/_search?query=route',200), - ('GET','/_s/q/2/s/%2Fmain/t/route/f/b/fc/type,tag',200), - ('GET',f'{read_repo}/info/refs?service=git-upload-pack',200), + app, _ = installed + key, subject, _ = await register(app, 'route-effect-owner') + post = await call( + app, + 'content.post_create', + {'parent': '/main', 'name': 'effect-probe', 'body': 'route effect marker'}, + key=key, + subject=subject, + ) + repo = await call( + app, + 'git.create', + {'parent': '/@route-effect-owner', 'name': 'code.git'}, + key=key, + subject=subject, + ) + assert post.status == repo.status == 'ok' + rid = post.resources[0].id + repo_id = repo.resources[0].id + read_repo = repo.data['read_url'].removeprefix(app.settings.service_url.rstrip('/')) + write = request_for( + 'content.post_create', + {'parent': '/main', 'body': 'must never be written by an ordinary path'}, + app.settings.service_url, + signer=key, + subject=subject, + request_id='ordinary-route-matrix', + expires_at=NOW + timedelta(seconds=90), + ) + encoded = b64(canonical(write)) + before = await database_snapshot(app) + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=create_app(app)), + base_url='http://testserver', + follow_redirects=False, + ) as http: + reads = ( + ('GET', '/', 200), + ('GET', '/main', 200), + ('HEAD', '/main', 200), + ('GET', '/main/effect-probe', 308), + ('HEAD', '/main/effect-probe', 308), + ('GET', f'/_read/{rid}/json', 200), + ('HEAD', f'/_r/{rid}/meta', 200), + ('GET', '/rss', 200), + ('GET', '/latest/post', 200), + ('GET', '/_search?query=route', 200), + ('GET', '/_s/q/2/s/%2Fmain/t/route/f/b/fc/type,tag', 200), + ('GET', f'{read_repo}/info/refs?service=git-upload-pack', 200), + ) + for method, path, status in reads: + response = await http.request(method, path) + assert response.status_code == status, (method, path, response.text) + facets = await http.get('/_search?scope=%2Fmain&terms=route&facets=type,tag') + assert facets.status_code == 200, facets.text + assert set(facets.json()['facets']) == {'type', 'tag'} + for path in ( + '/_search?scope=%2Fmain&terms=route&facets=type,type', + '/_s/q/2/s/%2Fmain/t/route/fc/invalid', + ): + response = await http.get(path) + assert response.status_code == 400, (path, response.text) + batch = {'operation': 'download', 'objects': [{'oid': '0' * 64, 'size': 4}]} + lfs = await http.post( + read_repo + '/info/lfs/objects/batch', + json=batch, + headers={'Content-Type': 'application/vnd.git-lfs+json'}, + ) + assert lfs.status_code == 200, lfs.text + assert lfs.json()['objects'][0]['error']['code'] == 404 + upload = await http.post( + read_repo + '/info/lfs/objects/batch', + json={**batch, 'operation': 'upload'}, + headers={'Content-Type': 'application/vnd.git-lfs+json'}, + ) + assert upload.status_code == 405 + rejected = ( + ('POST', '/main'), + ('PUT', '/main/effect-probe'), + ('POST', '/_read/graphql'), + ('POST', '/@route-effect-owner/code.git/git-receive-pack'), + ('GET', '/!content.post_create/run/j/' + encoded), + ('POST', '/!content.post_create/run/j/' + encoded), + ('GET', '/run/j/' + encoded), + ('POST', '/run/j/' + encoded), + ('GET', '/~content.post_create/run/j/' + encoded), + ) + for method, path in rejected: + response = await http.request(method, path, content=canonical(write)) + assert response.status_code in {400, 404, 405}, (method, path, response.text) + gql = {'query': 'mutation($p: JSON!) { call(packet: $p) }', 'variables': {'p': wire(write)}} + for path in ('/_read/graphql', '/_r/graphql'): + response = await http.post(path, json=gql) + assert ( + response.status_code == 400 + and response.json()['error']['code'] == 'graphql_effect_mismatch' + ) + signed_read = request_for( + 'discovery.get', + {'id': rid}, + app.settings.service_url, + signer=key, + subject=subject, + expires_at=NOW + timedelta(seconds=90), + ) + invalid = await http.get( + f'/_read/{rid}/json', headers={'X-Msg-Request': b64(canonical(write))} ) - for method,path,status in reads: - response=await http.request(method,path) - assert response.status_code==status,(method,path,response.text) - facets=await http.get('/_search?scope=%2Fmain&terms=route&facets=type,tag') - assert facets.status_code==200,facets.text - assert set(facets.json()['facets'])=={'type','tag'} - for path in ('/_search?scope=%2Fmain&terms=route&facets=type,type', - '/_s/q/2/s/%2Fmain/t/route/fc/invalid'): - response=await http.get(path) - assert response.status_code==400,(path,response.text) - batch={'operation':'download','objects':[{'oid':'0'*64,'size':4}]} - lfs=await http.post(read_repo+'/info/lfs/objects/batch',json=batch, - headers={'Content-Type':'application/vnd.git-lfs+json'}) - assert lfs.status_code==200,lfs.text - assert lfs.json()['objects'][0]['error']['code']==404 - upload=await http.post(read_repo+'/info/lfs/objects/batch', - json={**batch,'operation':'upload'}, - headers={'Content-Type':'application/vnd.git-lfs+json'}) - assert upload.status_code==405 - rejected=( - ('POST','/main'),('PUT','/main/effect-probe'), - ('POST','/_read/graphql'),('POST','/@route-effect-owner/code.git/git-receive-pack'), - ('GET','/!content.post_create/run/j/'+encoded), - ('POST','/!content.post_create/run/j/'+encoded), - ('GET','/run/j/'+encoded),('POST','/run/j/'+encoded), - ('GET','/~content.post_create/run/j/'+encoded), + assert invalid.status_code == 400 + good = await http.get( + f'/_read/{rid}/json', headers={'X-Msg-Request': b64(canonical(signed_read))} ) - for method,path in rejected: - response=await http.request(method,path,content=canonical(write)) - assert response.status_code in {400,404,405},(method,path,response.text) - gql={'query':'mutation($p: JSON!) { call(packet: $p) }', - 'variables':{'p':wire(write)}} - for path in ('/_read/graphql','/_r/graphql'): - response=await http.post(path,json=gql) - assert response.status_code==400 and response.json()['error']['code']==\ - 'graphql_effect_mismatch' - signed_read=request_for('discovery.get',{'id':rid},app.settings.service_url, - signer=key,subject=subject, - expires_at=NOW+timedelta(seconds=90)) - invalid=await http.get(f'/_read/{rid}/json', - headers={'X-Msg-Request':b64(canonical(write))}) - assert invalid.status_code==400 - good=await http.get(f'/_read/{rid}/json', - headers={'X-Msg-Request':b64(canonical(signed_read))}) - assert good.status_code==200 - assert await database_snapshot(app)==before - assert not app.settings.server.repositories_dir.joinpath(repo_id+'.git','lfs').exists() + assert good.status_code == 200 + assert await database_snapshot(app) == before + assert not app.settings.server.repositories_dir.joinpath(repo_id + '.git', 'lfs').exists() @pytest.mark.asyncio async def test_effect_classification_matches_native_git_and_lfs_boundaries(installed): - app,_=installed - cases={ - ('/main','POST'):RouteEffect.PURE_READ, - ('/@owner/code.git/git-upload-pack','POST'):RouteEffect.PURE_READ, - ('/@owner/code.git/info/lfs/objects/batch','POST'):RouteEffect.PURE_READ, - ('/-/git/repo/info/lfs/objects/batch','POST'):RouteEffect.PURE_READ, - ('/-/git/repo/info/lfs/objects/'+'a'*64+'/10','PUT'):RouteEffect.EXTERNAL_EFFECT, - ('/-/git/repo/git-receive-pack','POST'):RouteEffect.EXTERNAL_EFFECT, - ('/-/git/repo/info/refs','GET'):RouteEffect.PURE_READ, - ('/-/graphql','POST'):RouteEffect.EXTERNAL_EFFECT, - ('/-/mcp','POST'):RouteEffect.EXTERNAL_EFFECT, - ('/-/transfer','POST'):RouteEffect.BUSINESS_WRITE, - ('/-/p/content.post_create','POST'):RouteEffect.BUSINESS_WRITE, + app, _ = installed + cases = { + ('/main', 'POST'): RouteEffect.PURE_READ, + ('/@owner/code.git/git-upload-pack', 'POST'): RouteEffect.PURE_READ, + ('/@owner/code.git/info/lfs/objects/batch', 'POST'): RouteEffect.PURE_READ, + ('/-/git/repo/info/lfs/objects/batch', 'POST'): RouteEffect.PURE_READ, + ('/-/git/repo/info/lfs/objects/' + 'a' * 64 + '/10', 'PUT'): RouteEffect.EXTERNAL_EFFECT, + ('/-/git/repo/git-receive-pack', 'POST'): RouteEffect.EXTERNAL_EFFECT, + ('/-/git/repo/info/refs', 'GET'): RouteEffect.PURE_READ, + ('/-/graphql', 'POST'): RouteEffect.EXTERNAL_EFFECT, + ('/-/mcp', 'POST'): RouteEffect.EXTERNAL_EFFECT, + ('/-/transfer', 'POST'): RouteEffect.BUSINESS_WRITE, + ('/-/p/content.post_create', 'POST'): RouteEffect.BUSINESS_WRITE, } - for (path,method),effect in cases.items(): - assert classify_route(path,method,app.registry).effect is effect,(method,path) + for (path, method), effect in cases.items(): + assert classify_route(path, method, app.registry).effect is effect, (method, path) diff --git a/tests/test_saved_queries.py b/tests/test_saved_queries.py index 5ac2b320..33e88c13 100644 --- a/tests/test_saved_queries.py +++ b/tests/test_saved_queries.py @@ -1,4 +1,5 @@ """Explicit saved descriptors outlive read tickets, never their live authority.""" + from datetime import timedelta import pytest @@ -10,25 +11,55 @@ async def make_query_ref(app, key, subject, arguments, kind='read'): payload = canonical({'version': 1, 'kind': kind, 'arguments': arguments}) - opened = await call(app, 'transfer.open', {'direction': 'upload', 'size': len(payload), - 'digest': digest(payload), 'media_type': 'application/vnd.msg.read-query+json'}, - key=key, subject=subject) + opened = await call( + app, + 'transfer.open', + { + 'direction': 'upload', + 'size': len(payload), + 'digest': digest(payload), + 'media_type': 'application/vnd.msg.read-query+json', + }, + key=key, + subject=subject, + ) assert opened.status == 'ok', opened.error transfer = opened.data['transfer_id'] - part = await call(app, 'transfer.part_put', {'transfer_id': transfer, 'offset': 0, - 'data': b64(payload), 'digest': digest(payload)}, key=key, subject=subject) + part = await call( + app, + 'transfer.part_put', + {'transfer_id': transfer, 'offset': 0, 'data': b64(payload), 'digest': digest(payload)}, + key=key, + subject=subject, + ) assert part.status == 'ok', part.error - sealed = await call(app, 'transfer.seal', {'transfer_id': transfer, 'final_size': len(payload), - 'final_digest': digest(payload)}, key=key, subject=subject) + sealed = await call( + app, + 'transfer.seal', + {'transfer_id': transfer, 'final_size': len(payload), 'final_digest': digest(payload)}, + key=key, + subject=subject, + ) assert sealed.status == 'ok', sealed.error - ticket = await call(app, 'transfer.query_seal', {'transfer_id': transfer}, key=key, subject=subject) + ticket = await call( + app, 'transfer.query_seal', {'transfer_id': transfer}, key=key, subject=subject + ) assert ticket.status == 'ok', ticket.error return ticket.data['query_ref'] async def current_call(app, op, args, key, subject, **kwargs): - return await app.executor.execute(request_for(op, args, app.settings.service_url, - signer=key, subject=subject, expires_at=app.clock() + timedelta(seconds=120), **kwargs)) + return await app.executor.execute( + request_for( + op, + args, + app.settings.service_url, + signer=key, + subject=subject, + expires_at=app.clock() + timedelta(seconds=120), + **kwargs, + ) + ) @pytest.mark.asyncio @@ -37,11 +68,23 @@ async def test_save_pins_descriptor_replays_and_outlives_only_ticket(installed): key, subject, _ = await register(app, 'saved-query-owner') args = {'parent': '/main', 'type': 'post', 'tag': 'work'} token = await make_query_ref(app, key, subject, args) - saved = await call(app, 'query.save', {'query_ref': token}, key=key, subject=subject, - rid='save-descriptor-once') + saved = await call( + app, + 'query.save', + {'query_ref': token}, + key=key, + subject=subject, + rid='save-descriptor-once', + ) assert saved.status == 'ok', saved.error - replay = await call(app, 'query.save', {'query_ref': token}, key=key, subject=subject, - rid='save-descriptor-once') + replay = await call( + app, + 'query.save', + {'query_ref': token}, + key=key, + subject=subject, + rid='save-descriptor-once', + ) assert replay.replayed and replay.resources == saved.resources ref = wire(saved.resources[0]) got = await call(app, 'query.saved_get', {'ref': ref}, key=key, subject=subject) @@ -50,7 +93,9 @@ async def test_save_pins_descriptor_replays_and_outlives_only_ticket(installed): assert got.data['operation'] == 'discovery.read_query' assert got.data['digest'] == saved.data['descriptor_digest'] assert 'principal' not in got.data - assert (await call(app, 'discovery.get', {'id': ref['id']}, key=key, subject=subject)).status == 'error' + assert ( + await call(app, 'discovery.get', {'id': ref['id']}, key=key, subject=subject) + ).status == 'error' app.executor.clock = lambda: NOW + timedelta(minutes=16) app.clock = app.executor.clock app.executor.authenticator.clock = app.clock @@ -60,10 +105,13 @@ async def test_save_pins_descriptor_replays_and_outlives_only_ticket(installed): assert durable.status == 'ok', durable.error async with app.metadata.transaction(write=False) as tx: generation = (await tx.resource(ref['id'])).generation - archived = await current_call(app, 'query.saved_archive', {'ref': ref}, key, subject, - expected=((ref['id'], generation),)) + archived = await current_call( + app, 'query.saved_archive', {'ref': ref}, key, subject, expected=((ref['id'], generation),) + ) assert archived.status == 'ok', archived.error - assert (await current_call(app, 'query.saved_get', {'ref': ref}, key, subject)).error.code == 'saved_query_not_found' + assert ( + await current_call(app, 'query.saved_get', {'ref': ref}, key, subject) + ).error.code == 'saved_query_not_found' @pytest.mark.asyncio @@ -71,8 +119,13 @@ async def test_saved_query_rechecks_scope_and_never_shares_captured_authority(in app, _ = installed key, subject, _ = await register(app, 'saved-scope-owner') other_key, other, _ = await register(app, 'saved-scope-reader') - topic = await call(app, 'content.topic_create', {'parent': '/main', 'name': 'saved-scope'}, - key=key, subject=subject) + topic = await call( + app, + 'content.topic_create', + {'parent': '/main', 'name': 'saved-scope'}, + key=key, + subject=subject, + ) rid = topic.resources[0].id token = await make_query_ref(app, other_key, other, {'parent': rid, 'type': 'post'}) saved = await call(app, 'query.save', {'query_ref': token}, key=other_key, subject=other) @@ -80,12 +133,25 @@ async def test_saved_query_rechecks_scope_and_never_shares_captured_authority(in ref = wire(saved.resources[0]) wrong = await call(app, 'query.save', {'query_ref': token}, key=key, subject=subject) assert wrong.error.code == 'query_ref_principal_mismatch' - assert (await call(app, 'query.saved_get', {'ref': ref}, key=key, subject=subject)).status == 'error' - grant = await call(app, 'sharing.grant', {'resource': ref['id'], 'grantee': subject, - 'expires_at': wire(NOW + timedelta(hours=1))}, key=other_key, subject=other) + assert ( + await call(app, 'query.saved_get', {'ref': ref}, key=key, subject=subject) + ).status == 'error' + grant = await call( + app, + 'sharing.grant', + {'resource': ref['id'], 'grantee': subject, 'expires_at': wire(NOW + timedelta(hours=1))}, + key=other_key, + subject=other, + ) assert grant.status == 'error' - hidden = await call(app, 'content.chmod', {'id': rid, 'mode': '0700'}, key=key, subject=subject, - expected=((rid, topic.data['generation']),)) + hidden = await call( + app, + 'content.chmod', + {'id': rid, 'mode': '0700'}, + key=key, + subject=subject, + expected=((rid, topic.data['generation']),), + ) assert hidden.status == 'ok', hidden.error denied = await call(app, 'query.saved_get', {'ref': ref}, key=other_key, subject=other) assert denied.status == 'error' @@ -100,32 +166,43 @@ async def test_saved_query_requires_pinned_revision_and_unchanged_digest(install assert saved.status == 'ok', saved.error ref = wire(saved.resources[0]) for bad_ref in ({'id': ref['id']}, {**ref, 'revision': 'nonexistent_revision'}): - assert (await call(app, 'query.saved_get', {'ref': bad_ref}, key=key, subject=subject)).status == 'error' + assert ( + await call(app, 'query.saved_get', {'ref': bad_ref}, key=key, subject=subject) + ).status == 'error' tampered = token[:-1] + ('a' if token[-1] != 'a' else 'b') - assert (await call(app, 'query.save', {'query_ref': tampered}, key=key, subject=subject)).status == 'error' + assert ( + await call(app, 'query.save', {'query_ref': tampered}, key=key, subject=subject) + ).status == 'error' @pytest.mark.asyncio async def test_saved_authority_is_current_and_internal_body_cannot_be_projected(installed): from dataclasses import replace + app, _ = installed key, subject, _ = await register(app, 'saved-authority-owner') token = await make_query_ref(app, key, subject, {'parent': '/main'}) - forbidden_projection = await current_call(app, 'query.save', {'query_ref': token}, key, subject, - return_fields=('content',)) + forbidden_projection = await current_call( + app, 'query.save', {'query_ref': token}, key, subject, return_fields=('content',) + ) assert forbidden_projection.error.code == 'projection_unavailable' saved = await call(app, 'query.save', {'query_ref': token}, key=key, subject=subject) assert saved.status == 'ok', saved.error ref = wire(saved.resources[0]) async with app.metadata.transaction(write=True) as tx: credential = await tx.credential(key.key_id) - ceiling = tuple(replace(grant, operations=grant.operations - {'query.save@1'}) - for grant in credential.ceiling) - await tx.save_credential(replace(credential, ceiling=ceiling), (await tx.subject(subject)).auth_version) + ceiling = tuple( + replace(grant, operations=grant.operations - {'query.save@1'}) + for grant in credential.ceiling + ) + await tx.save_credential( + replace(credential, ceiling=ceiling), (await tx.subject(subject)).auth_version + ) generation = (await tx.resource(ref['id'])).generation denied = await call(app, 'query.saved_get', {'ref': ref}, key=key, subject=subject) assert denied.error.code == 'credential_ceiling' # Revocation remains possible without the removed creation authority. - archived = await current_call(app, 'query.saved_archive', {'ref': ref}, key, subject, - expected=((ref['id'], generation),)) + archived = await current_call( + app, 'query.saved_archive', {'ref': ref}, key, subject, expected=((ref['id'], generation),) + ) assert archived.status == 'ok', archived.error diff --git a/tests/test_saved_query_typed_scopes.py b/tests/test_saved_query_typed_scopes.py index 7ddf8bd5..8c800e7d 100644 --- a/tests/test_saved_query_typed_scopes.py +++ b/tests/test_saved_query_typed_scopes.py @@ -1,4 +1,5 @@ """Saved search descriptors preserve the typed scope and its live authority.""" + import pytest from test_saved_queries import make_query_ref from test_service import call, register @@ -24,15 +25,21 @@ async def test_signed_typed_search_save_read_and_current_revocation(installed, k supplied = {'org': await tx.path(target)} normalized = {'org': target} else: - topic = await call(app, 'content.topic_create', {'parent': '/main', 'name': 'typed-query-topic'}, - key=key, subject=owner) + topic = await call( + app, + 'content.topic_create', + {'parent': '/main', 'name': 'typed-query-topic'}, + key=key, + subject=owner, + ) assert topic.status == 'ok', topic.error target = topic.resources[0].id supplied = {'resource_refs': [{'id': '/main/typed-query-topic'}, {'id': '/main'}]} normalized = {'resource_refs': [{'id': item} for item in sorted([target, 't_main'])]} args = {'scope': supplied, 'terms': 'typed-query-test'} - direct = await call(app, 'discovery.lexical_search', args, key=reader_key, subject=reader, - contract_version=5) + direct = await call( + app, 'discovery.lexical_search', args, key=reader_key, subject=reader, contract_version=5 + ) assert direct.status == 'ok', direct.error token = await make_query_ref(app, reader_key, reader, args, kind='search') saved = await call(app, 'query.save', {'query_ref': token}, key=reader_key, subject=reader) @@ -40,8 +47,11 @@ async def test_signed_typed_search_save_read_and_current_revocation(installed, k ref = wire(saved.resources[0]) got = await call(app, 'query.saved_get', {'ref': ref}, key=reader_key, subject=reader) assert got.status == 'ok', got.error - expected = {'operation': 'discovery.lexical_search', 'contract_version': 5, - 'arguments': {**args, 'scope': normalized}} + expected = { + 'operation': 'discovery.lexical_search', + 'contract_version': 5, + 'arguments': {**args, 'scope': normalized}, + } assert wire(got.data) == {**expected, 'digest': digest(expected)} assert reader_key.key_id not in str(got.data) assert 'principal' not in got.data @@ -49,14 +59,25 @@ async def test_signed_typed_search_save_read_and_current_revocation(installed, k raw = await call(app, 'discovery.get', {'id': ref['id']}, key=reader_key, subject=reader) assert raw.status == 'error' # Saving a legal search does not silently broaden the ReadQuery-only watch. - watch = await call(app, 'communication.watch_create', + watch = await call( + app, + 'communication.watch_create', {'query_ref': ref, 'event_types': ['content.post_create'], 'delivery': 'inbox'}, - key=reader_key, subject=reader, contract_version=2) + key=reader_key, + subject=reader, + contract_version=2, + ) assert watch.error.code == 'watch_query_unsupported' async with app.metadata.transaction(write=False) as tx: generation = (await tx.resource(target)).generation - revoked = await call(app, 'content.chmod', {'id': target, 'mode': '0700'}, - key=key, subject=owner, expected=((target, generation),)) + revoked = await call( + app, + 'content.chmod', + {'id': target, 'mode': '0700'}, + key=key, + subject=owner, + expected=((target, generation),), + ) assert revoked.status == 'ok', revoked.error denied = await call(app, 'query.saved_get', {'ref': ref}, key=reader_key, subject=reader) assert denied.status == 'error' diff --git a/tests/test_saved_query_watch.py b/tests/test_saved_query_watch.py index 62147cbe..87bff802 100644 --- a/tests/test_saved_query_watch.py +++ b/tests/test_saved_query_watch.py @@ -1,32 +1,62 @@ """Pinned saved-query watches match only current authorized Event references.""" + from datetime import timedelta import pytest +from test_service import NOW, call, register from msg.core.codec import b64, canonical, digest, wire -from test_service import NOW, call, register async def save_query(app, key, subject, arguments): payload = canonical({'version': 1, 'kind': 'read', 'arguments': arguments}) - opened = await call(app, 'transfer.open', {'direction': 'upload', 'size': len(payload), - 'digest': digest(payload), 'media_type': 'application/vnd.msg.read-query+json'}, key=key, subject=subject) + opened = await call( + app, + 'transfer.open', + { + 'direction': 'upload', + 'size': len(payload), + 'digest': digest(payload), + 'media_type': 'application/vnd.msg.read-query+json', + }, + key=key, + subject=subject, + ) assert opened.status == 'ok', wire(opened) tid = opened.data['transfer_id'] - part = await call(app, 'transfer.part_put', {'transfer_id': tid, 'offset': 0, 'data': b64(payload), 'digest': digest(payload)}, key=key, subject=subject) + part = await call( + app, + 'transfer.part_put', + {'transfer_id': tid, 'offset': 0, 'data': b64(payload), 'digest': digest(payload)}, + key=key, + subject=subject, + ) assert part.status == 'ok', wire(part) - sealed = await call(app, 'transfer.seal', {'transfer_id': tid, 'final_size': len(payload), 'final_digest': digest(payload)}, key=key, subject=subject) + sealed = await call( + app, + 'transfer.seal', + {'transfer_id': tid, 'final_size': len(payload), 'final_digest': digest(payload)}, + key=key, + subject=subject, + ) assert sealed.status == 'ok', wire(sealed) token = await call(app, 'transfer.query_seal', {'transfer_id': tid}, key=key, subject=subject) assert token.status == 'ok', wire(token) - saved = await call(app, 'query.save', {'query_ref': token.data['query_ref']}, key=key, subject=subject) + saved = await call( + app, 'query.save', {'query_ref': token.data['query_ref']}, key=key, subject=subject + ) assert saved.status == 'ok', wire(saved) return saved async def messages(app, recipient): async with app.metadata.transaction(write=False) as tx: - return [row[0] for row in tx.rows('SELECT resource FROM messages WHERE recipient=? ORDER BY resource', (recipient,))] + return [ + row[0] + for row in tx.rows( + 'SELECT resource FROM messages WHERE recipient=? ORDER BY resource', (recipient,) + ) + ] @pytest.mark.asyncio @@ -36,27 +66,68 @@ async def test_saved_watch_matches_shared_read_predicates_and_source_revoke(inst rk, reader, _ = await register(app, 'saved-watch-reader') saved = await save_query(app, rk, reader, {'parent': '/main', 'type': 'post', 'author': author}) ref = wire(saved.resources[0]) - args = {'query_ref': ref, 'event_types': ['content.post_create', 'content.post_edit'], 'delivery': 'inbox'} - created = await call(app, 'communication.watch_create', args, key=rk, subject=reader, contract_version=2) + args = { + 'query_ref': ref, + 'event_types': ['content.post_create', 'content.post_edit'], + 'delivery': 'inbox', + } + created = await call( + app, 'communication.watch_create', args, key=rk, subject=reader, contract_version=2 + ) assert created.status == 'ok', wire(created) - chosen = await call(app, 'content.post_create', {'parent': '/main', 'body': 'match'}, key=ak, subject=author) - skipped = await call(app, 'content.post_create', {'parent': '/main', 'body': 'different author'}, key=rk, subject=reader) + chosen = await call( + app, 'content.post_create', {'parent': '/main', 'body': 'match'}, key=ak, subject=author + ) + skipped = await call( + app, + 'content.post_create', + {'parent': '/main', 'body': 'different author'}, + key=rk, + subject=reader, + ) assert chosen.status == skipped.status == 'ok' assert await messages(app, reader) == [chosen.resources[0].id] - hidden = await call(app, 'content.chmod', {'id': chosen.resources[0].id, 'mode': '0600'}, - key=ak, subject=author, expected=((chosen.resources[0].id, chosen.data['generation']),)) + hidden = await call( + app, + 'content.chmod', + {'id': chosen.resources[0].id, 'mode': '0600'}, + key=ak, + subject=author, + expected=((chosen.resources[0].id, chosen.data['generation']),), + ) assert hidden.status == 'ok', wire(hidden) - edited = await call(app, 'content.post_edit', {'id': chosen.resources[0].id, - 'expected_revision': chosen.resources[0].revision, 'body': 'private change'}, - key=ak, subject=author, expected=((chosen.resources[0].id, hidden.data['generation']),)) + edited = await call( + app, + 'content.post_edit', + { + 'id': chosen.resources[0].id, + 'expected_revision': chosen.resources[0].revision, + 'body': 'private change', + }, + key=ak, + subject=author, + expected=((chosen.resources[0].id, hidden.data['generation']),), + ) assert edited.status == 'ok', wire(edited) assert await messages(app, reader) == [chosen.resources[0].id] async with app.metadata.transaction(write=False) as tx: source = await tx.resource(ref['id']) - archived = await call(app, 'query.saved_archive', {'ref': ref}, key=rk, subject=reader, - expected=((source.id, source.generation),)) + archived = await call( + app, + 'query.saved_archive', + {'ref': ref}, + key=rk, + subject=reader, + expected=((source.id, source.generation),), + ) assert archived.status == 'ok', wire(archived) - after = await call(app, 'content.post_create', {'parent': '/main', 'body': 'source revoked'}, key=ak, subject=author) + after = await call( + app, + 'content.post_create', + {'parent': '/main', 'body': 'source revoked'}, + key=ak, + subject=author, + ) assert after.status == 'ok', wire(after) assert await messages(app, reader) == [chosen.resources[0].id] @@ -66,43 +137,88 @@ async def test_saved_watch_rejects_unsupported_predicates_and_missing_revision(i app, _ = installed key, user, _ = await register(app, 'saved-watch-reject') saved = await save_query(app, key, user, {'parent': '/main', 'limit': 1}) - args = {'query_ref': wire(saved.resources[0]), 'event_types': ['content.post_create'], 'delivery': 'inbox'} - rejected = await call(app, 'communication.watch_create', args, key=key, subject=user, contract_version=2) + args = { + 'query_ref': wire(saved.resources[0]), + 'event_types': ['content.post_create'], + 'delivery': 'inbox', + } + rejected = await call( + app, 'communication.watch_create', args, key=key, subject=user, contract_version=2 + ) assert rejected.error.code == 'watch_query_unsupported', wire(rejected) args['query_ref'] = {'id': saved.resources[0].id} - rejected = await call(app, 'communication.watch_create', args, key=key, subject=user, contract_version=2) + rejected = await call( + app, 'communication.watch_create', args, key=key, subject=user, contract_version=2 + ) assert rejected.error.code == 'watch_query_revision_required', wire(rejected) @pytest.mark.asyncio -async def test_saved_watch_cli_signs_pinned_v2_and_expiry_stops_delivery(installed, tmp_path, monkeypatch, capsys): +async def test_saved_watch_cli_signs_pinned_v2_and_expiry_stops_delivery( + installed, tmp_path, monkeypatch, capsys +): import httpx + from msg import cli from msg.client import ClientState, MsgClient from msg.core.codec import loads from msg.transports.client import HTTPTransport from msg.transports.http import create_app + app, _ = installed ak, author, _ = await register(app, 'saved-watch-cli-author') - async with httpx.AsyncClient(transport=httpx.ASGITransport(app=create_app(app)), base_url=app.settings.service_url) as http: + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=create_app(app)), base_url=app.settings.service_url + ) as http: monkeypatch.setitem(cli.TRANSPORTS, 'http', lambda server: HTTPTransport(server, http=http)) - directory = tmp_path/'client' - client = MsgClient(ClientState(directory, server=app.settings.service_url), HTTPTransport(app.settings.service_url, http=http), clock=lambda: NOW) + directory = tmp_path / 'client' + client = MsgClient( + ClientState(directory, server=app.settings.service_url), + HTTPTransport(app.settings.service_url, http=http), + clock=lambda: NOW, + ) assert (await client.register('saved-watch-cli')).status == 'ok' - saved = await save_query(app, client.state.signer, client.state.subject, {'parent': '/main', 'type': 'post'}) + saved = await save_query( + app, client.state.signer, client.state.subject, {'parent': '/main', 'type': 'post'} + ) ref = saved.resources[0] - monkeypatch.setattr(cli, 'MsgClient', lambda state, transport: MsgClient(state, transport, clock=lambda: NOW)) - args = cli.parser().parse_args(['--config-dir', str(directory), '--server', app.settings.service_url, - 'watch', 'create', '--query-ref', ref.id, '--query-revision', ref.revision, - '--event', 'content.post_create', '--expires-at', wire(NOW+timedelta(seconds=1))]) + monkeypatch.setattr( + cli, + 'MsgClient', + lambda state, transport: MsgClient(state, transport, clock=lambda: NOW), + ) + args = cli.parser().parse_args([ + '--config-dir', + str(directory), + '--server', + app.settings.service_url, + 'watch', + 'create', + '--query-ref', + ref.id, + '--query-revision', + ref.revision, + '--event', + 'content.post_create', + '--expires-at', + wire(NOW + timedelta(seconds=1)), + ]) status = await cli.run(args) output = loads(capsys.readouterr().out.encode().strip()) assert status == 0 and output['status'] == 'ok', output - before = await call(app, 'content.post_create', {'parent': '/main', 'body': 'live'}, key=ak, subject=author) + before = await call( + app, 'content.post_create', {'parent': '/main', 'body': 'live'}, key=ak, subject=author + ) assert before.status == 'ok' assert await messages(app, client.state.subject) == [before.resources[0].id] - app.clock = lambda: NOW+timedelta(seconds=2) - after = await call(app, 'content.post_create', {'parent': '/main', 'body': 'expired'}, key=ak, subject=author) + app.clock = lambda: NOW + timedelta(seconds=2) + after = await call( + app, + 'content.post_create', + {'parent': '/main', 'body': 'expired'}, + key=ak, + subject=author, + ) assert after.status == 'ok', wire(after) assert await messages(app, client.state.subject) == [before.resources[0].id] @@ -113,15 +229,32 @@ async def test_saved_watch_delivers_new_tag_match_without_query_execution(instal ak, author, _ = await register(app, 'saved-watch-tag-author') rk, reader, _ = await register(app, 'saved-watch-tag-reader') saved = await save_query(app, rk, reader, {'parent': '/main', 'tag': 'research'}) - watched = await call(app, 'communication.watch_create', { - 'query_ref': wire(saved.resources[0]), 'event_types': ['content.post_create', 'content.tags_set'], - 'delivery': 'inbox'}, key=rk, subject=reader, contract_version=2) + watched = await call( + app, + 'communication.watch_create', + { + 'query_ref': wire(saved.resources[0]), + 'event_types': ['content.post_create', 'content.tags_set'], + 'delivery': 'inbox', + }, + key=rk, + subject=reader, + contract_version=2, + ) assert watched.status == 'ok', wire(watched) - post = await call(app, 'content.post_create', {'parent': '/main', 'body': 'untagged'}, key=ak, subject=author) + post = await call( + app, 'content.post_create', {'parent': '/main', 'body': 'untagged'}, key=ak, subject=author + ) assert post.status == 'ok' assert await messages(app, reader) == [] - tagged = await call(app, 'content.tags_set', {'id': post.resources[0].id, 'tags': ['research']}, - key=ak, subject=author, expected=((post.resources[0].id, post.data['generation']),)) + tagged = await call( + app, + 'content.tags_set', + {'id': post.resources[0].id, 'tags': ['research']}, + key=ak, + subject=author, + expected=((post.resources[0].id, post.data['generation']),), + ) assert tagged.status == 'ok', wire(tagged) assert await messages(app, reader) == [post.resources[0].id] @@ -129,6 +262,7 @@ async def test_saved_watch_delivers_new_tag_match_without_query_execution(instal @pytest.mark.asyncio async def test_saved_watch_stops_when_source_key_revoked_but_watch_key_remains_valid(installed): from msg.security.crypto import Ed25519Signer + app, _ = installed ak, author, _ = await register(app, 'saved-watch-dual-author') first, reader, _ = await register(app, 'saved-watch-dual-reader') @@ -138,17 +272,39 @@ async def test_saved_watch_stops_when_source_key_revoked_but_watch_key_remains_v async with app.metadata.transaction(write=False) as tx: ceiling = wire((await tx.credential(first.key_id)).ceiling) proof = second.sign(canonical({'subject_id': reader, 'public_key': public}), purpose='key-add') - added = await call(app, 'identity.key_add', {'public_key': public, 'possession_proof': wire(proof), - 'ceiling': ceiling}, key=first, subject=reader) + added = await call( + app, + 'identity.key_add', + {'public_key': public, 'possession_proof': wire(proof), 'ceiling': ceiling}, + key=first, + subject=reader, + ) assert added.status == 'ok', wire(added) - watched = await call(app, 'communication.watch_create', { - 'query_ref': wire(saved.resources[0]), 'event_types': ['content.post_create'], 'delivery': 'inbox'}, - key=second, subject=reader, contract_version=2) + watched = await call( + app, + 'communication.watch_create', + { + 'query_ref': wire(saved.resources[0]), + 'event_types': ['content.post_create'], + 'delivery': 'inbox', + }, + key=second, + subject=reader, + contract_version=2, + ) assert watched.status == 'ok', wire(watched) - revoked = await call(app, 'identity.key_revoke', {'key_id': first.key_id}, key=second, subject=reader) + revoked = await call( + app, 'identity.key_revoke', {'key_id': first.key_id}, key=second, subject=reader + ) assert revoked.status == 'ok', wire(revoked) alive = await call(app, 'communication.watch_list', {}, key=second, subject=reader) assert alive.status == 'ok', wire(alive) - posted = await call(app, 'content.post_create', {'parent': '/main', 'body': 'source key gone'}, key=ak, subject=author) + posted = await call( + app, + 'content.post_create', + {'parent': '/main', 'body': 'source key gone'}, + key=ak, + subject=author, + ) assert posted.status == 'ok', wire(posted) assert await messages(app, reader) == [] diff --git a/tests/test_search_source_relation_filters.py b/tests/test_search_source_relation_filters.py index c1755e97..548eb3a5 100644 --- a/tests/test_search_source_relation_filters.py +++ b/tests/test_search_source_relation_filters.py @@ -1,91 +1,152 @@ """SearchQuery v3 filters inspect only currently readable current revisions.""" + from datetime import timedelta import httpx import pytest +from test_service import NOW, call, register -from msg.core.codec import b64,canonical,digest,wire +from msg.core.codec import b64, canonical, digest, wire from msg.core.requests import request_for from msg.transports.http import create_app -from test_service import NOW,call,register @pytest.mark.asyncio @pytest.mark.parametrize('version', [3, 5]) async def test_v3_query_string_path_and_sealed_ref_share_filters(installed, version): - app,_=installed - key,user,_=await register(app,'search-v3-http') - root=await call(app,'content.post_create',{'parent':'/main','body':'v3needle root'}, - key=key,subject=user) - reply=await call(app,'discussion.reply',{'target':{'id':root.resources[0].id}, - 'body':'v3needle reply'},key=key,subject=user) - assert root.status==reply.status=='ok' - async with httpx.AsyncClient(transport=httpx.ASGITransport(app=create_app(app)), - base_url=app.settings.service_url) as http: - query=await http.get('/_search?scope=%2Fmain&terms=v3needle&relation_type=reply_to') - path=await http.get('/_s/q/3/s/%2Fmain/t/v3needle/rt/reply_to') - assert query.status_code==path.status_code==200,(query.text,path.text) - assert query.content==path.content - assert [row['id'] for row in query.json()['items']]==[reply.resources[0].id] - descriptor_args={'scope':'/main','terms':'v3needle','relation_type':'reply_to'} - if version==5: - descriptor_args['revision']=reply.resources[0].revision - descriptor_args['relation_to']=root.resources[0].id - descriptor_args['scope']={'resource_refs':[{'id':reply.resources[0].id}]} - descriptor=canonical({'version':1,'kind':'search','arguments':descriptor_args}) - opened=await call(app,'transfer.open',{'direction':'upload','size':len(descriptor), - 'digest':digest(descriptor),'media_type':'application/vnd.msg.read-query+json'}, - key=key,subject=user) - tid=opened.data['transfer_id'] - await call(app,'transfer.part_put',{'transfer_id':tid,'offset':0, - 'data':b64(descriptor),'digest':digest(descriptor)},key=key,subject=user) - await call(app,'transfer.seal',{'transfer_id':tid,'final_size':len(descriptor), - 'final_digest':digest(descriptor)},key=key,subject=user) - sealed=await call(app,'transfer.query_seal',{'transfer_id':tid},key=key,subject=user) - assert sealed.status=='ok',wire(sealed) - token=sealed.data['query_ref'] - proof=request_for('transfer.query_get',{'query_ref':token},app.settings.service_url, - signer=key,subject=user,expires_at=NOW+timedelta(seconds=60)) - async with httpx.AsyncClient(transport=httpx.ASGITransport(app=create_app(app)), - base_url=app.settings.service_url) as http: - resolved=await http.get('/_r/q/'+token+'/p/'+b64(canonical(proof))) - assert resolved.status_code==200,resolved.text - assert [row['id'] for row in resolved.json()['items']]==[reply.resources[0].id] + app, _ = installed + key, user, _ = await register(app, 'search-v3-http') + root = await call( + app, + 'content.post_create', + {'parent': '/main', 'body': 'v3needle root'}, + key=key, + subject=user, + ) + reply = await call( + app, + 'discussion.reply', + {'target': {'id': root.resources[0].id}, 'body': 'v3needle reply'}, + key=key, + subject=user, + ) + assert root.status == reply.status == 'ok' + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=create_app(app)), base_url=app.settings.service_url + ) as http: + query = await http.get('/_search?scope=%2Fmain&terms=v3needle&relation_type=reply_to') + path = await http.get('/_s/q/3/s/%2Fmain/t/v3needle/rt/reply_to') + assert query.status_code == path.status_code == 200, (query.text, path.text) + assert query.content == path.content + assert [row['id'] for row in query.json()['items']] == [reply.resources[0].id] + descriptor_args = {'scope': '/main', 'terms': 'v3needle', 'relation_type': 'reply_to'} + if version == 5: + descriptor_args['revision'] = reply.resources[0].revision + descriptor_args['relation_to'] = root.resources[0].id + descriptor_args['scope'] = {'resource_refs': [{'id': reply.resources[0].id}]} + descriptor = canonical({'version': 1, 'kind': 'search', 'arguments': descriptor_args}) + opened = await call( + app, + 'transfer.open', + { + 'direction': 'upload', + 'size': len(descriptor), + 'digest': digest(descriptor), + 'media_type': 'application/vnd.msg.read-query+json', + }, + key=key, + subject=user, + ) + tid = opened.data['transfer_id'] + await call( + app, + 'transfer.part_put', + {'transfer_id': tid, 'offset': 0, 'data': b64(descriptor), 'digest': digest(descriptor)}, + key=key, + subject=user, + ) + await call( + app, + 'transfer.seal', + {'transfer_id': tid, 'final_size': len(descriptor), 'final_digest': digest(descriptor)}, + key=key, + subject=user, + ) + sealed = await call(app, 'transfer.query_seal', {'transfer_id': tid}, key=key, subject=user) + assert sealed.status == 'ok', wire(sealed) + token = sealed.data['query_ref'] + proof = request_for( + 'transfer.query_get', + {'query_ref': token}, + app.settings.service_url, + signer=key, + subject=user, + expires_at=NOW + timedelta(seconds=60), + ) + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=create_app(app)), base_url=app.settings.service_url + ) as http: + resolved = await http.get('/_r/q/' + token + '/p/' + b64(canonical(proof))) + assert resolved.status_code == 200, resolved.text + assert [row['id'] for row in resolved.json()['items']] == [reply.resources[0].id] @pytest.mark.asyncio async def test_source_and_relation_filters_are_v3_and_use_current_revision(installed): app, _ = installed key, user, _ = await register(app, 'search-relation-owner') - root = await call(app, 'content.post_create', - {'parent': '/main', 'body': 'relationfilterneedle root'}, - key=key, subject=user) - reply = await call(app, 'discussion.reply', - {'target': {'id': root.resources[0].id}, - 'body': 'relationfilterneedle reply'}, key=key, subject=user) - quoted = await call(app, 'discussion.quote', - {'target': {'id': root.resources[0].id}, 'parent': '/main', - 'body': 'relationfilterneedle quote'}, key=key, subject=user) + root = await call( + app, + 'content.post_create', + {'parent': '/main', 'body': 'relationfilterneedle root'}, + key=key, + subject=user, + ) + reply = await call( + app, + 'discussion.reply', + {'target': {'id': root.resources[0].id}, 'body': 'relationfilterneedle reply'}, + key=key, + subject=user, + ) + quoted = await call( + app, + 'discussion.quote', + { + 'target': {'id': root.resources[0].id}, + 'parent': '/main', + 'body': 'relationfilterneedle quote', + }, + key=key, + subject=user, + ) assert root.status == reply.status == quoted.status == 'ok' base = {'scope': '/main', 'terms': 'relationfilterneedle', 'field': 'body'} - reply_only = await call(app, 'discovery.lexical_search', - {**base, 'relation_type': 'reply_to'}, contract_version=3) + reply_only = await call( + app, 'discovery.lexical_search', {**base, 'relation_type': 'reply_to'}, contract_version=3 + ) assert reply_only.status == 'ok', wire(reply_only) assert [row['id'] for row in reply_only.data['items']] == [reply.resources[0].id] - quote_only = await call(app, 'discovery.lexical_search', - {**base, 'relation_type': 'quote'}, contract_version=3) + quote_only = await call( + app, 'discovery.lexical_search', {**base, 'relation_type': 'quote'}, contract_version=3 + ) assert [row['id'] for row in quote_only.data['items']] == [quoted.resources[0].id] - no_release = await call(app, 'discovery.lexical_search', - {**base, 'source_kind': 'release'}, contract_version=3) + no_release = await call( + app, 'discovery.lexical_search', {**base, 'source_kind': 'release'}, contract_version=3 + ) assert no_release.status == 'ok' and list(no_release.data['items']) == [] - released = await call(app, 'discovery.lexical_search', - {'scope': '/_rules', 'terms': 'identity', 'field': 'body', - 'source_kind': 'release'}, contract_version=3) + released = await call( + app, + 'discovery.lexical_search', + {'scope': '/_rules', 'terms': 'identity', 'field': 'body', 'source_kind': 'release'}, + contract_version=3, + ) assert released.status == 'ok', wire(released) assert released.data['items'] assert all(row['path'].startswith('/_rules/') for row in released.data['items']) - old_contract = await call(app, 'discovery.lexical_search', - {**base, 'relation_type': 'reply_to'}, contract_version=2) + old_contract = await call( + app, 'discovery.lexical_search', {**base, 'relation_type': 'reply_to'}, contract_version=2 + ) assert old_contract.status == 'error' @@ -93,35 +154,56 @@ async def test_source_and_relation_filters_are_v3_and_use_current_revision(insta async def test_v3_old_cursor_rechecks_grants_before_items_facets_and_rank(installed): app, _ = installed key, user, _ = await register(app, 'search-revoke-owner') - root = await call(app, 'content.post_create', - {'parent': '/main', 'body': 'revoke-filter-root'}, - key=key, subject=user) + root = await call( + app, + 'content.post_create', + {'parent': '/main', 'body': 'revoke-filter-root'}, + key=key, + subject=user, + ) replies = [] for _ in range(3): - reply = await call(app, 'discussion.reply', - {'target': {'id': root.resources[0].id}, - 'body': 'revokerelationneedle'}, key=key, subject=user) + reply = await call( + app, + 'discussion.reply', + {'target': {'id': root.resources[0].id}, 'body': 'revokerelationneedle'}, + key=key, + subject=user, + ) assert reply.status == 'ok', wire(reply) replies.append(reply) - args = {'scope': '/main', 'terms': 'revokerelationneedle', 'field': 'body', - 'relation_type': 'reply_to', 'limit': 1, 'facets': ['type']} + args = { + 'scope': '/main', + 'terms': 'revokerelationneedle', + 'field': 'body', + 'relation_type': 'reply_to', + 'limit': 1, + 'facets': ['type'], + } first = await call(app, 'discovery.lexical_search', args, contract_version=3) assert first.status == 'ok', wire(first) assert wire(first)['data']['facets'] == {'type': [{'value': 'post', 'count': 3}]} assert first.data['cursor'] - downgraded = await call(app, 'discovery.lexical_search', - {'cursor': first.data['cursor']}, contract_version=2) + downgraded = await call( + app, 'discovery.lexical_search', {'cursor': first.data['cursor']}, contract_version=2 + ) assert downgraded.status == 'error' and downgraded.error.code == 'cursor_query_mismatch' visible_first = first.data['items'][0]['id'] for reply in replies: rid = reply.resources[0].id if rid != visible_first: - locked = await call(app, 'content.chmod', {'id': rid, 'mode': '0600'}, - key=key, subject=user, - expected=((rid, reply.data['generation']),)) + locked = await call( + app, + 'content.chmod', + {'id': rid, 'mode': '0600'}, + key=key, + subject=user, + expected=((rid, reply.data['generation']),), + ) assert locked.status == 'ok', wire(locked) - continued = await call(app, 'discovery.lexical_search', - {'cursor': first.data['cursor']}, contract_version=3) + continued = await call( + app, 'discovery.lexical_search', {'cursor': first.data['cursor']}, contract_version=3 + ) assert continued.status == 'ok', wire(continued) assert list(continued.data['items']) == [] assert wire(continued)['data']['facets'] == {'type': [{'value': 'post', 'count': 1}]} @@ -133,8 +215,13 @@ async def test_v3_old_cursor_rechecks_grants_before_items_facets_and_rank(instal async def test_v5_revision_and_source_version_filters_use_current_readable_revision(installed): app, _ = installed key, subject, _ = await register(app, 'search-v5-owner') - created = await call(app, 'content.post_create', {'parent': '/main', 'body': 'v5revisionneedle'}, - key=key, subject=subject) + created = await call( + app, + 'content.post_create', + {'parent': '/main', 'body': 'v5revisionneedle'}, + key=key, + subject=subject, + ) assert created.status == 'ok', wire(created) rid, revision = created.resources[0].id, created.resources[0].revision args = {'scope': '/main', 'terms': 'v5revisionneedle', 'revision': revision} @@ -143,48 +230,86 @@ async def test_v5_revision_and_source_version_filters_use_current_readable_revis assert [item['id'] for item in found.data['items']] == [rid] old = await call(app, 'discovery.lexical_search', args, contract_version=4) assert old.status == 'error' - async with httpx.AsyncClient(transport=httpx.ASGITransport(app=create_app(app)), - base_url=app.settings.service_url) as http: + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=create_app(app)), base_url=app.settings.service_url + ) as http: query = await http.get('/_search', params=args) path = await http.get('/_s/q/5/s/%2Fmain/t/v5revisionneedle/rv/' + revision) assert query.status_code == path.status_code == 200, (query.text, path.text) assert query.content == path.content - edited = await call(app, 'content.post_edit', {'id': rid, 'body': 'v5revisionneedle updated', - 'expected_revision': revision}, key=key, subject=subject, - expected=((rid, created.data['generation']),)) + edited = await call( + app, + 'content.post_edit', + {'id': rid, 'body': 'v5revisionneedle updated', 'expected_revision': revision}, + key=key, + subject=subject, + expected=((rid, created.data['generation']),), + ) assert edited.status == 'ok', wire(edited) gone = await call(app, 'discovery.lexical_search', args, contract_version=5) assert gone.status == 'ok' and not gone.data['items'] - release = await call(app, 'discovery.lexical_search', - {'scope': '/_rules', 'terms': 'identity', 'source_kind': 'release'}, - contract_version=3) + release = await call( + app, + 'discovery.lexical_search', + {'scope': '/_rules', 'terms': 'identity', 'source_kind': 'release'}, + contract_version=3, + ) ref = release.data['items'][0]['id'] from msg.core.models import ResourceRef + async with app.metadata.transaction(write=False) as tx: source = await tx.revision(ResourceRef(id=ref)) - filtered = await call(app, 'discovery.lexical_search', { - 'scope': '/_rules', 'terms': 'identity', 'revision': source.id, - 'source_kind': 'release', 'source_version': source.source_version}, contract_version=5) + filtered = await call( + app, + 'discovery.lexical_search', + { + 'scope': '/_rules', + 'terms': 'identity', + 'revision': source.id, + 'source_kind': 'release', + 'source_version': source.source_version, + }, + contract_version=5, + ) assert filtered.status == 'ok', wire(filtered) assert [item['id'] for item in filtered.data['items']] == [ref] - missed = await call(app, 'discovery.lexical_search', { - 'scope': '/_rules', 'terms': 'identity', 'revision': source.id, - 'source_version': source.source_version + 1}, contract_version=5) + missed = await call( + app, + 'discovery.lexical_search', + { + 'scope': '/_rules', + 'terms': 'identity', + 'revision': source.id, + 'source_version': source.source_version + 1, + }, + contract_version=5, + ) assert missed.status == 'ok' and not missed.data['items'] @pytest.mark.asyncio async def test_v5_source_version_cursor_preserves_contract_and_rejects_downgrade(installed): app, _ = installed - first = await call(app, 'discovery.lexical_search', { - 'scope': '/_rules', 'terms': 'identity', 'source_kind': 'release', - 'source_version': 1, 'limit': 1}, contract_version=5) + first = await call( + app, + 'discovery.lexical_search', + { + 'scope': '/_rules', + 'terms': 'identity', + 'source_kind': 'release', + 'source_version': 1, + 'limit': 1, + }, + contract_version=5, + ) assert first.status == 'ok' and first.data['cursor'], wire(first) - downgraded = await call(app, 'discovery.lexical_search', - {'cursor': first.data['cursor']}, contract_version=4) + downgraded = await call( + app, 'discovery.lexical_search', {'cursor': first.data['cursor']}, contract_version=4 + ) assert downgraded.status == 'error' and downgraded.error.code == 'cursor_query_mismatch' - async with httpx.AsyncClient(transport=httpx.ASGITransport(app=create_app(app)), - base_url=app.settings.service_url) as http: + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=create_app(app)), base_url=app.settings.service_url + ) as http: next_page = await http.get(first.data['next']) assert next_page.status_code == 200, next_page.text query = await http.get('/_search?scope=%2F_rules&terms=identity&source_version=1') @@ -197,18 +322,36 @@ async def test_v5_source_version_cursor_preserves_contract_and_rejects_downgrade async def test_v5_revision_filter_does_not_reveal_private_resource(installed): app, _ = installed key, subject, _ = await register(app, 'private-revision-owner') - created = await call(app, 'content.post_create', - {'parent': '/main', 'body': 'privatev5needle'}, key=key, subject=subject) + created = await call( + app, + 'content.post_create', + {'parent': '/main', 'body': 'privatev5needle'}, + key=key, + subject=subject, + ) rid = created.resources[0].id - changed = await call(app, 'content.chmod', {'id': rid, 'mode': '0600'}, - key=key, subject=subject, expected=((rid, created.data['generation']),)) + changed = await call( + app, + 'content.chmod', + {'id': rid, 'mode': '0600'}, + key=key, + subject=subject, + expected=((rid, created.data['generation']),), + ) assert changed.status == 'ok', wire(changed) - args = {'scope': '/main', 'terms': 'privatev5needle', - 'revision': created.resources[0].revision, 'facets': ['type'], 'snippet': True} + args = { + 'scope': '/main', + 'terms': 'privatev5needle', + 'revision': created.resources[0].revision, + 'facets': ['type'], + 'snippet': True, + } denied = await call(app, 'discovery.lexical_search', args, contract_version=5) assert denied.status == 'ok' and not denied.data['items'] assert wire(denied.data['facets']) == {'type': []} - allowed = await call(app, 'discovery.lexical_search', args, key=key, subject=subject, contract_version=5) + allowed = await call( + app, 'discovery.lexical_search', args, key=key, subject=subject, contract_version=5 + ) assert allowed.status == 'ok' and [item['id'] for item in allowed.data['items']] == [rid] @@ -216,42 +359,79 @@ async def test_v5_revision_filter_does_not_reveal_private_resource(installed): async def test_v5_directional_relations_and_presence_recheck_both_endpoints(installed): app, _ = installed key, subject, _ = await register(app, 'directional-search-owner') - root = await call(app, 'content.post_create', {'parent': '/main', 'body': 'directionneedle root'}, - key=key, subject=subject) - reply = await call(app, 'discussion.reply', {'target': {'id': root.resources[0].id}, - 'body': 'directionneedle reply'}, key=key, subject=subject) + root = await call( + app, + 'content.post_create', + {'parent': '/main', 'body': 'directionneedle root'}, + key=key, + subject=subject, + ) + reply = await call( + app, + 'discussion.reply', + {'target': {'id': root.resources[0].id}, 'body': 'directionneedle reply'}, + key=key, + subject=subject, + ) assert root.status == reply.status == 'ok' root_id, reply_id = root.resources[0].id, reply.resources[0].id base = {'scope': '/main', 'terms': 'directionneedle', 'facets': ['type']} + async def find(extra, *, signed=False): - result = await call(app, 'discovery.lexical_search', {**base, **extra}, contract_version=5, - key=key if signed else None, subject=subject if signed else None) + result = await call( + app, + 'discovery.lexical_search', + {**base, **extra}, + contract_version=5, + key=key if signed else None, + subject=subject if signed else None, + ) assert result.status == 'ok', wire(result) return [item['id'] for item in result.data['items']] + assert await find({'relation_to': root_id, 'relation_type': 'reply_to'}) == [reply_id] assert await find({'relation_from': reply_id, 'relation_type': 'reply_to'}) == [root_id] assert await find({'has_replies': True}) == [root_id] assert await find({'has_references': True}) == [root_id] assert await find({'has_replies': False}) == [reply_id] - private = await call(app, 'content.chmod', {'id': reply_id, 'mode': '0600'}, - key=key, subject=subject, expected=((reply_id, reply.data['generation']),)) + private = await call( + app, + 'content.chmod', + {'id': reply_id, 'mode': '0600'}, + key=key, + subject=subject, + expected=((reply_id, reply.data['generation']),), + ) assert private.status == 'ok', wire(private) assert await find({'relation_from': reply_id}) == [] assert await find({'has_replies': True}) == [] assert await find({'has_references': True}) == [] assert await find({'has_replies': True}, signed=True) == [root_id] - async with httpx.AsyncClient(transport=httpx.ASGITransport(app=create_app(app)), - base_url=app.settings.service_url) as http: + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=create_app(app)), base_url=app.settings.service_url + ) as http: query = await http.get('/_search', params={**base, 'facets': 'type', 'has_replies': '1'}) path = await http.get('/_s/q/5/s/%2Fmain/t/directionneedle/fc/type/hr/1') assert query.status_code == path.status_code == 200, (query.text, path.text) assert query.content == path.content # Readable source cannot reveal its now-private target through a predicate. - hidden = await call(app, 'content.chmod', {'id': root_id, 'mode': '0600'}, - key=key, subject=subject, expected=((root_id, root.data['generation']),)) + hidden = await call( + app, + 'content.chmod', + {'id': root_id, 'mode': '0600'}, + key=key, + subject=subject, + expected=((root_id, root.data['generation']),), + ) assert hidden.status == 'ok', wire(hidden) - public_reply = await call(app, 'content.chmod', {'id': reply_id, 'mode': '0644'}, - key=key, subject=subject, expected=((reply_id, private.data['generation']),)) + public_reply = await call( + app, + 'content.chmod', + {'id': reply_id, 'mode': '0644'}, + key=key, + subject=subject, + expected=((reply_id, private.data['generation']),), + ) assert public_reply.status == 'ok', wire(public_reply) assert await find({'relation_to': root_id}) == [] assert await find({'relation_to': root_id}, signed=True) == [reply_id] @@ -261,20 +441,38 @@ async def find(extra, *, signed=False): async def test_v5_title_is_explicit_resource_display_name_alias(installed): app, _ = installed key, subject, _ = await register(app, 'title-search-owner') - created = await call(app, 'content.post_create', { - 'parent': '/main', 'name': 'NeedleTitle', 'body': 'unrelated content'}, key=key, subject=subject) + created = await call( + app, + 'content.post_create', + {'parent': '/main', 'name': 'NeedleTitle', 'body': 'unrelated content'}, + key=key, + subject=subject, + ) assert created.status == 'ok', wire(created) - args = {'scope': '/main', 'terms': 'needletitle', 'field': 'title', - 'fields': ['id', 'title', 'snippet'], 'snippet': True} + args = { + 'scope': '/main', + 'terms': 'needletitle', + 'field': 'title', + 'fields': ['id', 'title', 'snippet'], + 'snippet': True, + } found = await call(app, 'discovery.lexical_search', args, contract_version=5) assert found.status == 'ok', wire(found) - assert wire(found.data['items']) == [{'id': created.resources[0].id, 'title': 'NeedleTitle.md', - 'snippet': {'field': 'title', 'text': 'NeedleTitle.md', 'range': [0, 11]}}] + assert wire(found.data['items']) == [ + { + 'id': created.resources[0].id, + 'title': 'NeedleTitle.md', + 'snippet': {'field': 'title', 'text': 'NeedleTitle.md', 'range': [0, 11]}, + } + ] old = await call(app, 'discovery.lexical_search', args, contract_version=4) assert old.status == 'error' - async with httpx.AsyncClient(transport=httpx.ASGITransport(app=create_app(app)), - base_url=app.settings.service_url) as http: - query = await http.get('/_search', params={**args, 'fields': 'id,title,snippet', 'snippet': '1'}) + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=create_app(app)), base_url=app.settings.service_url + ) as http: + query = await http.get( + '/_search', params={**args, 'fields': 'id,title,snippet', 'snippet': '1'} + ) path = await http.get('/_s/q/5/s/%2Fmain/t/needletitle/f/t/fi/id,title,snippet/x/1') assert query.status_code == path.status_code == 200, (query.text, path.text) assert query.content == path.content @@ -285,12 +483,18 @@ async def test_v5_title_is_explicit_resource_display_name_alias(installed): @pytest.mark.asyncio async def test_v5_typed_scopes_union_resources_and_subject_with_current_access(installed): from urllib.parse import quote + app, _ = installed key, subject, _ = await register(app, 'scope-search-owner') posts = [] for name in ('scope-one', 'scope-two', 'scope-outside'): - post = await call(app, 'content.post_create', {'parent': '/main', 'name': name, 'body': 'scopev5needle'}, - key=key, subject=subject) + post = await call( + app, + 'content.post_create', + {'parent': '/main', 'name': name, 'body': 'scopev5needle'}, + key=key, + subject=subject, + ) assert post.status == 'ok', wire(post) posts.append(post) ids = [post.resources[0].id for post in posts] @@ -299,48 +503,117 @@ async def test_v5_typed_scopes_union_resources_and_subject_with_current_access(i result = await call(app, 'discovery.lexical_search', args, contract_version=5) assert result.status == 'ok', wire(result) assert [item['id'] for item in result.data['items']] == [ids[0]] - continued = await call(app, 'discovery.lexical_search', {'cursor': result.data['cursor']}, contract_version=5) + continued = await call( + app, 'discovery.lexical_search', {'cursor': result.data['cursor']}, contract_version=5 + ) assert continued.status == 'ok' and [item['id'] for item in continued.data['items']] == [ids[1]] - owned = await call(app, 'discovery.lexical_search', {**args, 'scope': {'subject': subject}, 'limit': 10}, - key=key, subject=subject, contract_version=5) - assert owned.status == 'ok' and {item['id'] for item in owned.data['items']} == set(ids), wire(owned) - bad_type = await call(app, 'discovery.lexical_search', {**args, 'scope': {'org': subject}}, contract_version=5) + owned = await call( + app, + 'discovery.lexical_search', + {**args, 'scope': {'subject': subject}, 'limit': 10}, + key=key, + subject=subject, + contract_version=5, + ) + assert owned.status == 'ok' and {item['id'] for item in owned.data['items']} == set(ids), wire( + owned + ) + bad_type = await call( + app, 'discovery.lexical_search', {**args, 'scope': {'org': subject}}, contract_version=5 + ) assert bad_type.status == 'error' and bad_type.error.code == 'invalid_search_scope' - group = await call(app, 'group.create', {'name': 'typed-search-group'}, key=key, subject=subject) + group = await call( + app, 'group.create', {'name': 'typed-search-group'}, key=key, subject=subject + ) assert group.status == 'ok', wire(group) group_id = group.resources[0].id - grouped = await call(app, 'content.chgrp', {'id': ids[0], 'group': group_id}, - key=key, subject=subject, expected=((ids[0], posts[0].data['generation']),)) + grouped = await call( + app, + 'content.chgrp', + {'id': ids[0], 'group': group_id}, + key=key, + subject=subject, + expected=((ids[0], posts[0].data['generation']),), + ) assert grouped.status == 'ok', wire(grouped) - group_search = await call(app, 'discovery.lexical_search', - {**args, 'scope': {'org': group_id}, 'limit': 10}, key=key, subject=subject, contract_version=5) - assert group_search.status == 'ok' and [item['id'] for item in group_search.data['items']] == [ids[0]], wire(group_search) - async with httpx.AsyncClient(transport=httpx.ASGITransport(app=create_app(app)), - base_url=app.settings.service_url) as http: - query = await http.get('/_search', params={'scope': canonical(scope).decode(), 'terms': 'scopev5needle'}) - path = await http.get('/_s/q/5/s/' + quote(canonical(scope).decode(), safe=',') + '/t/scopev5needle') + group_search = await call( + app, + 'discovery.lexical_search', + {**args, 'scope': {'org': group_id}, 'limit': 10}, + key=key, + subject=subject, + contract_version=5, + ) + assert group_search.status == 'ok' and [item['id'] for item in group_search.data['items']] == [ + ids[0] + ], wire(group_search) + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=create_app(app)), base_url=app.settings.service_url + ) as http: + query = await http.get( + '/_search', params={'scope': canonical(scope).decode(), 'terms': 'scopev5needle'} + ) + path = await http.get( + '/_s/q/5/s/' + quote(canonical(scope).decode(), safe=',') + '/t/scopev5needle' + ) assert query.status_code == path.status_code == 200, (query.text, path.text) assert query.content == path.content - descriptor=canonical({'version':1,'kind':'search','arguments':args}) - opened=await call(app,'transfer.open',{'direction':'upload','size':len(descriptor), - 'digest':digest(descriptor),'media_type':'application/vnd.msg.read-query+json'}, - key=key,subject=subject) - tid=opened.data['transfer_id'] - await call(app,'transfer.part_put',{'transfer_id':tid,'offset':0, - 'data':b64(descriptor),'digest':digest(descriptor)},key=key,subject=subject) - await call(app,'transfer.seal',{'transfer_id':tid,'final_size':len(descriptor), - 'final_digest':digest(descriptor)},key=key,subject=subject) - sealed=await call(app,'transfer.query_seal',{'transfer_id':tid},key=key,subject=subject) - assert sealed.status=='ok',wire(sealed) - first=await call(app,'transfer.query_get',{'query_ref':sealed.data['query_ref']},key=key,subject=subject) - assert first.status=='ok' and first.data['cursor'],wire(first) - second=await call(app,'transfer.query_get',{'query_ref':sealed.data['query_ref'], - 'cursor':first.data['cursor']},key=key,subject=subject) - assert second.status=='ok' and [item['id'] for item in second.data['items']]==[ids[1]],wire(second) - changed = await call(app, 'content.chmod', {'id': ids[1], 'mode': '0600'}, key=key, subject=subject, - expected=((ids[1], posts[1].data['generation']),)) + descriptor = canonical({'version': 1, 'kind': 'search', 'arguments': args}) + opened = await call( + app, + 'transfer.open', + { + 'direction': 'upload', + 'size': len(descriptor), + 'digest': digest(descriptor), + 'media_type': 'application/vnd.msg.read-query+json', + }, + key=key, + subject=subject, + ) + tid = opened.data['transfer_id'] + await call( + app, + 'transfer.part_put', + {'transfer_id': tid, 'offset': 0, 'data': b64(descriptor), 'digest': digest(descriptor)}, + key=key, + subject=subject, + ) + await call( + app, + 'transfer.seal', + {'transfer_id': tid, 'final_size': len(descriptor), 'final_digest': digest(descriptor)}, + key=key, + subject=subject, + ) + sealed = await call(app, 'transfer.query_seal', {'transfer_id': tid}, key=key, subject=subject) + assert sealed.status == 'ok', wire(sealed) + first = await call( + app, 'transfer.query_get', {'query_ref': sealed.data['query_ref']}, key=key, subject=subject + ) + assert first.status == 'ok' and first.data['cursor'], wire(first) + second = await call( + app, + 'transfer.query_get', + {'query_ref': sealed.data['query_ref'], 'cursor': first.data['cursor']}, + key=key, + subject=subject, + ) + assert second.status == 'ok' and [item['id'] for item in second.data['items']] == [ids[1]], ( + wire(second) + ) + changed = await call( + app, + 'content.chmod', + {'id': ids[1], 'mode': '0600'}, + key=key, + subject=subject, + expected=((ids[1], posts[1].data['generation']),), + ) assert changed.status == 'ok', wire(changed) - revoked = await call(app, 'discovery.lexical_search', {'cursor': result.data['cursor']}, contract_version=5) + revoked = await call( + app, 'discovery.lexical_search', {'cursor': result.data['cursor']}, contract_version=5 + ) assert revoked.status == 'error' old = await call(app, 'discovery.lexical_search', args, contract_version=4) assert old.status == 'error' diff --git a/tests/test_search_spelling.py b/tests/test_search_spelling.py index 5fb2ad19..fda81354 100644 --- a/tests/test_search_spelling.py +++ b/tests/test_search_spelling.py @@ -1,4 +1,5 @@ """Spelling hints are explicit, deterministic and limited to visible names.""" + import httpx import pytest from test_lexical_search_grep import business_state @@ -14,49 +15,85 @@ async def test_spelling_is_opt_in_and_rechecks_visible_name_dictionary(installed key, subject, _ = await register(app, 'spelling-owner') posts = [] for name in ('gadget', 'gadger'): - created = await call(app, 'content.post_create', - {'parent': '/main', 'name': name, 'body': 'a product'}, key=key, subject=subject) + created = await call( + app, + 'content.post_create', + {'parent': '/main', 'name': name, 'body': 'a product'}, + key=key, + subject=subject, + ) assert created.status == 'ok', wire(created) posts.append(created) - private = await call(app, 'content.chmod', {'id': posts[1].resources[0].id, 'mode': '0600'}, - key=key, subject=subject, expected=((posts[1].resources[0].id, posts[1].data['generation']),)) + private = await call( + app, + 'content.chmod', + {'id': posts[1].resources[0].id, 'mode': '0600'}, + key=key, + subject=subject, + expected=((posts[1].resources[0].id, posts[1].data['generation']),), + ) assert private.status == 'ok', wire(private) args = {'scope': '/main', 'terms': 'gadge', 'field': 'name'} before = await business_state(app) ordinary = await call(app, 'discovery.lexical_search', args, contract_version=5) assert ordinary.status == 'ok' and 'spelling' not in ordinary.data - hinted = await call(app, 'discovery.lexical_search', {**args, 'spell': True}, contract_version=5) + hinted = await call( + app, 'discovery.lexical_search', {**args, 'spell': True}, contract_version=5 + ) assert hinted.status == 'ok', wire(hinted) - assert wire(hinted.data['spelling']) == {'source': 'name', 'terms': [ - {'term': 'gadge', 'suggestions': [{'value': 'gadget', 'distance': 1, 'count': 1}]}]} - assert [row['id'] for row in hinted.data['items']] == [row['id'] for row in ordinary.data['items']] + assert wire(hinted.data['spelling']) == { + 'source': 'name', + 'terms': [ + {'term': 'gadge', 'suggestions': [{'value': 'gadget', 'distance': 1, 'count': 1}]} + ], + } + assert [row['id'] for row in hinted.data['items']] == [ + row['id'] for row in ordinary.data['items'] + ] old = await call(app, 'discovery.lexical_search', {**args, 'spell': True}, contract_version=4) assert old.status == 'error' - short = await call(app, 'discovery.lexical_search', {**args, 'terms': 'g', 'spell': True}, contract_version=5) + short = await call( + app, 'discovery.lexical_search', {**args, 'terms': 'g', 'spell': True}, contract_version=5 + ) assert short.status == 'error' and short.error.code == 'query_cost_exceeded' - async with httpx.AsyncClient(transport=httpx.ASGITransport(app=create_app(app)), - base_url=app.settings.service_url) as http: + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=create_app(app)), base_url=app.settings.service_url + ) as http: query = await http.get('/_search', params={**args, 'spell': '1'}) path = await http.get('/_s/q/5/s/%2Fmain/t/gadge/f/n/sp/1') assert query.status_code == path.status_code == 200, (query.text, path.text) assert query.content == path.content assert await business_state(app) == before - revoked = await call(app, 'content.chmod', {'id': posts[0].resources[0].id, 'mode': '0600'}, - key=key, subject=subject, expected=((posts[0].resources[0].id, posts[0].data['generation']),)) + revoked = await call( + app, + 'content.chmod', + {'id': posts[0].resources[0].id, 'mode': '0600'}, + key=key, + subject=subject, + expected=((posts[0].resources[0].id, posts[0].data['generation']),), + ) assert revoked.status == 'ok', wire(revoked) - hidden = await call(app, 'discovery.lexical_search', {**args, 'spell': True}, contract_version=5) + hidden = await call( + app, 'discovery.lexical_search', {**args, 'spell': True}, contract_version=5 + ) assert hidden.status == 'ok' and not hidden.data['items'] - assert wire(hidden.data['spelling']) == {'source': 'name', 'terms': [{'term': 'gadge', 'suggestions': []}]} + assert wire(hidden.data['spelling']) == { + 'source': 'name', + 'terms': [{'term': 'gadge', 'suggestions': []}], + } @pytest.mark.asyncio async def test_v5_registered_collaboration_relation_names_select_same_http_contract(installed): app, _ = installed - async with httpx.AsyncClient(transport=httpx.ASGITransport(app=create_app(app)), - base_url=app.settings.service_url) as http: + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=create_app(app)), base_url=app.settings.service_url + ) as http: for kind in ('reference', 'state', 'target', 'content'): - query = await http.get('/_search', params={ - 'scope': '/main', 'terms': 'relationneedle', 'relation_type': kind}) + query = await http.get( + '/_search', + params={'scope': '/main', 'terms': 'relationneedle', 'relation_type': kind}, + ) path = await http.get('/_s/q/5/s/%2Fmain/t/relationneedle/rt/' + kind) assert query.status_code == path.status_code == 200, (query.text, path.text) assert query.content == path.content diff --git a/tests/test_search_suggestions.py b/tests/test_search_suggestions.py index 2357d7bc..49a75ef7 100644 --- a/tests/test_search_suggestions.py +++ b/tests/test_search_suggestions.py @@ -1,111 +1,172 @@ """Search suggestions are a bounded projection of current readable matches.""" + from datetime import timedelta import httpx import pytest +from test_service import NOW, call, register -from msg.core.codec import b64,canonical,digest,wire +from msg.core.codec import b64, canonical, digest, wire from msg.core.requests import request_for from msg.transports.http import create_app -from test_service import NOW,call,register @pytest.mark.asyncio async def test_v4_http_and_query_ref_use_same_authorized_suggestions(installed): - app,_=installed - key,user,_=await register(app,'suggest-http-owner') - created=await call(app,'content.file_put',{'parent':'/main', - 'name':'suggestpath-one.txt','data':b64(b'body'),'media_type':'text/plain'}, - key=key,subject=user) - assert created.status=='ok' - query='/_search?scope=%2Fmain&terms=suggestpath&field=name&suggest=1' - path='/_s/q/4/s/%2Fmain/t/suggestpath/f/n/sg/1' - async with httpx.AsyncClient(transport=httpx.ASGITransport(app=create_app(app)), - base_url=app.settings.service_url) as http: - left=await http.get(query) - right=await http.get(path) - assert left.status_code==right.status_code==200,(left.text,right.text) - assert left.content==right.content - assert left.json()['suggestions']==[{'value':'suggestpath-one','count':1}] - args={'scope':'/main','terms':'suggestpath','field':'name','suggest':True} - descriptor=canonical({'version':1,'kind':'search','arguments':args}) - opened=await call(app,'transfer.open',{'direction':'upload','size':len(descriptor), - 'digest':digest(descriptor),'media_type':'application/vnd.msg.read-query+json'}, - key=key,subject=user) - tid=opened.data['transfer_id'] - await call(app,'transfer.part_put',{'transfer_id':tid,'offset':0, - 'data':b64(descriptor),'digest':digest(descriptor)},key=key,subject=user) - await call(app,'transfer.seal',{'transfer_id':tid,'final_size':len(descriptor), - 'final_digest':digest(descriptor)},key=key,subject=user) - sealed=await call(app,'transfer.query_seal',{'transfer_id':tid},key=key,subject=user) - assert sealed.status=='ok',wire(sealed) - token=sealed.data['query_ref'] - proof=request_for('transfer.query_get',{'query_ref':token},app.settings.service_url, - signer=key,subject=user,expires_at=NOW+timedelta(seconds=60)) - async with httpx.AsyncClient(transport=httpx.ASGITransport(app=create_app(app)), - base_url=app.settings.service_url) as http: - resolved=await http.get('/_r/q/'+token+'/p/'+b64(canonical(proof))) - assert resolved.status_code==200,resolved.text - assert resolved.json()['suggestions']==left.json()['suggestions'] + app, _ = installed + key, user, _ = await register(app, 'suggest-http-owner') + created = await call( + app, + 'content.file_put', + { + 'parent': '/main', + 'name': 'suggestpath-one.txt', + 'data': b64(b'body'), + 'media_type': 'text/plain', + }, + key=key, + subject=user, + ) + assert created.status == 'ok' + query = '/_search?scope=%2Fmain&terms=suggestpath&field=name&suggest=1' + path = '/_s/q/4/s/%2Fmain/t/suggestpath/f/n/sg/1' + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=create_app(app)), base_url=app.settings.service_url + ) as http: + left = await http.get(query) + right = await http.get(path) + assert left.status_code == right.status_code == 200, (left.text, right.text) + assert left.content == right.content + assert left.json()['suggestions'] == [{'value': 'suggestpath-one', 'count': 1}] + args = {'scope': '/main', 'terms': 'suggestpath', 'field': 'name', 'suggest': True} + descriptor = canonical({'version': 1, 'kind': 'search', 'arguments': args}) + opened = await call( + app, + 'transfer.open', + { + 'direction': 'upload', + 'size': len(descriptor), + 'digest': digest(descriptor), + 'media_type': 'application/vnd.msg.read-query+json', + }, + key=key, + subject=user, + ) + tid = opened.data['transfer_id'] + await call( + app, + 'transfer.part_put', + {'transfer_id': tid, 'offset': 0, 'data': b64(descriptor), 'digest': digest(descriptor)}, + key=key, + subject=user, + ) + await call( + app, + 'transfer.seal', + {'transfer_id': tid, 'final_size': len(descriptor), 'final_digest': digest(descriptor)}, + key=key, + subject=user, + ) + sealed = await call(app, 'transfer.query_seal', {'transfer_id': tid}, key=key, subject=user) + assert sealed.status == 'ok', wire(sealed) + token = sealed.data['query_ref'] + proof = request_for( + 'transfer.query_get', + {'query_ref': token}, + app.settings.service_url, + signer=key, + subject=user, + expires_at=NOW + timedelta(seconds=60), + ) + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=create_app(app)), base_url=app.settings.service_url + ) as http: + resolved = await http.get('/_r/q/' + token + '/p/' + b64(canonical(proof))) + assert resolved.status_code == 200, resolved.text + assert resolved.json()['suggestions'] == left.json()['suggestions'] @pytest.mark.asyncio async def test_v4_suggestions_count_only_matching_readable_names_and_recheck_page(installed): - app,_=installed - key,user,_=await register(app,'suggest-owner') - created=[] - for name in ('suggestible-one.txt','suggestible-two.txt','suggestible-secret.txt'): - result=await call(app,'content.file_put',{'parent':'/main','name':name, - 'data':b64(b'body'),'media_type':'text/plain'},key=key,subject=user) - assert result.status=='ok',wire(result) + app, _ = installed + key, user, _ = await register(app, 'suggest-owner') + created = [] + for name in ('suggestible-one.txt', 'suggestible-two.txt', 'suggestible-secret.txt'): + result = await call( + app, + 'content.file_put', + {'parent': '/main', 'name': name, 'data': b64(b'body'), 'media_type': 'text/plain'}, + key=key, + subject=user, + ) + assert result.status == 'ok', wire(result) created.append(result) - hidden=created[2] - locked=await call(app,'content.chmod',{'id':hidden.resources[0].id,'mode':'0600'}, - key=key,subject=user, - expected=((hidden.resources[0].id,hidden.data['generation']),)) - assert locked.status=='ok',wire(locked) - args={'scope':'/main','terms':'suggest','field':'name','limit':1,'suggest':True} - first=await call(app,'discovery.lexical_search',args,contract_version=4) - assert first.status=='ok',wire(first) - assert wire(first)['data']['suggestions']==[ - {'value':'suggestible-one','count':1}, - {'value':'suggestible-two','count':1}] + hidden = created[2] + locked = await call( + app, + 'content.chmod', + {'id': hidden.resources[0].id, 'mode': '0600'}, + key=key, + subject=user, + expected=((hidden.resources[0].id, hidden.data['generation']),), + ) + assert locked.status == 'ok', wire(locked) + args = {'scope': '/main', 'terms': 'suggest', 'field': 'name', 'limit': 1, 'suggest': True} + first = await call(app, 'discovery.lexical_search', args, contract_version=4) + assert first.status == 'ok', wire(first) + assert wire(first)['data']['suggestions'] == [ + {'value': 'suggestible-one', 'count': 1}, + {'value': 'suggestible-two', 'count': 1}, + ] assert first.data['cursor'] - old=await call(app,'discovery.lexical_search',{'cursor':first.data['cursor']}, - contract_version=3) - assert old.status=='error' and old.error.code=='cursor_query_mismatch' - first_id=first.data['items'][0]['id'] - newly_hidden=next(item for item in created[:2] - if item.resources[0].id!=first_id) - locked=await call(app,'content.chmod',{'id':newly_hidden.resources[0].id,'mode':'0600'}, - key=key,subject=user, - expected=((newly_hidden.resources[0].id,newly_hidden.data['generation']),)) - assert locked.status=='ok',wire(locked) - continued=await call(app,'discovery.lexical_search', - {'cursor':first.data['cursor']},contract_version=4) - assert continued.status=='ok',wire(continued) - expected_word=next(name[:-4] for name,item in zip( - ('suggestible-one.txt','suggestible-two.txt'),created[:2]) - if item.resources[0].id==first_id) - assert wire(continued)['data']['suggestions']==[{'value':expected_word,'count':1}] - assert list(continued.data['items'])==[] - owner=await call(app,'discovery.lexical_search',args,key=key,subject=user, - contract_version=4) - assert owner.status=='ok',wire(owner) - assert len(owner.data['suggestions'])==3 + old = await call( + app, 'discovery.lexical_search', {'cursor': first.data['cursor']}, contract_version=3 + ) + assert old.status == 'error' and old.error.code == 'cursor_query_mismatch' + first_id = first.data['items'][0]['id'] + newly_hidden = next(item for item in created[:2] if item.resources[0].id != first_id) + locked = await call( + app, + 'content.chmod', + {'id': newly_hidden.resources[0].id, 'mode': '0600'}, + key=key, + subject=user, + expected=((newly_hidden.resources[0].id, newly_hidden.data['generation']),), + ) + assert locked.status == 'ok', wire(locked) + continued = await call( + app, 'discovery.lexical_search', {'cursor': first.data['cursor']}, contract_version=4 + ) + assert continued.status == 'ok', wire(continued) + expected_word = next( + name[:-4] + for name, item in zip( + ('suggestible-one.txt', 'suggestible-two.txt'), created[:2], strict=True + ) + if item.resources[0].id == first_id + ) + assert wire(continued)['data']['suggestions'] == [{'value': expected_word, 'count': 1}] + assert list(continued.data['items']) == [] + owner = await call( + app, 'discovery.lexical_search', args, key=key, subject=user, contract_version=4 + ) + assert owner.status == 'ok', wire(owner) + assert len(owner.data['suggestions']) == 3 @pytest.mark.asyncio async def test_v4_suggest_is_explicit_and_old_schemas_are_stable(installed): - app,_=installed - args={'scope':'/main','terms':'sug','field':'name'} - for version in (1,2,3): - result=await call(app,'discovery.lexical_search', - {**args,'suggest':True},contract_version=version) - assert result.status=='error',wire(result) - result=await call(app,'discovery.lexical_search',args,contract_version=4) - assert result.status=='ok',wire(result) + app, _ = installed + args = {'scope': '/main', 'terms': 'sug', 'field': 'name'} + for version in (1, 2, 3): + result = await call( + app, 'discovery.lexical_search', {**args, 'suggest': True}, contract_version=version + ) + assert result.status == 'error', wire(result) + result = await call(app, 'discovery.lexical_search', args, contract_version=4) + assert result.status == 'ok', wire(result) assert 'suggestions' not in result.data - too_short=await call(app,'discovery.lexical_search', - {**args,'terms':'s','suggest':True},contract_version=4) - assert too_short.status=='error' and too_short.error.code=='query_cost_exceeded' + too_short = await call( + app, 'discovery.lexical_search', {**args, 'terms': 's', 'suggest': True}, contract_version=4 + ) + assert too_short.status == 'error' and too_short.error.code == 'query_cost_exceeded' diff --git a/tests/test_security.py b/tests/test_security.py index 70308082..574b0631 100644 --- a/tests/test_security.py +++ b/tests/test_security.py @@ -1,60 +1,72 @@ from dataclasses import replace -from datetime import UTC, datetime, timedelta + import pytest +from test_foundation import item -from msg.core.codec import canonical, decode, wire +from msg.core.codec import canonical from msg.core.errors import Failure -from msg.security.crypto import Ed25519Signer, verify, seal_private_key, open_private_key -from msg.security.policy import selected_class, class_bits, scope_contains, grant_covers -from msg.core.models import Scope, CapabilityGrant, ResourceId +from msg.core.models import CapabilityGrant, Scope +from msg.security.crypto import Ed25519Signer, open_private_key, seal_private_key, verify +from msg.security.policy import class_bits, grant_covers, scope_contains, selected_class from msg.storage.sqlite import SqliteMetadataStore -from test_foundation import item def test_signatures_separate_purposes_and_reject_tampering(): - signer=Ed25519Signer.generate() - sig=signer.sign(b'hello',purpose='request') - verify(signer.public_key,b'hello',sig,purpose='request') - for payload,purpose in [(b'hello','receipt'),(b'bye','request')]: - with pytest.raises(Failure,match='invalid_signature'): - verify(signer.public_key,payload,sig,purpose=purpose) + signer = Ed25519Signer.generate() + sig = signer.sign(b'hello', purpose='request') + verify(signer.public_key, b'hello', sig, purpose='request') + for payload, purpose in [(b'hello', 'receipt'), (b'bye', 'request')]: + with pytest.raises(Failure, match='invalid_signature'): + verify(signer.public_key, payload, sig, purpose=purpose) def test_encrypted_key_roundtrip_and_wrong_pin(): - signer=Ed25519Signer.generate() - wrapped=seal_private_key(signer.private_bytes(), 'correct horse battery staple') - assert open_private_key(wrapped,'correct horse battery staple')==signer.private_bytes() + signer = Ed25519Signer.generate() + wrapped = seal_private_key(signer.private_bytes(), 'correct horse battery staple') + assert open_private_key(wrapped, 'correct horse battery staple') == signer.private_bytes() assert signer.private_bytes() not in canonical(wrapped) - with pytest.raises(Failure,match='invalid_pin'): - open_private_key(wrapped,'wrong horse battery staple') - altered=dict(wrapped,public_key='altered') - with pytest.raises(Failure): open_private_key(altered,'correct horse battery staple') - for pin in ['', '1234','password','00000000']: - with pytest.raises(Failure,match='weak_pin'): - seal_private_key(signer.private_bytes(),pin) + with pytest.raises(Failure, match='invalid_pin'): + open_private_key(wrapped, 'wrong horse battery staple') + altered = dict(wrapped, public_key='altered') + with pytest.raises(Failure): + open_private_key(altered, 'correct horse battery staple') + for pin in ['', '1234', 'password', '00000000']: + with pytest.raises(Failure, match='weak_pin'): + seal_private_key(signer.private_bytes(), pin) def test_mode_selection_is_not_a_union(): - r=replace(item(),owner='alice',group='staff',mode=0o046) - assert selected_class(r,'alice',{'staff'})=='owner' - assert class_bits(r.mode,'owner')==0 - assert selected_class(r,'bob',{'staff'})=='group' - assert selected_class(r,None,set())=='other' + r = replace(item(), owner='alice', group='staff', mode=0o046) + assert selected_class(r, 'alice', {'staff'}) == 'owner' + assert class_bits(r.mode, 'owner') == 0 + assert selected_class(r, 'bob', {'staff'}) == 'group' + assert selected_class(r, None, set()) == 'other' async def test_scopes_follow_current_parent_and_operation_exactness(tmp_path): - store=SqliteMetadataStore(tmp_path/'db') + store = SqliteMetadataStore(tmp_path / 'db') async with store.transaction(write=True) as tx: - for r in [item('root'),item('a','root'),item('b','root'),item('child','a')]: + for r in [item('root'), item('a', 'root'), item('b', 'root'), item('child', 'a')]: await tx.insert(r) - scope=Scope(resource_id='a',descendants=True) - assert await scope_contains(scope,'child',tx) - assert not await scope_contains(Scope(resource_id='a'),'child',tx) - grant=CapabilityGrant(capability='resource.certified_write',version=1,scope=scope, - operations=frozenset({'content.post_create@1'}),constraints={}) - assert await grant_covers(grant,'resource.certified_write','content.post_create@1','child',tx) - assert not await grant_covers(grant,'resource.write_override','content.post_create@1','child',tx) - assert not await grant_covers(grant,'resource.certified_write','content.post_edit@1','child',tx) - child=await tx.resource('child') - await tx.replace(replace(child,parent='b',generation=1),0) - assert not await scope_contains(scope,'child',tx) + scope = Scope(resource_id='a', descendants=True) + assert await scope_contains(scope, 'child', tx) + assert not await scope_contains(Scope(resource_id='a'), 'child', tx) + grant = CapabilityGrant( + capability='resource.certified_write', + version=1, + scope=scope, + operations=frozenset({'content.post_create@1'}), + constraints={}, + ) + assert await grant_covers( + grant, 'resource.certified_write', 'content.post_create@1', 'child', tx + ) + assert not await grant_covers( + grant, 'resource.write_override', 'content.post_create@1', 'child', tx + ) + assert not await grant_covers( + grant, 'resource.certified_write', 'content.post_edit@1', 'child', tx + ) + child = await tx.resource('child') + await tx.replace(replace(child, parent='b', generation=1), 0) + assert not await scope_contains(scope, 'child', tx) diff --git a/tests/test_security_patch_regressions.py b/tests/test_security_patch_regressions.py index 793a3a6d..bfa93139 100644 --- a/tests/test_security_patch_regressions.py +++ b/tests/test_security_patch_regressions.py @@ -1,4 +1,5 @@ """Real authenticated regressions for security PRs 121--128 on current main.""" + import asyncio import os from datetime import timedelta @@ -18,12 +19,30 @@ from msg.workers import maintenance -async def fresh_call(app, operation, arguments, *, key=None, subject=None, token=None, - rid=None, expected=(), return_fields=()): - packet = request_for(operation, arguments, app.settings.service_url, signer=key, - subject=subject, token=token, request_id=rid, expected=expected, - return_fields=return_fields, - expires_at=app.clock() + timedelta(seconds=120)) +async def fresh_call( + app, + operation, + arguments, + *, + key=None, + subject=None, + token=None, + rid=None, + expected=(), + return_fields=(), +): + packet = request_for( + operation, + arguments, + app.settings.service_url, + signer=key, + subject=subject, + token=token, + request_id=rid, + expected=expected, + return_fields=return_fields, + expires_at=app.clock() + timedelta(seconds=120), + ) return await app.executor.execute(packet) @@ -38,26 +57,47 @@ async def test_package_projection_failure_does_not_leak_pins(installed): app, _ = installed key, subject, _ = await register(app, 'security-deposit') listing = await call(app, 'store.listing_create', listing_args(), key=key, subject=subject) - source = await call(app, 'content.file_put', {'parent': '/@security-deposit/files', - 'name': 'secret.bin', 'media_type': 'application/octet-stream', 'data': b64(b'private')}, - key=key, subject=subject) + source = await call( + app, + 'content.file_put', + { + 'parent': '/@security-deposit/files', + 'name': 'secret.bin', + 'media_type': 'application/octet-stream', + 'data': b64(b'private'), + }, + key=key, + subject=subject, + ) assert listing.status == source.status == 'ok' - args = {'listing_id': listing.resources[0].id, - 'listing_revision': listing.resources[0].revision, - 'manifest': {}, 'payload_refs': [wire(source.resources[0])]} + args = { + 'listing_id': listing.resources[0].id, + 'listing_revision': listing.resources[0].revision, + 'manifest': {}, + 'payload_refs': [wire(source.resources[0])], + } before = {p for p in (app.contents.path / 'pins').glob('*/*') if p.is_file()} for _ in range(2): - result = await fresh_call(app, 'store.package_deposit', args, key=key, subject=subject, - rid='failed-deposit', return_fields=('not_a_field',)) + result = await fresh_call( + app, + 'store.package_deposit', + args, + key=key, + subject=subject, + rid='failed-deposit', + return_fields=('not_a_field',), + ) assert result.error.code == 'unknown_projection_field', wire(result) assert {p for p in (app.contents.path / 'pins').glob('*/*') if p.is_file()} == before async with app.metadata.transaction(write=False) as tx: assert tx.one('SELECT COUNT(*) FROM store_packages')[0] == 0 - accepted = await fresh_call(app, 'store.package_deposit', args, key=key, subject=subject, - rid='valid-deposit') + accepted = await fresh_call( + app, 'store.package_deposit', args, key=key, subject=subject, rid='valid-deposit' + ) assert accepted.status == 'ok', wire(accepted) - replay = await fresh_call(app, 'store.package_deposit', args, key=key, subject=subject, - rid='valid-deposit') + replay = await fresh_call( + app, 'store.package_deposit', args, key=key, subject=subject, rid='valid-deposit' + ) assert replay.status == 'ok' and replay.replayed async with app.metadata.transaction(write=False) as tx: assert tx.one('SELECT COUNT(*) FROM store_packages')[0] == 1 @@ -69,21 +109,29 @@ async def test_package_projection_failure_does_not_leak_pins(installed): async def test_achievement_concurrent_start_and_expired_replacement(installed): app, _ = installed key, subject, _ = await register(app, 'security-achievement') - results = await asyncio.gather(*(fresh_call(app, 'achievement.start', {}, key=key, - subject=subject, rid=f'start-{i}') for i in range(2))) + results = await asyncio.gather( + *( + fresh_call(app, 'achievement.start', {}, key=key, subject=subject, rid=f'start-{i}') + for i in range(2) + ) + ) successful = [r for r in results if r.status == 'ok'] denied = [r for r in results if r.status == 'error'] assert len(successful) == len(denied) == 1, [wire(r) for r in results] assert denied[0].error.code == 'achievement_ceremony_active' - replay = await fresh_call(app, 'achievement.start', {}, key=key, subject=subject, - rid=successful[0].request_id) + replay = await fresh_call( + app, 'achievement.start', {}, key=key, subject=subject, rid=successful[0].request_id + ) assert replay.status == 'ok' and replay.replayed set_clock(app, NOW + timedelta(seconds=CEREMONY_TTL_SECONDS + 1)) replaced = await fresh_call(app, 'achievement.start', {}, key=key, subject=subject) assert replaced.status == 'ok', wire(replaced) assert replaced.data['challenge_id'] != successful[0].data['challenge_id'] async with app.metadata.transaction(write=False) as tx: - assert tx.one('SELECT COUNT(*) FROM achievement_ceremonies WHERE subject=?', (subject,))[0] == 1 + assert ( + tx.one('SELECT COUNT(*) FROM achievement_ceremonies WHERE subject=?', (subject,))[0] + == 1 + ) @pytest.mark.asyncio @@ -96,7 +144,10 @@ async def test_achievement_failed_round_can_restart(installed): replacement = await call(app, 'achievement.start', {}, key=key, subject=subject) assert replacement.status == 'ok' async with app.metadata.transaction(write=False) as tx: - assert tx.one('SELECT COUNT(*) FROM achievement_ceremonies WHERE subject=?', (subject,))[0] == 1 + assert ( + tx.one('SELECT COUNT(*) FROM achievement_ceremonies WHERE subject=?', (subject,))[0] + == 1 + ) @pytest.mark.asyncio @@ -106,37 +157,79 @@ async def test_new_persistent_operations_obey_write_pause(installed, operation): key, subject, _ = await register(app, 'security-pause') arguments, expected = {}, () if operation == 'hosting.preview': - website = await call(app, 'hosting.create', {'parent': '/@security-pause', 'name': 'site'}, - key=key, subject=subject) - source = await call(app, 'content.file_put', {'parent': '/@security-pause/files', - 'name': 'page.html', 'media_type': 'text/html', 'data': b64(b'hello')}, key=key, subject=subject) + website = await call( + app, + 'hosting.create', + {'parent': '/@security-pause', 'name': 'site'}, + key=key, + subject=subject, + ) + source = await call( + app, + 'content.file_put', + { + 'parent': '/@security-pause/files', + 'name': 'page.html', + 'media_type': 'text/html', + 'data': b64(b'hello'), + }, + key=key, + subject=subject, + ) assert website.status == source.status == 'ok' - arguments = {'id': website.resources[0].id, - 'entries': [{'path': 'index.html', 'source': wire(source.resources[0])}]} + arguments = { + 'id': website.resources[0].id, + 'entries': [{'path': 'index.html', 'source': wire(source.resources[0])}], + } expected = ((website.resources[0].id, website.data['generation']),) async with app.metadata.transaction(write=True) as tx: tx.set_setting('runtime_config', {'accept_writes': False, 'cleanup_enabled': True}) - before = tuple(tx.one(f'SELECT COUNT(*) FROM {table}')[0] - for table in ('resources', 'revisions', 'achievement_ceremonies')) - result = await fresh_call(app, operation, arguments, key=key, subject=subject, expected=expected) + before = tuple( + tx.one(f'SELECT COUNT(*) FROM {table}')[0] + for table in ('resources', 'revisions', 'achievement_ceremonies') + ) + result = await fresh_call( + app, operation, arguments, key=key, subject=subject, expected=expected + ) assert result.status == 'error' and result.error.code == 'writes_paused', wire(result) async with app.metadata.transaction(write=False) as tx: - assert tuple(tx.one(f'SELECT COUNT(*) FROM {table}')[0] - for table in ('resources', 'revisions', 'achievement_ceremonies')) == before + assert ( + tuple( + tx.one(f'SELECT COUNT(*) FROM {table}')[0] + for table in ('resources', 'revisions', 'achievement_ceremonies') + ) + == before + ) @pytest.mark.asyncio async def test_hosting_preview_rejects_129_entries_before_storage(installed, monkeypatch): app, _ = installed key, subject, _ = await register(app, 'security-preview-limit') - website = await call(app, 'hosting.create', - {'parent': '/@security-preview-limit', 'name': 'site'}, key=key, subject=subject) - source = await call(app, 'content.file_put', {'parent': '/@security-preview-limit/files', - 'name': 'page.html', 'media_type': 'text/html', 'data': b64(b'hello')}, - key=key, subject=subject) + website = await call( + app, + 'hosting.create', + {'parent': '/@security-preview-limit', 'name': 'site'}, + key=key, + subject=subject, + ) + source = await call( + app, + 'content.file_put', + { + 'parent': '/@security-preview-limit/files', + 'name': 'page.html', + 'media_type': 'text/html', + 'data': b64(b'hello'), + }, + key=key, + subject=subject, + ) assert website.status == source.status == 'ok' - arguments = {'id': website.resources[0].id, 'entries': [ - {'path': f'{i}.html', 'source': wire(source.resources[0])} for i in range(129)]} + arguments = { + 'id': website.resources[0].id, + 'entries': [{'path': f'{i}.html', 'source': wire(source.resources[0])} for i in range(129)], + } expected = ((website.resources[0].id, website.data['generation']),) # A published wire schema is immutable. The runtime work bound is checked # after current authorization but before any source read or creation. @@ -144,29 +237,48 @@ async def test_hosting_preview_rejects_129_entries_before_storage(installed, mon assert 'maxItems' not in app.registry.schema(spec.input_schema)['properties']['entries'] app.registry.validate(spec.input_schema, arguments) async with app.metadata.transaction(write=False) as tx: - before = tuple(tx.one(f'SELECT COUNT(*) FROM {table}')[0] - for table in ('resources', 'revisions')) + before = tuple( + tx.one(f'SELECT COUNT(*) FROM {table}')[0] for table in ('resources', 'revisions') + ) + async def no_materialization(*args, **kwargs): pytest.fail('oversized preview touched the content store') + with monkeypatch.context() as patch: patch.setattr(app.contents, 'read_bytes', no_materialization) patch.setattr(app.contents, 'put_bytes', no_materialization) - denied = await fresh_call(app, 'hosting.preview', arguments, - key=key, subject=subject, expected=expected) - assert denied.status == 'error' and denied.error.code == 'too_many_preview_entries', wire(denied) + denied = await fresh_call( + app, 'hosting.preview', arguments, key=key, subject=subject, expected=expected + ) + assert denied.status == 'error' and denied.error.code == 'too_many_preview_entries', wire( + denied + ) async with app.metadata.transaction(write=False) as tx: - assert tuple(tx.one(f'SELECT COUNT(*) FROM {table}')[0] - for table in ('resources', 'revisions')) == before + assert ( + tuple( + tx.one(f'SELECT COUNT(*) FROM {table}')[0] for table in ('resources', 'revisions') + ) + == before + ) # An actual authorized 128-entry preview remains valid, not just its schema. - allowed = await fresh_call(app, 'hosting.preview', + allowed = await fresh_call( + app, + 'hosting.preview', {**arguments, 'entries': arguments['entries'][:128]}, - key=key, subject=subject, expected=expected) + key=key, + subject=subject, + expected=expected, + ) assert allowed.status == 'ok' and allowed.data['files'] == 128, wire(allowed) async def custody(app, handle): - created = await call(app, 'identity.custodial_create', {'handle': handle, - 'nonce': b64(os.urandom(32)), 'recovery_secret': b64(os.urandom(32))}, contract_version=2) + created = await call( + app, + 'identity.custodial_create', + {'handle': handle, 'nonce': b64(os.urandom(32)), 'recovery_secret': b64(os.urandom(32))}, + contract_version=2, + ) assert created.status == 'ok', wire(created) return created.data['subject_id'], (created.data['credential_id'], unb64(created.data['token'])) @@ -180,19 +292,30 @@ async def test_custodial_cleanup_preserves_accepted_upgrade_facts(installed, com private, recipient = generate_age_key() challenge = await begin(app, subject, token, signer, recipient) assert challenge.status == 'ok', wire(challenge) - result = await finish(app, subject, token, signer, private, challenge.data, - external_migrated=complete) + result = await finish( + app, subject, token, signer, private, challenge.data, external_migrated=complete + ) assert result.status == 'ok', wire(result) assert result.data['status'] == ('completed' if complete else 'pending_rewrap') async with app.metadata.transaction(write=False) as tx: - before = tx.one('SELECT * FROM custodial_upgrades WHERE id=?', (challenge.data['challenge_id'],)) + before = tx.one( + 'SELECT * FROM custodial_upgrades WHERE id=?', (challenge.data['challenge_id'],) + ) set_clock(app, NOW + timedelta(seconds=301)) await maintenance.run_maintenance(app, 'cleanup_expired', scheduled=True) async with app.metadata.transaction(write=False) as tx: - assert tx.one('SELECT * FROM custodial_upgrades WHERE id=?', (challenge.data['challenge_id'],)) == before + assert ( + tx.one('SELECT * FROM custodial_upgrades WHERE id=?', (challenge.data['challenge_id'],)) + == before + ) if complete: - recovered = await fresh_call(app, 'identity.custodial_upgrade_result', - {'challenge_id': challenge.data['challenge_id']}, key=signer, subject=subject) + recovered = await fresh_call( + app, + 'identity.custodial_upgrade_result', + {'challenge_id': challenge.data['challenge_id']}, + key=signer, + subject=subject, + ) assert recovered.status == 'ok' and recovered.data['status'] == 'completed', wire(recovered) @@ -207,7 +330,9 @@ async def test_expired_unaccepted_custodial_challenges_are_reclaimed(installed): async with app.metadata.transaction(write=False) as tx: assert tx.one('SELECT COUNT(*) FROM custodial_upgrades WHERE subject=?', (subject,))[0] == 0 assert (await tx.subject(subject)).kind == 'custodial' - assert tx.one('SELECT status FROM custodial_vault WHERE subject=?', (subject,))[0] == 'active' + assert ( + tx.one('SELECT status FROM custodial_vault WHERE subject=?', (subject,))[0] == 'active' + ) @pytest.mark.asyncio @@ -216,15 +341,22 @@ async def test_todo_sweep_has_bounded_content_reads_and_fair_cursor(installed, m key, subject, _ = await register(app, 'security-todos') monkeypatch.setattr(maintenance, 'TODO_DELIVERY_BATCH_SIZE', 3, raising=False) for i in range(7): - created = await call(app, 'identity.todo_put', {'name': f'item-{i}', 'title': f'Todo {i}', - 'due_at': wire(NOW - timedelta(hours=1))}, key=key, subject=subject) + created = await call( + app, + 'identity.todo_put', + {'name': f'item-{i}', 'title': f'Todo {i}', 'due_at': wire(NOW - timedelta(hours=1))}, + key=key, + subject=subject, + ) assert created.status == 'ok', wire(created) reads = 0 original = app.contents.read_bytes + async def observed_read(*args, **kwargs): nonlocal reads reads += 1 return await original(*args, **kwargs) + monkeypatch.setattr(app.contents, 'read_bytes', observed_read) for _ in range(3): reads = 0 @@ -240,17 +372,29 @@ async def test_expired_transfer_releases_only_its_pins(installed, sealed): app, _ = installed key, subject, _ = await register(app, 'security-upload') data = b'abandoned or published upload' - opened = await call(app, 'transfer.open', {'direction': 'upload', 'size': len(data)}, key=key, subject=subject) + opened = await call( + app, 'transfer.open', {'direction': 'upload', 'size': len(data)}, key=key, subject=subject + ) assert opened.status == 'ok', wire(opened) tid = opened.data['transfer_id'] - uploaded = await call(app, 'transfer.part_put', {'transfer_id': tid, 'offset': 0, - 'data': b64(data), 'digest': digest(data)}, key=key, subject=subject) + uploaded = await call( + app, + 'transfer.part_put', + {'transfer_id': tid, 'offset': 0, 'data': b64(data), 'digest': digest(data)}, + key=key, + subject=subject, + ) assert uploaded.status == 'ok', wire(uploaded) blob = decode(BlobRef, uploaded.data['chunk']['content']) assert await app.contents.pinned(blob, tid + ':0') if sealed: - result = await call(app, 'transfer.seal', {'transfer_id': tid, 'final_size': len(data), - 'final_digest': digest(data)}, key=key, subject=subject) + result = await call( + app, + 'transfer.seal', + {'transfer_id': tid, 'final_size': len(data), 'final_digest': digest(data)}, + key=key, + subject=subject, + ) assert result.status == 'ok', wire(result) async with app.metadata.transaction(write=False) as tx: revision = await tx.revision(result.resources[0]) diff --git a/tests/test_security_rollback_fences.py b/tests/test_security_rollback_fences.py index 645957a6..ff05008b 100644 --- a/tests/test_security_rollback_fences.py +++ b/tests/test_security_rollback_fences.py @@ -1,4 +1,5 @@ """Rollback compensation remains serialized with real independent writers.""" + import asyncio import sqlite3 @@ -14,8 +15,11 @@ def stores(request, tmp_path): if request.param == 'postgres': dsn = request.getfixturevalue('pg_dsn') - return (PostgresMetadataStore(dsn), - PostgresMetadataStore(dsn, initialize=False), request.param) + return ( + PostgresMetadataStore(dsn), + PostgresMetadataStore(dsn, initialize=False), + request.param, + ) path = tmp_path / 'fence.db' return SqliteMetadataStore(path), SqliteMetadataStore(path), request.param @@ -41,7 +45,7 @@ async def abort(): if sql_failure: # Exercise automatic SQLite transaction abort as well as PG's # INERROR state, not only an application exception before ROLLBACK. - sql = ('INSERT OR ROLLBACK' if backend == 'sqlite' else 'INSERT') + sql = 'INSERT OR ROLLBACK' if backend == 'sqlite' else 'INSERT' tx.execute(sql + ' INTO schema_version VALUES (1)', write=True) raise ValueError('original failure') @@ -62,9 +66,13 @@ async def compete(): remained_blocked = not entered.is_set() finally: release.set() - results = await asyncio.wait_for(asyncio.gather( - *([first_task, second_task] if second_task else [first_task]), - return_exceptions=True), 5) + results = await asyncio.wait_for( + asyncio.gather( + *([first_task, second_task] if second_task else [first_task]), + return_exceptions=True, + ), + 5, + ) if sql_failure: assert isinstance(results[0], Failure) and results[0].code == 'constraint_conflict', results expected_cause = sqlite3.IntegrityError if backend == 'sqlite' else psycopg.IntegrityError @@ -80,7 +88,9 @@ async def compete(): @pytest.mark.parametrize('initial_cancel', [False, True]) -async def test_repeated_cancellation_cannot_interrupt_compensation(stores, tmp_path, initial_cancel): +async def test_repeated_cancellation_cannot_interrupt_compensation( + stores, tmp_path, initial_cancel +): store, other, _ = stores started, release = asyncio.Event(), asyncio.Event() pin = tmp_path / 'temporary-pin' @@ -106,7 +116,7 @@ async def abort(): await asyncio.sleep(0) finally: release.set() - result, = await asyncio.wait_for(asyncio.gather(task, return_exceptions=True), 5) + (result,) = await asyncio.wait_for(asyncio.gather(task, return_exceptions=True), 5) if initial_cancel: assert isinstance(result, asyncio.CancelledError) else: @@ -141,8 +151,9 @@ async def wait(): waiter.cancel() finally: release.set() - results = await asyncio.wait_for(asyncio.gather( - *([holder, waiter] if waiter else [holder]), return_exceptions=True), 5) + results = await asyncio.wait_for( + asyncio.gather(*([holder, waiter] if waiter else [holder]), return_exceptions=True), 5 + ) assert results[0] is None assert isinstance(results[1], asyncio.CancelledError), results async with second.transaction(write=True) as tx: @@ -197,7 +208,7 @@ async def upload(): waited_for_io = not task.done() finally: release.set() - result, = await asyncio.wait_for(asyncio.gather(task, return_exceptions=True), 5) + (result,) = await asyncio.wait_for(asyncio.gather(task, return_exceptions=True), 5) assert isinstance(result, asyncio.CancelledError), result assert waited_for_io, 'rollback overtook a still-running Git pin write' assert not await contents.pinned(blob, 'cancelled-upload') diff --git a/tests/test_security_storage_rollback.py b/tests/test_security_storage_rollback.py index 7b35dac6..3428e8bf 100644 --- a/tests/test_security_storage_rollback.py +++ b/tests/test_security_storage_rollback.py @@ -1,4 +1,5 @@ """Compensation retains savepoint ownership and never hides cancellation.""" + import asyncio import pytest @@ -16,8 +17,10 @@ async def test_nested_rollback_compensates_only_its_scope(installed, tmp_path, b app, _ = installed store = app.metadata if backend == 'postgres' else SqliteMetadataStore(tmp_path / 'nested.db') effects = [] + async def record(value): effects.append(value) + async with store.transaction(write=True) as outer: outer.on_rollback(lambda: record('outer')) with pytest.raises(ValueError, match='savepoint'): @@ -42,15 +45,20 @@ async def record(value): @pytest.mark.asyncio @pytest.mark.parametrize('backend', ['postgres', 'sqlite']) -async def test_compensation_failure_preserves_original_and_runs_remaining(installed, tmp_path, backend): +async def test_compensation_failure_preserves_original_and_runs_remaining( + installed, tmp_path, backend +): app, _ = installed store = app.metadata if backend == 'postgres' else SqliteMetadataStore(tmp_path / 'fail.db') effects = [] + async def record(value): effects.append(value) + async def fail(): effects.append('failure') raise OSError('injected') + original = ValueError('primary') with pytest.raises(ValueError) as captured: async with store.transaction(write=True) as tx: @@ -70,8 +78,10 @@ async def test_task_cancellation_rolls_back_sql_and_external_effect(installed, t store = app.metadata if backend == 'postgres' else SqliteMetadataStore(tmp_path / 'cancel.db') entered = asyncio.Event() external = tmp_path / 'external' + async def compensate(): external.unlink() + async def work(): async with store.transaction(write=True) as tx: tx.set_setting('cancelled', True) @@ -79,6 +89,7 @@ async def work(): external.write_text('uncommitted') entered.set() await asyncio.Event().wait() + task = asyncio.create_task(work()) await entered.wait() assert external.exists() @@ -97,19 +108,27 @@ async def test_failed_transfer_cleanup_restores_pin_and_metadata(installed, monk opened = await call(app, 'transfer.open', {'direction': 'upload'}, key=key, subject=subject) assert opened.status == 'ok', wire(opened) tid = opened.data['transfer_id'] - result = await call(app, 'transfer.part_put', {'transfer_id': tid, 'offset': 0, - 'data': b64(b'abcd'), 'digest': digest(b'abcd')}, key=key, subject=subject) + result = await call( + app, + 'transfer.part_put', + {'transfer_id': tid, 'offset': 0, 'data': b64(b'abcd'), 'digest': digest(b'abcd')}, + key=key, + subject=subject, + ) assert result.status == 'ok', wire(result) blob = decode(BlobRef, result.data['chunk']['content']) from datetime import timedelta from test_security_patch_regressions import set_clock from test_service import NOW + set_clock(app, NOW + timedelta(seconds=app.settings.transfer_ttl + 1)) unpin = app.contents.unpin + async def fail_after_unpin(*args): await unpin(*args) raise OSError('injected after external change') + monkeypatch.setattr(app.contents, 'unpin', fail_after_unpin) with pytest.raises(OSError, match='injected'): await maintenance.run_maintenance(app, 'cleanup_expired', scheduled=True) @@ -127,13 +146,17 @@ async def test_scheduled_collection_is_rate_limited_and_respects_cleanup_pause(i app, _ = installed async with app.metadata.transaction(write=True) as tx: tx.set_setting('runtime_config', {'cleanup_enabled': False}) - assert await maintenance.run_maintenance(app, 'collect_garbage', scheduled=True) == {'skipped': 'cleanup_disabled'} + assert await maintenance.run_maintenance(app, 'collect_garbage', scheduled=True) == { + 'skipped': 'cleanup_disabled' + } async with app.metadata.transaction(write=True) as tx: assert tx.setting('garbage_collection_last') is None tx.set_setting('runtime_config', {'cleanup_enabled': True}) result = await maintenance.run_maintenance(app, 'collect_garbage', scheduled=True) assert result['unreferenced_contents_removed'] == 0 - assert await maintenance.run_maintenance(app, 'collect_garbage', scheduled=True) == {'skipped': 'not_due'} + assert await maintenance.run_maintenance(app, 'collect_garbage', scheduled=True) == { + 'skipped': 'not_due' + } @pytest.mark.asyncio @@ -142,23 +165,42 @@ async def test_purged_sealed_output_does_not_break_expiry_or_collection(installe from test_security_patch_regressions import set_clock from test_service import NOW + app, _ = installed key, subject, _ = await register(app, 'security-purged-upload') data = b'unreferenced after explicit purge' - opened = await call(app, 'transfer.open', {'direction': 'upload', 'size': len(data)}, key=key, subject=subject) + opened = await call( + app, 'transfer.open', {'direction': 'upload', 'size': len(data)}, key=key, subject=subject + ) assert opened.status == 'ok', wire(opened) tid = opened.data['transfer_id'] - uploaded = await call(app, 'transfer.part_put', {'transfer_id': tid, 'offset': 0, - 'data': b64(data), 'digest': digest(data)}, key=key, subject=subject) + uploaded = await call( + app, + 'transfer.part_put', + {'transfer_id': tid, 'offset': 0, 'data': b64(data), 'digest': digest(data)}, + key=key, + subject=subject, + ) assert uploaded.status == 'ok', wire(uploaded) - sealed = await call(app, 'transfer.seal', {'transfer_id': tid, 'final_size': len(data), - 'final_digest': digest(data)}, key=key, subject=subject) + sealed = await call( + app, + 'transfer.seal', + {'transfer_id': tid, 'final_size': len(data), 'final_digest': digest(data)}, + key=key, + subject=subject, + ) assert sealed.status == 'ok', wire(sealed) async with app.metadata.transaction(write=True) as tx: revision = await tx.revision(sealed.resources[0]) resource = await tx.resource(sealed.resources[0].id) - await maintenance.purge_revisions(app, tx, resource, actor=subject, - request_id='test-explicit-purge', reason='retention_expired') + await maintenance.purge_revisions( + app, + tx, + resource, + actor=subject, + request_id='test-explicit-purge', + reason='retention_expired', + ) assert await app.contents.pinned(revision.content, tid + ':sealed') set_clock(app, NOW + timedelta(seconds=app.settings.transfer_ttl + 1)) await maintenance.run_maintenance(app, 'cleanup_expired', scheduled=True) diff --git a/tests/test_security_topic_events.py b/tests/test_security_topic_events.py index acdc34ca..6d812ab1 100644 --- a/tests/test_security_topic_events.py +++ b/tests/test_security_topic_events.py @@ -1,4 +1,5 @@ """Topic history work is bounded by the requested page, not global activity.""" + import pytest from test_service import call, register @@ -10,26 +11,43 @@ async def test_topic_history_decodes_only_one_page(installed, monkeypatch): app, _ = installed key, subject, _ = await register(app, 'security-events') - made = await call(app, 'content.topic_create', {'parent': '/main', 'name': 'indexed-events'}, - key=key, subject=subject) + made = await call( + app, + 'content.topic_create', + {'parent': '/main', 'name': 'indexed-events'}, + key=key, + subject=subject, + ) assert made.status == 'ok', wire(made) topic = made.resources[0].id for policy in ('approval', 'open', 'invite', 'closed'): - result = await call(app, 'content.topic_policy_set', {'id': topic, 'membership_policy': policy}, - key=key, subject=subject) + result = await call( + app, + 'content.topic_policy_set', + {'id': topic, 'membership_policy': policy}, + key=key, + subject=subject, + ) assert result.status == 'ok', wire(result) for i in range(12): - posted = await call(app, 'content.post_create', {'parent': '/main', 'body': f'Unrelated {i}'}, - key=key, subject=subject) + posted = await call( + app, + 'content.post_create', + {'parent': '/main', 'body': f'Unrelated {i}'}, + key=key, + subject=subject, + ) assert posted.status == 'ok', wire(posted) decoded = 0 original = content.loads + def observed(raw): nonlocal decoded value = original(raw) if isinstance(value, dict) and 'request_id' in value and 'type' in value: decoded += 1 return value + monkeypatch.setattr(content, 'loads', observed) cursor = None seen = [] @@ -53,31 +71,52 @@ async def test_backfill_is_atomic_and_not_repeated(installed, tmp_path, backend) from msg.core.codec import canonical from msg.storage.sqlite import SqliteMetadataStore from msg.storage.topic_event_migration import EVENT_TYPES, MIGRATION_KEY, migrate_topic_events + app, _ = installed store = app.metadata if backend == 'postgres' else SqliteMetadataStore(tmp_path / 'events.db') assert set(EVENT_TYPES) == set(content.TOPIC_EVENT_CODES) async with store.transaction(write=True) as tx: tx.execute('DELETE FROM settings WHERE key=?', (MIGRATION_KEY,), write=True) - for identifier, kind, topic in [('historical', 'topic.create', 't_index'), - ('unrelated', 'content.post_create', 't_index'), - ('missing-topic', 'topic.create', None)]: + for identifier, kind, topic in [ + ('historical', 'topic.create', 't_index'), + ('unrelated', 'content.post_create', 't_index'), + ('missing-topic', 'topic.create', None), + ]: body = canonical({'type': kind, 'data': {'topic_id': topic}}).decode() tx.execute('INSERT INTO events (id,body) VALUES (?,?)', (identifier, body), write=True) # Constructor migrations run on the already-held physical schema lock. migrate_topic_events(tx._connection, postgres=backend == 'postgres') - assert tx.one('SELECT COUNT(*) FROM topic_event_projection WHERE topic=?', ('t_index',))[0] == 1 + assert ( + tx.one('SELECT COUNT(*) FROM topic_event_projection WHERE topic=?', ('t_index',))[0] + == 1 + ) assert tx.setting(MIGRATION_KEY) is True - tx.execute('INSERT INTO events (id,body) VALUES (?,?)', ('not-a-normal-write', - canonical({'type': 'topic.create', 'data': {'topic_id': 't_index'}}).decode()), write=True) + tx.execute( + 'INSERT INTO events (id,body) VALUES (?,?)', + ( + 'not-a-normal-write', + canonical({'type': 'topic.create', 'data': {'topic_id': 't_index'}}).decode(), + ), + write=True, + ) migrate_topic_events(tx._connection, postgres=backend == 'postgres') - assert tx.one('SELECT COUNT(*) FROM topic_event_projection WHERE topic=?', ('t_index',))[0] == 1 + assert ( + tx.one('SELECT COUNT(*) FROM topic_event_projection WHERE topic=?', ('t_index',))[0] + == 1 + ) # Both range shapes used by the rotating todo worker have supporting indexes. if backend == 'sqlite': - plan = tx.rows("EXPLAIN QUERY PLAN SELECT id,body FROM resources " - "WHERE type='todo' AND state='active' AND id>? ORDER BY id LIMIT ?", ('', 3)) + plan = tx.rows( + 'EXPLAIN QUERY PLAN SELECT id,body FROM resources ' + "WHERE type='todo' AND state='active' AND id>? ORDER BY id LIMIT ?", + ('', 3), + ) assert 'resources_type_state' in str(plan) else: tx.execute('SET LOCAL enable_seqscan=off') - plan = tx.rows("EXPLAIN SELECT id,body FROM resources WHERE type='todo' " - "AND state='active' AND id>? ORDER BY id LIMIT ?", ('', 3)) + plan = tx.rows( + "EXPLAIN SELECT id,body FROM resources WHERE type='todo' " + "AND state='active' AND id>? ORDER BY id LIMIT ?", + ('', 3), + ) assert 'resources_type_state' in str(plan) diff --git a/tests/test_security_transfer_rollback.py b/tests/test_security_transfer_rollback.py index b94832c8..81db17b7 100644 --- a/tests/test_security_transfer_rollback.py +++ b/tests/test_security_transfer_rollback.py @@ -1,4 +1,5 @@ """Authenticated upload failures must not leave permanent chunk/output pins.""" + import asyncio import pytest @@ -11,42 +12,59 @@ async def pins(app): - binary = sorted(str(p.relative_to(app.contents.path)) - for p in (app.contents.path / 'pins').glob('*/*') if p.is_file()) - git = await asyncio.to_thread(app.contents._run, 'for-each-ref', - '--format=%(refname)', 'refs/pins') + binary = sorted( + str(p.relative_to(app.contents.path)) + for p in (app.contents.path / 'pins').glob('*/*') + if p.is_file() + ) + git = await asyncio.to_thread( + app.contents._run, 'for-each-ref', '--format=%(refname)', 'refs/pins' + ) return set(binary) | set(git.splitlines()) @pytest.mark.parametrize('media_type', ['application/octet-stream', 'text/plain']) @pytest.mark.parametrize('operation', ['transfer.part_put', 'transfer.seal']) @pytest.mark.parametrize('failure', ['projection', 'partial_pin']) -async def test_failed_upload_removes_only_its_new_pins(installed, monkeypatch, operation, failure, media_type): +async def test_failed_upload_removes_only_its_new_pins( + installed, monkeypatch, operation, failure, media_type +): app, _ = installed key, subject, _ = await register(app, 'rollback-transfer') # The sealed digest differs from either chunk, so no shared chunk can hide a leak. data = b'first chunk' + b'second chunk' - opened = await call(app, 'transfer.open', {'direction': 'upload', 'size': len(data), 'media_type': media_type}, - key=key, subject=subject) + opened = await call( + app, + 'transfer.open', + {'direction': 'upload', 'size': len(data), 'media_type': media_type}, + key=key, + subject=subject, + ) assert opened.status == 'ok', wire(opened) tid = opened.data['transfer_id'] args = {'transfer_id': tid, 'offset': 0, 'data': b64(data), 'digest': digest(data)} lease = tid + ':0' if operation == 'transfer.seal': for offset, chunk in ((0, b'first chunk'), (11, b'second chunk')): - part = await call(app, 'transfer.part_put', {'transfer_id': tid, 'offset': offset, - 'data': b64(chunk), 'digest': digest(chunk)}, - key=key, subject=subject) + part = await call( + app, + 'transfer.part_put', + {'transfer_id': tid, 'offset': offset, 'data': b64(chunk), 'digest': digest(chunk)}, + key=key, + subject=subject, + ) assert part.status == 'ok', wire(part) args = {'transfer_id': tid, 'final_size': len(data), 'final_digest': digest(data)} lease = tid + ':sealed' before = await pins(app) async with app.metadata.transaction(write=False) as tx: transfer_before = await tx.transfer(tid) - counts_before = tuple(tx.one(f'SELECT COUNT(*) FROM {table}')[0] - for table in ('resources', 'revisions')) - chunks_before = tx.rows('SELECT offset,body FROM chunks WHERE transfer_id=? ORDER BY offset', - (tid,)) + counts_before = tuple( + tx.one(f'SELECT COUNT(*) FROM {table}')[0] for table in ('resources', 'revisions') + ) + chunks_before = tx.rows( + 'SELECT offset,body FROM chunks WHERE transfer_id=? ORDER BY offset', (tid,) + ) original_pin = app.contents.pin async def partial_pin(blob, lease_id): @@ -58,11 +76,20 @@ async def partial_pin(blob, lease_id): if failure == 'partial_pin': patch.setattr(app.contents, 'pin', partial_pin) for n in range(2): - result = await fresh_call(app, operation, args, key=key, subject=subject, - rid=f'failed-upload-{n}', - return_fields=('invalid_field',) if failure == 'projection' else ()) - projection_error = ('projection_unavailable' if operation == 'transfer.part_put' - else 'unknown_projection_field') + result = await fresh_call( + app, + operation, + args, + key=key, + subject=subject, + rid=f'failed-upload-{n}', + return_fields=('invalid_field',) if failure == 'projection' else (), + ) + projection_error = ( + 'projection_unavailable' + if operation == 'transfer.part_put' + else 'unknown_projection_field' + ) error = projection_error if failure == 'projection' else 'injected_pin_failure' assert result.status == 'error' and result.error.code == error, wire(result) after = await pins(app) @@ -70,11 +97,22 @@ async def partial_pin(blob, lease_id): assert not any(changes.values()), changes async with app.metadata.transaction(write=False) as tx: assert await tx.transfer(tid) == transfer_before - assert tuple(tx.one(f'SELECT COUNT(*) FROM {table}')[0] - for table in ('resources', 'revisions')) == counts_before - assert tx.rows('SELECT offset,body FROM chunks WHERE transfer_id=? ORDER BY offset', - (tid,)) == chunks_before - accepted = await fresh_call(app, operation, args, key=key, subject=subject, rid='accepted-upload') + assert ( + tuple( + tx.one(f'SELECT COUNT(*) FROM {table}')[0] + for table in ('resources', 'revisions') + ) + == counts_before + ) + assert ( + tx.rows( + 'SELECT offset,body FROM chunks WHERE transfer_id=? ORDER BY offset', (tid,) + ) + == chunks_before + ) + accepted = await fresh_call( + app, operation, args, key=key, subject=subject, rid='accepted-upload' + ) assert accepted.status == 'ok', wire(accepted) if operation == 'transfer.part_put': blob = decode(BlobRef, accepted.data['chunk']['content']) diff --git a/tests/test_service.py b/tests/test_service.py index 3dc43e30..d1c775e8 100644 --- a/tests/test_service.py +++ b/tests/test_service.py @@ -1,56 +1,97 @@ """End-to-end requests go through real signature verification and SQLite commits.""" -from dataclasses import replace -from datetime import UTC, datetime, timedelta + import hashlib import os +from datetime import UTC, datetime, timedelta from uuid import uuid4 import pytest -from msg.application import Application -from msg.admin.root import _provision, _approve_csr -from msg.config import write_example from msg.constants import ROOT_SUBJECT from msg.core.codec import b64, canonical, unb64, wire -from msg.core.requests import request_for, receipt_bytes -from msg.security.crypto import Ed25519Signer, subject_id, verify +from msg.core.requests import receipt_bytes, request_for from msg.security.age_keys import generate_age_key +from msg.security.crypto import Ed25519Signer, subject_id, verify NOW = datetime(2026, 9, 27, tzinfo=UTC) -def temporary_v3_args(*, nonce=None, recovery_secret=None, request_id=None, - signer=None, recipient=None): - nonce=nonce or b64(os.urandom(32)) - recovery_secret=recovery_secret or b64(os.urandom(32)) - request_id=request_id or uuid4().hex - signer=signer or Ed25519Signer.generate() - recipient=recipient or generate_age_key()[1] - subject='u_tmp_'+hashlib.sha256(unb64(nonce)).hexdigest()[:32] - proof=signer.sign(canonical({'subject_id':subject,'nonce':nonce, - 'public_key':b64(signer.public_key),'encryption_recipient':recipient, - 'request_id':request_id}),purpose='temporary-key-possession-v1') - return ({'nonce':nonce,'recovery_secret':recovery_secret, - 'public_key':b64(signer.public_key),'encryption_recipient':recipient, - 'possession_proof':wire(proof)},request_id,signer,recipient) - - -async def call(app, op, args, *, key=None, subject=None, certs=(), expected=(), rid=None, token=None, - contract_version=1): - packet = request_for(op, args, app.settings.service_url, - signer=key, subject=subject, certificates=certs, expected=expected, - request_id=rid, token=token, expires_at=NOW+timedelta(seconds=120), - contract_version=contract_version) +def temporary_v3_args( + *, nonce=None, recovery_secret=None, request_id=None, signer=None, recipient=None +): + nonce = nonce or b64(os.urandom(32)) + recovery_secret = recovery_secret or b64(os.urandom(32)) + request_id = request_id or uuid4().hex + signer = signer or Ed25519Signer.generate() + recipient = recipient or generate_age_key()[1] + subject = 'u_tmp_' + hashlib.sha256(unb64(nonce)).hexdigest()[:32] + proof = signer.sign( + canonical({ + 'subject_id': subject, + 'nonce': nonce, + 'public_key': b64(signer.public_key), + 'encryption_recipient': recipient, + 'request_id': request_id, + }), + purpose='temporary-key-possession-v1', + ) + return ( + { + 'nonce': nonce, + 'recovery_secret': recovery_secret, + 'public_key': b64(signer.public_key), + 'encryption_recipient': recipient, + 'possession_proof': wire(proof), + }, + request_id, + signer, + recipient, + ) + + +async def call( + app, + op, + args, + *, + key=None, + subject=None, + certs=(), + expected=(), + rid=None, + token=None, + contract_version=1, +): + packet = request_for( + op, + args, + app.settings.service_url, + signer=key, + subject=subject, + certificates=certs, + expected=expected, + request_id=rid, + token=token, + expires_at=NOW + timedelta(seconds=120), + contract_version=contract_version, + ) return await app.executor.execute(packet) async def register(app, handle): from msg.security.age_keys import generate_age_key + key = Ed25519Signer.generate() uid = subject_id(key.public_key) _, recipient = generate_age_key() - result = await call(app, 'identity.register', {'handle':handle,'public_key':b64(key.public_key), - 'encryption_recipient':recipient}, key=key, subject=uid, contract_version=2) + result = await call( + app, + 'identity.register', + {'handle': handle, 'public_key': b64(key.public_key), 'encryption_recipient': recipient}, + key=key, + subject=uid, + contract_version=2, + ) assert result.status == 'ok', wire(result) return key, uid, result.data['certificate_id'] @@ -59,25 +100,33 @@ async def register(app, handle): async def test_service_register_post_read_reply_receipt(installed): app, root = installed key, uid, cert = await register(app, 'alice') + async def invoke(op, args, **kw): return await call(app, op, args, key=key, subject=uid, certs=(cert,), **kw) - result = await invoke('content.post_create', {'parent':'/main','body':'A short message.'}, rid='post-once') + + result = await invoke( + 'content.post_create', {'parent': '/main', 'body': 'A short message.'}, rid='post-once' + ) assert result.status == 'ok', wire(result) verify(app.receipt_signer.public_key, receipt_bytes(result), result.receipt, purpose='receipt') - repeated = await invoke('content.post_create', {'parent':'/main','body':'A short message.'}, rid='post-once') + repeated = await invoke( + 'content.post_create', {'parent': '/main', 'body': 'A short message.'}, rid='post-once' + ) assert repeated.status == 'ok' and repeated.replayed - conflict = await invoke('content.post_create', {'parent':'/main','body':'Different.'}, rid='post-once') + conflict = await invoke( + 'content.post_create', {'parent': '/main', 'body': 'Different.'}, rid='post-once' + ) assert conflict.error.code == 'idempotency_conflict' rid = result.resources[0].id - read = await call(app, 'discovery.get', {'id':rid}) + read = await call(app, 'discovery.get', {'id': rid}) assert read.status == 'ok', wire(read) assert 'A short message.' in str(read.data) - reply = await invoke('discussion.reply', {'target':{'id':rid},'body':'Received.'}) + reply = await invoke('discussion.reply', {'target': {'id': rid}, 'body': 'Received.'}) assert reply.status == 'ok', wire(reply) async with app.metadata.transaction(write=False) as tx: assert (await tx.resource(reply.resources[0].id)).type == 'post' before = tx.one('SELECT COUNT(*) FROM events')[0] - await call(app, 'discovery.get', {'id':rid}) + await call(app, 'discovery.get', {'id': rid}) async with app.metadata.transaction(write=False) as tx: assert tx.one('SELECT COUNT(*) FROM events')[0] == before @@ -85,7 +134,7 @@ async def invoke(op, args, **kw): @pytest.mark.asyncio async def test_root_network_forbidden_even_cli_source(installed): app, root = installed - result = await call(app, 'discovery.get', {'id':'/private'}, key=root, subject=ROOT_SUBJECT) + result = await call(app, 'discovery.get', {'id': '/private'}, key=root, subject=ROOT_SUBJECT) assert result.status == 'error' and result.error.code == 'local_only', wire(result) assert not hasattr(app, 'root_signer') @@ -94,45 +143,88 @@ async def test_root_network_forbidden_even_cli_source(installed): async def test_default_deny_and_certgate(installed): app, _ = installed key, uid, cert = await register(app, 'bob') - for path in ('/private','/admins','/tools/curl'): - result = await call(app,'discovery.get',{'id':path},key=key,subject=uid,certs=(cert,)) - assert result.status == 'error', (path,wire(result)) - result = await call(app,'content.post_create',{'parent':'/certified','body':'gate'},key=key,subject=uid,certs=(cert,)) + for path in ('/private', '/admins', '/tools/curl'): + result = await call(app, 'discovery.get', {'id': path}, key=key, subject=uid, certs=(cert,)) + assert result.status == 'error', (path, wire(result)) + result = await call( + app, + 'content.post_create', + {'parent': '/certified', 'body': 'gate'}, + key=key, + subject=uid, + certs=(cert,), + ) assert result.error.code == 'certificate_gate', wire(result) @pytest.mark.asyncio async def test_temporary_rotation_and_upgrade_preserve_subject(installed): app, _ = installed - temporary_args,temporary_rid,new_key,recipient=temporary_v3_args() - result = await call(app,'identity.temporary',temporary_args, - rid=temporary_rid,contract_version=3) + temporary_args, temporary_rid, new_key, recipient = temporary_v3_args() + result = await call( + app, 'identity.temporary', temporary_args, rid=temporary_rid, contract_version=3 + ) assert result.status == 'ok', wire(result) uid = result.data['subject_id'] from msg.core.codec import unb64 + token = (result.data['credential_id'], unb64(result.data['token'])) - post = await call(app,'content.post_create',{'parent':'/tmp','body':'temporary'},subject=uid,token=token) + post = await call( + app, + 'content.post_create', + {'parent': '/tmp', 'body': 'temporary'}, + subject=uid, + token=token, + ) assert post.status == 'ok', wire(post) - forbidden = await call(app,'content.chmod',{'id':post.resources[0].id,'mode':'0777'},subject=uid,token=token, - expected=((post.resources[0].id,post.data['generation']),)) + forbidden = await call( + app, + 'content.chmod', + {'id': post.resources[0].id, 'mode': '0777'}, + subject=uid, + token=token, + expected=((post.resources[0].id, post.data['generation']),), + ) assert forbidden.error.code == 'credential_ceiling' - args = {'subject_id':uid,'public_key':b64(new_key.public_key),'handle':'promoted', - 'encryption_recipient':recipient} - upgrade = await call(app,'identity.upgrade',{'handle':'promoted','public_key':args['public_key'], - 'encryption_recipient':recipient, - 'possession_proof':wire(new_key.sign(canonical(args),purpose='upgrade'))}, - subject=uid,token=token,contract_version=2) + args = { + 'subject_id': uid, + 'public_key': b64(new_key.public_key), + 'handle': 'promoted', + 'encryption_recipient': recipient, + } + upgrade = await call( + app, + 'identity.upgrade', + { + 'handle': 'promoted', + 'public_key': args['public_key'], + 'encryption_recipient': recipient, + 'possession_proof': wire(new_key.sign(canonical(args), purpose='upgrade')), + }, + subject=uid, + token=token, + contract_version=2, + ) assert upgrade.status == 'ok' and upgrade.data['subject_id'] == uid, wire(upgrade) async with app.metadata.transaction(write=False) as tx: assert (await tx.resource(post.resources[0].id)).owner == uid + @pytest.mark.asyncio async def test_requested_write_projection_is_returned_without_readback(installed): - app,_=installed - key,uid,_=await register(app,'projection-author') + app, _ = installed + key, uid, _ = await register(app, 'projection-author') from msg.core.requests import request_for - packet=request_for('content.post_create',{'parent':'/main','body':'project me'},app.settings.service_url, - subject=uid,signer=key,expires_at=NOW+timedelta(seconds=120),return_fields=('id','revision','size')) - result=await app.executor.execute(packet) - assert result.status=='ok',result - assert result.data['projection'][0]['size']==10 + + packet = request_for( + 'content.post_create', + {'parent': '/main', 'body': 'project me'}, + app.settings.service_url, + subject=uid, + signer=key, + expires_at=NOW + timedelta(seconds=120), + return_fields=('id', 'revision', 'size'), + ) + result = await app.executor.execute(packet) + assert result.status == 'ok', result + assert result.data['projection'][0]['size'] == 10 diff --git a/tests/test_session_boundary.py b/tests/test_session_boundary.py index e6230fec..9ee1b9b8 100644 --- a/tests/test_session_boundary.py +++ b/tests/test_session_boundary.py @@ -1,32 +1,39 @@ """The same bounded query and domain contract on real SQLite and PostgreSQL.""" + import asyncio import importlib import importlib.util -from pathlib import Path import subprocess import sys +from pathlib import Path from typing import get_type_hints import pytest from msg.core.codec import canonical, digest -from msg.core.errors import Failure from msg.core.contracts import MetadataSession +from msg.core.errors import Failure from msg.market.policy import DEFAULT_POLICY, load_policy -from msg.storage.postgres import PostgresMetadataStore, PostgresSession, _SCHEMA as POSTGRES_SCHEMA +from msg.storage.postgres import _SCHEMA as POSTGRES_SCHEMA, PostgresMetadataStore, PostgresSession from msg.storage.sqlite import SqliteMetadataStore, SqliteSession @pytest.fixture(params=['sqlite', 'postgres']) async def store(request, tmp_path): - value = (PostgresMetadataStore(request.getfixturevalue('pg_dsn')) - if request.param == 'postgres' else SqliteMetadataStore(tmp_path / 'metadata.db')) + value = ( + PostgresMetadataStore(request.getfixturevalue('pg_dsn')) + if request.param == 'postgres' + else SqliteMetadataStore(tmp_path / 'metadata.db') + ) if request.param == 'sqlite': # SQLite is the small test backend, not a complete production bootstrap. # Install the real policy table DDL for this real consumer contract test; # do not pretend the test is a production schema migration or parity claim. - statement = next(sql for sql in POSTGRES_SCHEMA.split(';') - if sql.strip().startswith('CREATE TABLE IF NOT EXISTS arbitration_policies (')) + statement = next( + sql + for sql in POSTGRES_SCHEMA.split(';') + if sql.strip().startswith('CREATE TABLE IF NOT EXISTS arbitration_policies (') + ) async with value.transaction(write=True) as tx: tx.execute(statement, write=True) yield value @@ -63,8 +70,11 @@ def test_declared_session_covers_real_query_consumer_without_commit_rights(): assert hasattr(MetadataSession, name), name assert hasattr(query.QuerySession, name), name common = importlib.import_module('msg.storage.session').RelationalSession - domain_methods = {name for name, member in vars(common).items() - if callable(member) and not name.startswith('_')} + domain_methods = { + name + for name, member in vars(common).items() + if callable(member) and not name.startswith('_') + } # These two methods are storage lifecycle mechanics, not consumer ports. domain_methods -= {'check', 'run_rollback_effects'} assert all(hasattr(MetadataSession, name) for name in domain_methods) @@ -90,12 +100,15 @@ async def test_query_result_is_a_guarded_view_not_a_driver_cursor(store): async def test_query_result_and_session_refuse_cross_task_and_read_only_writes(store): async with store.transaction(write=False) as tx: result = tx.execute('SELECT 1') + async def misuse_result(): with pytest.raises(Failure, match='transaction_cross_task'): result.fetchone() + async def misuse_session(): with pytest.raises(Failure, match='transaction_cross_task'): tx.one('SELECT 1') + await asyncio.gather(misuse_result(), misuse_session()) with pytest.raises(Failure, match='read_only_transaction'): tx.set_setting('forbidden', True) @@ -109,11 +122,17 @@ async def test_declared_policy_consumer_and_nested_rollback_work_on_both_backend assert isinstance(tx, query.QuerySession) assert load_policy(tx, DEFAULT_POLICY['id']) == DEFAULT_POLICY policy = {**DEFAULT_POLICY, 'id': 'query-contract-policy'} - tx.execute('INSERT INTO arbitration_policies VALUES (?,?,?)', - (policy['id'], canonical(policy).decode(), digest(policy)), write=True) + tx.execute( + 'INSERT INTO arbitration_policies VALUES (?,?,?)', + (policy['id'], canonical(policy).decode(), digest(policy)), + write=True, + ) assert load_policy(tx, policy['id']) == policy - tx.execute('INSERT INTO arbitration_policies VALUES (?,?,?)', - ('corrupt-contract-policy', canonical(policy).decode(), 'wrong-digest'), write=True) + tx.execute( + 'INSERT INTO arbitration_policies VALUES (?,?,?)', + ('corrupt-contract-policy', canonical(policy).decode(), 'wrong-digest'), + write=True, + ) with pytest.raises(Failure, match='arbitration_policy_corrupt'): load_policy(tx, 'corrupt-contract-policy') tx.set_setting('outer', 'keep') diff --git a/tests/test_share_grants.py b/tests/test_share_grants.py index e2dccde2..8581f93b 100644 --- a/tests/test_share_grants.py +++ b/tests/test_share_grants.py @@ -1,10 +1,11 @@ """Direct sharing is a current read source for one leaf, never a directory right.""" + from datetime import timedelta import pytest +from test_service import NOW, call, register from msg.core.codec import wire -from test_service import NOW, call, register @pytest.mark.asyncio @@ -12,53 +13,86 @@ async def test_private_ancestor_exact_read_and_immediate_revocation(installed): app, _ = installed alice_key, alice, _ = await register(app, 'share-alice') bob_key, bob, _ = await register(app, 'share-bob') - topic = await call(app, 'content.topic_create', {'parent': '/main', 'name': 'share-private'}, - key=alice_key, subject=alice) + topic = await call( + app, + 'content.topic_create', + {'parent': '/main', 'name': 'share-private'}, + key=alice_key, + subject=alice, + ) assert topic.status == 'ok', wire(topic) folder = topic.resources[0].id - secret = await call(app, 'content.post_create', {'parent': folder, 'body': 'shared exact text'}, - key=alice_key, subject=alice) - sibling = await call(app, 'content.post_create', {'parent': folder, 'body': 'unshared sibling'}, - key=alice_key, subject=alice) + secret = await call( + app, + 'content.post_create', + {'parent': folder, 'body': 'shared exact text'}, + key=alice_key, + subject=alice, + ) + sibling = await call( + app, + 'content.post_create', + {'parent': folder, 'body': 'unshared sibling'}, + key=alice_key, + subject=alice, + ) assert secret.status == sibling.status == 'ok' leaf = secret.resources[0].id - chmod = await call(app, 'content.chmod', {'id': folder, 'mode': '0700'}, - key=alice_key, subject=alice, - expected=((folder, topic.data['generation']),)) + chmod = await call( + app, + 'content.chmod', + {'id': folder, 'mode': '0700'}, + key=alice_key, + subject=alice, + expected=((folder, topic.data['generation']),), + ) assert chmod.status == 'ok', wire(chmod) before = await call(app, 'discovery.get', {'id': leaf}, key=bob_key, subject=bob) assert before.status == 'error' and before.error.code == 'permission_denied' async with app.metadata.transaction(write=False) as tx: - epoch_before=tx.setting('authorization_epoch',0) - granted = await call(app, 'sharing.grant', {'resource': leaf, 'grantee': bob, - 'expires_at': wire(NOW + timedelta(days=1))}, key=alice_key, subject=alice) + epoch_before = tx.setting('authorization_epoch', 0) + granted = await call( + app, + 'sharing.grant', + {'resource': leaf, 'grantee': bob, 'expires_at': wire(NOW + timedelta(days=1))}, + key=alice_key, + subject=alice, + ) assert granted.status == 'ok', wire(granted) async with app.metadata.transaction(write=False) as tx: - assert tx.setting('authorization_epoch',0)==epoch_before+1 + assert tx.setting('authorization_epoch', 0) == epoch_before + 1 gid = granted.data['grant']['id'] for view in ({}, {'view': 'meta'}, {'view': 'history'}): read = await call(app, 'discovery.get', {'id': leaf, **view}, key=bob_key, subject=bob) assert read.status == 'ok', wire(read) raw = await call(app, 'discovery.raw', {'id': leaf}, key=bob_key, subject=bob) assert raw.status == 'ok', wire(raw) - for op, args in (('discovery.get', {'id': folder}), - ('discovery.list', {'parent': folder}), - ('discovery.get', {'id': sibling.resources[0].id}), - ('sharing.grant', {'resource': leaf, 'grantee': alice, - 'expires_at': wire(NOW + timedelta(days=1))})): + for op, args in ( + ('discovery.get', {'id': folder}), + ('discovery.list', {'parent': folder}), + ('discovery.get', {'id': sibling.resources[0].id}), + ( + 'sharing.grant', + {'resource': leaf, 'grantee': alice, 'expires_at': wire(NOW + timedelta(days=1))}, + ), + ): denied = await call(app, op, args, key=bob_key, subject=bob) assert denied.status == 'error', (op, wire(denied)) - revoked = await call(app, 'sharing.revoke', {'grant_id': gid}, - key=alice_key, subject=alice) + revoked = await call(app, 'sharing.revoke', {'grant_id': gid}, key=alice_key, subject=alice) assert revoked.status == 'ok', wire(revoked) async with app.metadata.transaction(write=False) as tx: - assert tx.setting('authorization_epoch',0)==epoch_before+2 + assert tx.setting('authorization_epoch', 0) == epoch_before + 2 after = await call(app, 'discovery.get', {'id': leaf}, key=bob_key, subject=bob) assert after.status == 'error' and after.error.code == 'permission_denied', wire(after) # Revoking one source does not disable ordinary mode access. - traversable = await call(app, 'content.chmod', {'id': folder, 'mode': '0711'}, - key=alice_key, subject=alice, - expected=((folder, chmod.data['generation']),)) + traversable = await call( + app, + 'content.chmod', + {'id': folder, 'mode': '0711'}, + key=alice_key, + subject=alice, + expected=((folder, chmod.data['generation']),), + ) assert traversable.status == 'ok', wire(traversable) mode_read = await call(app, 'discovery.get', {'id': leaf}, key=bob_key, subject=bob) assert mode_read.status == 'ok', wire(mode_read) @@ -72,47 +106,99 @@ async def test_share_rejects_containers_and_managed_or_dm_resources(installed): alice_key, alice, _ = await register(app, 'share-boundary-a') bob_key, bob, _ = await register(app, 'share-boundary-b') expiry = wire(NOW + timedelta(hours=1)) - topic = await call(app, 'content.topic_create', {'parent': '/main', 'name': 'share-container'}, - key=alice_key, subject=alice) + topic = await call( + app, + 'content.topic_create', + {'parent': '/main', 'name': 'share-container'}, + key=alice_key, + subject=alice, + ) assert topic.status == 'ok' - container = await call(app, 'sharing.grant', {'resource': topic.resources[0].id, - 'grantee': bob, 'expires_at': expiry}, key=alice_key, subject=alice) + container = await call( + app, + 'sharing.grant', + {'resource': topic.resources[0].id, 'grantee': bob, 'expires_at': expiry}, + key=alice_key, + subject=alice, + ) assert container.status == 'error' and container.error.code == 'share_container_forbidden' - personal = await call(app, 'identity.personal_put', {'kind': 'soul', 'body': 'private soul'}, - key=alice_key, subject=alice) + personal = await call( + app, + 'identity.personal_put', + {'kind': 'soul', 'body': 'private soul'}, + key=alice_key, + subject=alice, + ) assert personal.status == 'ok', wire(personal) soul = personal.resources[0].id - denied = await call(app, 'sharing.grant', {'resource': soul, 'grantee': bob, - 'expires_at': expiry}, key=alice_key, subject=alice) + denied = await call( + app, + 'sharing.grant', + {'resource': soul, 'grantee': bob, 'expires_at': expiry}, + key=alice_key, + subject=alice, + ) assert denied.status == 'error', wire(denied) - note = await call(app, 'identity.note_put', {'name': 'only-this.md', 'body': 'one note'}, - key=alice_key, subject=alice) + note = await call( + app, + 'identity.note_put', + {'name': 'only-this.md', 'body': 'one note'}, + key=alice_key, + subject=alice, + ) assert note.status == 'ok', wire(note) note_id = note.resources[0].id - shared_note = await call(app, 'sharing.grant', {'resource': note_id, 'grantee': bob, - 'expires_at': expiry}, key=alice_key, subject=alice) + shared_note = await call( + app, + 'sharing.grant', + {'resource': note_id, 'grantee': bob, 'expires_at': expiry}, + key=alice_key, + subject=alice, + ) assert shared_note.status == 'ok', wire(shared_note) read_note = await call(app, 'discovery.get', {'id': note_id}, key=bob_key, subject=bob) assert read_note.status == 'ok', wire(read_note) async with app.metadata.transaction(write=False) as tx: notes_folder = (await tx.resource(note_id)).parent - list_notes = await call(app, 'discovery.list', {'parent': notes_folder}, - key=bob_key, subject=bob) + list_notes = await call( + app, 'discovery.list', {'parent': notes_folder}, key=bob_key, subject=bob + ) assert list_notes.status == 'error', wire(list_notes) - todo = await call(app, 'identity.todo_put', {'name': 'do-it.md', 'title': 'one task'}, - key=alice_key, subject=alice) + todo = await call( + app, + 'identity.todo_put', + {'name': 'do-it.md', 'title': 'one task'}, + key=alice_key, + subject=alice, + ) assert todo.status == 'ok', wire(todo) - shared_todo = await call(app, 'sharing.grant', {'resource': todo.resources[0].id, - 'grantee': bob, 'expires_at': expiry}, key=alice_key, subject=alice) + shared_todo = await call( + app, + 'sharing.grant', + {'resource': todo.resources[0].id, 'grantee': bob, 'expires_at': expiry}, + key=alice_key, + subject=alice, + ) assert shared_todo.status == 'error', wire(shared_todo) - conversation = await call(app, 'communication.dm_request', {'recipient': bob}, - key=alice_key, subject=alice) + conversation = await call( + app, 'communication.dm_request', {'recipient': bob}, key=alice_key, subject=alice + ) assert conversation.status == 'ok', wire(conversation) - dm = await call(app, 'sharing.grant', {'resource': conversation.data['conversation_id'], - 'grantee': bob, 'expires_at': expiry}, key=alice_key, subject=alice) + dm = await call( + app, + 'sharing.grant', + {'resource': conversation.data['conversation_id'], 'grantee': bob, 'expires_at': expiry}, + key=alice_key, + subject=alice, + ) assert dm.status == 'error', wire(dm) - system = await call(app, 'sharing.grant', {'resource': 'r_rules', 'grantee': bob, - 'expires_at': expiry}, key=alice_key, subject=alice) + system = await call( + app, + 'sharing.grant', + {'resource': 'r_rules', 'grantee': bob, 'expires_at': expiry}, + key=alice_key, + subject=alice, + ) assert system.status == 'error', wire(system) @@ -122,37 +208,87 @@ async def test_owner_revocation_while_archived_survives_restore(installed): alice_key, alice, _ = await register(app, 'share-dormant-a') bob_key, bob, _ = await register(app, 'share-dormant-b') carol_key, carol, _ = await register(app, 'share-dormant-c') - post = await call(app, 'content.post_create', {'parent': '/main', 'body': 'dormant'}, - key=alice_key, subject=alice) + post = await call( + app, + 'content.post_create', + {'parent': '/main', 'body': 'dormant'}, + key=alice_key, + subject=alice, + ) rid = post.resources[0].id - private = await call(app, 'content.chmod', {'id': rid, 'mode': '0600'}, - key=alice_key, subject=alice, - expected=((rid, post.data['generation']),)) + private = await call( + app, + 'content.chmod', + {'id': rid, 'mode': '0600'}, + key=alice_key, + subject=alice, + expected=((rid, post.data['generation']),), + ) assert private.status == 'ok', wire(private) expiry = wire(NOW + timedelta(days=1)) - v1 = await call(app, 'sharing.grant', {'resource': rid, 'grantee': bob, - 'expires_at': expiry}, key=alice_key, subject=alice) - v2 = await call(app, 'sharing.grant', {'resource': rid, 'grantee': carol, - 'grantee_kind': 'user', 'operations': ['read'], 'expires_at': expiry}, - key=alice_key, subject=alice, contract_version=2) + v1 = await call( + app, + 'sharing.grant', + {'resource': rid, 'grantee': bob, 'expires_at': expiry}, + key=alice_key, + subject=alice, + ) + v2 = await call( + app, + 'sharing.grant', + { + 'resource': rid, + 'grantee': carol, + 'grantee_kind': 'user', + 'operations': ['read'], + 'expires_at': expiry, + }, + key=alice_key, + subject=alice, + contract_version=2, + ) assert v1.status == v2.status == 'ok', (wire(v1), wire(v2)) for key, subject in ((bob_key, bob), (carol_key, carol)): read = await call(app, 'discovery.get', {'id': rid}, key=key, subject=subject) assert read.status == 'ok', wire(read) - archived = await call(app, 'content.archive', {'id': rid}, key=alice_key, subject=alice, - expected=((rid, private.data['generation']),)) + archived = await call( + app, + 'content.archive', + {'id': rid}, + key=alice_key, + subject=alice, + expected=((rid, private.data['generation']),), + ) assert archived.status == 'ok', wire(archived) for grant_id, version in ((v1.data['grant']['id'], 1), (v2.data['grant']['id'], 2)): for key, subject in ((bob_key, bob), (carol_key, carol)): - denied = await call(app, 'sharing.revoke', {'grant_id': grant_id}, - key=key, subject=subject, contract_version=version) + denied = await call( + app, + 'sharing.revoke', + {'grant_id': grant_id}, + key=key, + subject=subject, + contract_version=version, + ) assert denied.status == 'error', wire(denied) - revoked = await call(app, 'sharing.revoke', {'grant_id': grant_id}, - key=alice_key, subject=alice, contract_version=version) + revoked = await call( + app, + 'sharing.revoke', + {'grant_id': grant_id}, + key=alice_key, + subject=alice, + contract_version=version, + ) assert revoked.status == 'ok', wire(revoked) assert revoked.data['grant']['revoked_at'] is not None - restored = await call(app, 'content.restore', {'id': rid}, key=alice_key, subject=alice, - expected=((rid, archived.data['generation']),)) + restored = await call( + app, + 'content.restore', + {'id': rid}, + key=alice_key, + subject=alice, + expected=((rid, archived.data['generation']),), + ) assert restored.status == 'ok', wire(restored) for key, subject in ((bob_key, bob), (carol_key, carol)): after = await call(app, 'discovery.get', {'id': rid}, key=key, subject=subject) @@ -164,21 +300,41 @@ async def test_expired_grant_does_not_authorize_and_can_be_replaced(installed): app, _ = installed alice_key, alice, _ = await register(app, 'share-expire-a') bob_key, bob, _ = await register(app, 'share-expire-b') - post = await call(app, 'content.post_create', {'parent': '/main', 'body': 'expiring'}, - key=alice_key, subject=alice) + post = await call( + app, + 'content.post_create', + {'parent': '/main', 'body': 'expiring'}, + key=alice_key, + subject=alice, + ) rid = post.resources[0].id - private = await call(app, 'content.chmod', {'id': rid, 'mode': '0600'}, - key=alice_key, subject=alice, - expected=((rid, post.data['generation']),)) + private = await call( + app, + 'content.chmod', + {'id': rid, 'mode': '0600'}, + key=alice_key, + subject=alice, + expected=((rid, post.data['generation']),), + ) assert private.status == 'ok', wire(private) - first = await call(app, 'sharing.grant', {'resource': rid, 'grantee': bob, - 'expires_at': wire(NOW + timedelta(seconds=1))}, key=alice_key, subject=alice) + first = await call( + app, + 'sharing.grant', + {'resource': rid, 'grantee': bob, 'expires_at': wire(NOW + timedelta(seconds=1))}, + key=alice_key, + subject=alice, + ) assert first.status == 'ok', wire(first) app.executor.clock = lambda: NOW + timedelta(seconds=2) expired = await call(app, 'discovery.get', {'id': rid}, key=bob_key, subject=bob) assert expired.status == 'error' and expired.error.code == 'permission_denied', wire(expired) - replacement = await call(app, 'sharing.grant', {'resource': rid, 'grantee': bob, - 'expires_at': wire(NOW + timedelta(days=1))}, key=alice_key, subject=alice) + replacement = await call( + app, + 'sharing.grant', + {'resource': rid, 'grantee': bob, 'expires_at': wire(NOW + timedelta(days=1))}, + key=alice_key, + subject=alice, + ) assert replacement.status == 'ok', wire(replacement) restored = await call(app, 'discovery.get', {'id': rid}, key=bob_key, subject=bob) assert restored.status == 'ok', wire(restored) diff --git a/tests/test_share_grants_v2.py b/tests/test_share_grants_v2.py index c2cceaf4..776a65c3 100644 --- a/tests/test_share_grants_v2.py +++ b/tests/test_share_grants_v2.py @@ -1,137 +1,294 @@ """Share sources are independent and delegated grants track live group membership.""" + from datetime import timedelta import pytest +from test_service import NOW, call, register from msg.core.codec import wire -from test_service import NOW,call,register -async def invoke(app,identity,operation,args,*,version=1,expected=()): - return await call(app,operation,args,key=identity[0],subject=identity[1], - contract_version=version,expected=expected) +async def invoke(app, identity, operation, args, *, version=1, expected=()): + return await call( + app, + operation, + args, + key=identity[0], + subject=identity[1], + contract_version=version, + expected=expected, + ) def ok(result): - assert result.status=='ok',(result.error.code if result.error else None,wire(result)) + assert result.status == 'ok', (result.error.code if result.error else None, wire(result)) return result @pytest.mark.asyncio async def test_group_reshare_tracks_source_membership_and_independent_grants(installed): - app,_=installed - owner=await register(app,'share2-owner') - member=await register(app,'share2-member') - recipient=await register(app,'share2-recipient') - group=ok(await invoke(app,owner,'group.create',{'name':'share2-group'})).resources[0].id - ok(await invoke(app,owner,'group.invite',{'group':group,'subject':member[1]})) - ok(await invoke(app,member,'group.join',{'group':group})) - post=ok(await invoke(app,owner,'content.post_create', - {'parent':'/main','body':'v2 private'})) - rid=post.resources[0].id - ok(await invoke(app,owner,'content.chmod',{'id':rid,'mode':'0600'}, - expected=((rid,post.data['generation']),))) - expiry=wire(NOW+timedelta(days=2)) - group_grant=ok(await invoke(app,owner,'sharing.grant', - {'resource':rid,'grantee':group,'grantee_kind':'group','operations':['read'], - 'expires_at':expiry,'allow_reshare':True},version=2)).data['grant']['id'] - ok(await invoke(app,member,'discovery.get',{'id':rid})) - missing_source=await invoke(app,member,'sharing.grant', - {'resource':rid,'grantee':recipient[1],'grantee_kind':'user', - 'operations':['read'],'expires_at':expiry},version=2) - assert missing_source.error.code=='share_source_required' - overlong=await invoke(app,member,'sharing.grant', - {'resource':rid,'grantee':recipient[1],'grantee_kind':'user', - 'operations':['read'],'expires_at':wire(NOW+timedelta(days=3)), - 'source_grant_id':group_grant},version=2) - assert overlong.error.code=='share_scope_exceeded' - delegated=ok(await invoke(app,member,'sharing.grant', - {'resource':rid,'grantee':recipient[1],'grantee_kind':'user', - 'operations':['read'],'expires_at':wire(NOW+timedelta(days=1)), - 'source_grant_id':group_grant},version=2)).data['grant'] + app, _ = installed + owner = await register(app, 'share2-owner') + member = await register(app, 'share2-member') + recipient = await register(app, 'share2-recipient') + group = ok(await invoke(app, owner, 'group.create', {'name': 'share2-group'})).resources[0].id + ok(await invoke(app, owner, 'group.invite', {'group': group, 'subject': member[1]})) + ok(await invoke(app, member, 'group.join', {'group': group})) + post = ok( + await invoke(app, owner, 'content.post_create', {'parent': '/main', 'body': 'v2 private'}) + ) + rid = post.resources[0].id + ok( + await invoke( + app, + owner, + 'content.chmod', + {'id': rid, 'mode': '0600'}, + expected=((rid, post.data['generation']),), + ) + ) + expiry = wire(NOW + timedelta(days=2)) + group_grant = ok( + await invoke( + app, + owner, + 'sharing.grant', + { + 'resource': rid, + 'grantee': group, + 'grantee_kind': 'group', + 'operations': ['read'], + 'expires_at': expiry, + 'allow_reshare': True, + }, + version=2, + ) + ).data['grant']['id'] + ok(await invoke(app, member, 'discovery.get', {'id': rid})) + missing_source = await invoke( + app, + member, + 'sharing.grant', + { + 'resource': rid, + 'grantee': recipient[1], + 'grantee_kind': 'user', + 'operations': ['read'], + 'expires_at': expiry, + }, + version=2, + ) + assert missing_source.error.code == 'share_source_required' + overlong = await invoke( + app, + member, + 'sharing.grant', + { + 'resource': rid, + 'grantee': recipient[1], + 'grantee_kind': 'user', + 'operations': ['read'], + 'expires_at': wire(NOW + timedelta(days=3)), + 'source_grant_id': group_grant, + }, + version=2, + ) + assert overlong.error.code == 'share_scope_exceeded' + delegated = ok( + await invoke( + app, + member, + 'sharing.grant', + { + 'resource': rid, + 'grantee': recipient[1], + 'grantee_kind': 'user', + 'operations': ['read'], + 'expires_at': wire(NOW + timedelta(days=1)), + 'source_grant_id': group_grant, + }, + version=2, + ) + ).data['grant'] assert delegated['allow_reshare'] is False - ok(await invoke(app,recipient,'discovery.get',{'id':rid})) - denied=await invoke(app,recipient,'sharing.grant', - {'resource':rid,'grantee':member[1],'grantee_kind':'user', - 'operations':['read'],'expires_at':wire(NOW+timedelta(hours=1)), - 'source_grant_id':delegated['id']},version=2) - assert denied.error.code=='share_source_unavailable' - ok(await invoke(app,member,'group.leave',{'group':group})) - for identity in (member,recipient): - denied=await invoke(app,identity,'discovery.get',{'id':rid}) - assert denied.error.code=='permission_denied',wire(denied) - direct=ok(await invoke(app,owner,'sharing.grant', - {'resource':rid,'grantee':recipient[1],'grantee_kind':'user', - 'operations':['read'],'expires_at':expiry},version=2)).data['grant']['id'] - ok(await invoke(app,recipient,'discovery.get',{'id':rid})) - ok(await invoke(app,owner,'sharing.revoke',{'grant_id':group_grant},version=2)) - ok(await invoke(app,recipient,'discovery.get',{'id':rid})) - ok(await invoke(app,owner,'sharing.revoke',{'grant_id':direct},version=2)) - denied=await invoke(app,recipient,'discovery.get',{'id':rid}) - assert denied.error.code=='permission_denied' + ok(await invoke(app, recipient, 'discovery.get', {'id': rid})) + denied = await invoke( + app, + recipient, + 'sharing.grant', + { + 'resource': rid, + 'grantee': member[1], + 'grantee_kind': 'user', + 'operations': ['read'], + 'expires_at': wire(NOW + timedelta(hours=1)), + 'source_grant_id': delegated['id'], + }, + version=2, + ) + assert denied.error.code == 'share_source_unavailable' + ok(await invoke(app, member, 'group.leave', {'group': group})) + for identity in (member, recipient): + denied = await invoke(app, identity, 'discovery.get', {'id': rid}) + assert denied.error.code == 'permission_denied', wire(denied) + direct = ok( + await invoke( + app, + owner, + 'sharing.grant', + { + 'resource': rid, + 'grantee': recipient[1], + 'grantee_kind': 'user', + 'operations': ['read'], + 'expires_at': expiry, + }, + version=2, + ) + ).data['grant']['id'] + ok(await invoke(app, recipient, 'discovery.get', {'id': rid})) + ok(await invoke(app, owner, 'sharing.revoke', {'grant_id': group_grant}, version=2)) + ok(await invoke(app, recipient, 'discovery.get', {'id': rid})) + ok(await invoke(app, owner, 'sharing.revoke', {'grant_id': direct}, version=2)) + denied = await invoke(app, recipient, 'discovery.get', {'id': rid}) + assert denied.error.code == 'permission_denied' @pytest.mark.asyncio async def test_reshare_revocation_and_expiry_do_not_remove_other_sources(installed): - app,_=installed - owner=await register(app,'share2-chain-owner') - intermediary=await register(app,'share2-chain-middle') - recipient=await register(app,'share2-chain-end') - post=ok(await invoke(app,owner,'content.post_create', - {'parent':'/main','body':'chain private'})) - rid=post.resources[0].id - ok(await invoke(app,owner,'content.chmod',{'id':rid,'mode':'0600'}, - expected=((rid,post.data['generation']),))) - source=ok(await invoke(app,owner,'sharing.grant', - {'resource':rid,'grantee':intermediary[1],'grantee_kind':'user', - 'operations':['read'],'expires_at':wire(NOW+timedelta(days=2)), - 'allow_reshare':True},version=2)).data['grant']['id'] - child=ok(await invoke(app,intermediary,'sharing.grant', - {'resource':rid,'grantee':recipient[1],'grantee_kind':'user', - 'operations':['read'],'expires_at':wire(NOW+timedelta(days=1)), - 'source_grant_id':source},version=2)).data['grant']['id'] - independent=ok(await invoke(app,owner,'sharing.grant', - {'resource':rid,'grantee':recipient[1],'grantee_kind':'user', - 'operations':['read'],'expires_at':wire(NOW+timedelta(hours=1))}, - version=2)).data['grant']['id'] - ok(await invoke(app,recipient,'discovery.get',{'id':rid})) - ok(await invoke(app,owner,'sharing.revoke',{'grant_id':source},version=2)) - ok(await invoke(app,recipient,'discovery.get',{'id':rid})) - ok(await invoke(app,owner,'sharing.revoke',{'grant_id':independent},version=2)) - denied=await invoke(app,recipient,'discovery.get',{'id':rid}) - assert denied.error.code=='permission_denied' - listed=ok(await invoke(app,intermediary,'sharing.list',{'resource':rid},version=2)) - assert listed.data['grants'][0]['id']==child - app.executor.clock=lambda:NOW+timedelta(days=3) - denied=await invoke(app,intermediary,'discovery.get',{'id':rid}) - assert denied.error.code=='permission_denied' + app, _ = installed + owner = await register(app, 'share2-chain-owner') + intermediary = await register(app, 'share2-chain-middle') + recipient = await register(app, 'share2-chain-end') + post = ok( + await invoke( + app, owner, 'content.post_create', {'parent': '/main', 'body': 'chain private'} + ) + ) + rid = post.resources[0].id + ok( + await invoke( + app, + owner, + 'content.chmod', + {'id': rid, 'mode': '0600'}, + expected=((rid, post.data['generation']),), + ) + ) + source = ok( + await invoke( + app, + owner, + 'sharing.grant', + { + 'resource': rid, + 'grantee': intermediary[1], + 'grantee_kind': 'user', + 'operations': ['read'], + 'expires_at': wire(NOW + timedelta(days=2)), + 'allow_reshare': True, + }, + version=2, + ) + ).data['grant']['id'] + child = ok( + await invoke( + app, + intermediary, + 'sharing.grant', + { + 'resource': rid, + 'grantee': recipient[1], + 'grantee_kind': 'user', + 'operations': ['read'], + 'expires_at': wire(NOW + timedelta(days=1)), + 'source_grant_id': source, + }, + version=2, + ) + ).data['grant']['id'] + independent = ok( + await invoke( + app, + owner, + 'sharing.grant', + { + 'resource': rid, + 'grantee': recipient[1], + 'grantee_kind': 'user', + 'operations': ['read'], + 'expires_at': wire(NOW + timedelta(hours=1)), + }, + version=2, + ) + ).data['grant']['id'] + ok(await invoke(app, recipient, 'discovery.get', {'id': rid})) + ok(await invoke(app, owner, 'sharing.revoke', {'grant_id': source}, version=2)) + ok(await invoke(app, recipient, 'discovery.get', {'id': rid})) + ok(await invoke(app, owner, 'sharing.revoke', {'grant_id': independent}, version=2)) + denied = await invoke(app, recipient, 'discovery.get', {'id': rid}) + assert denied.error.code == 'permission_denied' + listed = ok(await invoke(app, intermediary, 'sharing.list', {'resource': rid}, version=2)) + assert listed.data['grants'][0]['id'] == child + app.executor.clock = lambda: NOW + timedelta(days=3) + denied = await invoke(app, intermediary, 'discovery.get', {'id': rid}) + assert denied.error.code == 'permission_denied' @pytest.mark.asyncio async def test_v2_group_share_does_not_open_parent_or_sensitive_resource(installed): - app,_=installed - owner=await register(app,'share2-private-owner') - member=await register(app,'share2-private-member') - group=ok(await invoke(app,owner,'group.create',{'name':'share2-private-group'})).resources[0].id - ok(await invoke(app,owner,'group.invite',{'group':group,'subject':member[1]})) - ok(await invoke(app,member,'group.join',{'group':group})) - note=ok(await invoke(app,owner,'identity.note_put', - {'name':'alone.md','body':'single private note'})) - rid=note.resources[0].id - grant=ok(await invoke(app,owner,'sharing.grant', - {'resource':rid,'grantee':group,'grantee_kind':'group', - 'operations':['read'],'expires_at':wire(NOW+timedelta(days=1))}, - version=2)) - assert grant.data['grant']['status']=='active' - ok(await invoke(app,member,'discovery.get',{'id':rid})) + app, _ = installed + owner = await register(app, 'share2-private-owner') + member = await register(app, 'share2-private-member') + group = ( + ok(await invoke(app, owner, 'group.create', {'name': 'share2-private-group'})) + .resources[0] + .id + ) + ok(await invoke(app, owner, 'group.invite', {'group': group, 'subject': member[1]})) + ok(await invoke(app, member, 'group.join', {'group': group})) + note = ok( + await invoke( + app, owner, 'identity.note_put', {'name': 'alone.md', 'body': 'single private note'} + ) + ) + rid = note.resources[0].id + grant = ok( + await invoke( + app, + owner, + 'sharing.grant', + { + 'resource': rid, + 'grantee': group, + 'grantee_kind': 'group', + 'operations': ['read'], + 'expires_at': wire(NOW + timedelta(days=1)), + }, + version=2, + ) + ) + assert grant.data['grant']['status'] == 'active' + ok(await invoke(app, member, 'discovery.get', {'id': rid})) async with app.metadata.transaction(write=False) as tx: - parent=(await tx.resource(rid)).parent - denied=await invoke(app,member,'discovery.list',{'parent':parent}) - assert denied.error.code=='permission_denied' - for kind in ('soul','agents'): - personal=ok(await invoke(app,owner,'identity.personal_put', - {'kind':kind,'body':'private'})) - result=await invoke(app,owner,'sharing.grant', - {'resource':personal.resources[0].id,'grantee':group,'grantee_kind':'group', - 'operations':['read'],'expires_at':wire(NOW+timedelta(days=1))},version=2) - assert result.status=='error',wire(result) + parent = (await tx.resource(rid)).parent + denied = await invoke(app, member, 'discovery.list', {'parent': parent}) + assert denied.error.code == 'permission_denied' + for kind in ('soul', 'agents'): + personal = ok( + await invoke(app, owner, 'identity.personal_put', {'kind': kind, 'body': 'private'}) + ) + result = await invoke( + app, + owner, + 'sharing.grant', + { + 'resource': personal.resources[0].id, + 'grantee': group, + 'grantee_kind': 'group', + 'operations': ['read'], + 'expires_at': wire(NOW + timedelta(days=1)), + }, + version=2, + ) + assert result.status == 'error', wire(result) diff --git a/tests/test_share_links.py b/tests/test_share_links.py index 74807a31..b27bbbea 100644 --- a/tests/test_share_links.py +++ b/tests/test_share_links.py @@ -1,54 +1,71 @@ """Bearer links stay off by default and authorize exactly one bounded read.""" -from datetime import timedelta + import hashlib import os +from datetime import timedelta import httpx import pytest +from test_authorization import approve, scoped +from test_service import NOW, call, register from msg.core.codec import b64, canonical, unb64, wire from msg.core.requests import request_for from msg.transports.dictionary import build_dictionary from msg.transports.http import create_app -from test_service import NOW, call, register -from test_authorization import approve, scoped def proof(): token = b64(os.urandom(32)) - verifier = 'sha256:' + hashlib.sha256(b'share-link-v1\0'+unb64(token)).hexdigest() + verifier = 'sha256:' + hashlib.sha256(b'share-link-v1\0' + unb64(token)).hexdigest() return token, verifier @pytest.mark.asyncio async def test_bearer_link_rejects_get_packet_and_short_paths(installed): - app,_=installed - token,_=proof() - packet=request_for('sharing.link_read',{'link_id':'link_missing','token':token}, - app.settings.service_url,expires_at=NOW+timedelta(seconds=120)) - encoded=b64(canonical(wire(packet))) - code=build_dictionary(app.registry).code_for('operation','sharing.link_read@1') - async with httpx.AsyncClient(transport=httpx.ASGITransport(app=create_app(app)), - base_url=app.settings.service_url) as http: - named=await http.get('/-/g/sharing.link_read/j/'+encoded) - short=await http.get('/-/g/'+code) - assert named.status_code==short.status_code==405 - posted=await http.post('/-/p/sharing.link_read',content=canonical(wire(packet))) - assert posted.status_code==400 - assert posted.json()['error']['code']=='share_links_disabled' + app, _ = installed + token, _ = proof() + packet = request_for( + 'sharing.link_read', + {'link_id': 'link_missing', 'token': token}, + app.settings.service_url, + expires_at=NOW + timedelta(seconds=120), + ) + encoded = b64(canonical(wire(packet))) + code = build_dictionary(app.registry).code_for('operation', 'sharing.link_read@1') + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=create_app(app)), base_url=app.settings.service_url + ) as http: + named = await http.get('/-/g/sharing.link_read/j/' + encoded) + short = await http.get('/-/g/' + code) + assert named.status_code == short.status_code == 405 + posted = await http.post('/-/p/sharing.link_read', content=canonical(wire(packet))) + assert posted.status_code == 400 + assert posted.json()['error']['code'] == 'share_links_disabled' @pytest.mark.asyncio async def test_share_links_require_explicit_system_enable(installed): - app,root=installed - key,subject,_=await register(app,'link-operator') - denied=await call(app,'system.share_links_set',{'enabled':True},key=key,subject=subject) - assert denied.status=='error' and denied.error.code=='capability_required' - cert=await approve(app,root,subject,key,( - scoped(app,'system.config','r_root',('system.share_links_set@1',)),)) - enabled=await call(app,'system.share_links_set',{'enabled':True}, - key=key,subject=subject,certs=(cert.resource_id,)) - assert enabled.status=='ok' and enabled.data['enabled'] is True + app, root = installed + key, subject, _ = await register(app, 'link-operator') + denied = await call(app, 'system.share_links_set', {'enabled': True}, key=key, subject=subject) + assert denied.status == 'error' and denied.error.code == 'capability_required' + cert = await approve( + app, + root, + subject, + key, + (scoped(app, 'system.config', 'r_root', ('system.share_links_set@1',)),), + ) + enabled = await call( + app, + 'system.share_links_set', + {'enabled': True}, + key=key, + subject=subject, + certs=(cert.resource_id,), + ) + assert enabled.status == 'ok' and enabled.data['enabled'] is True async with app.metadata.transaction(write=False) as tx: assert tx.setting('share_links_enabled') is True @@ -57,18 +74,39 @@ async def test_share_links_require_explicit_system_enable(installed): async def test_share_link_is_explicit_bounded_and_revocable(installed): app, _ = installed key, owner, _ = await register(app, 'link-owner') - topic = await call(app, 'content.topic_create', {'parent':'/main','name':'link-folder'}, - key=key, subject=owner) + topic = await call( + app, + 'content.topic_create', + {'parent': '/main', 'name': 'link-folder'}, + key=key, + subject=owner, + ) folder = topic.resources[0].id - post = await call(app, 'content.post_create', {'parent':folder,'body':'one private leaf'}, - key=key, subject=owner) - sibling = await call(app, 'content.post_create', {'parent':folder,'body':'hidden sibling'}, - key=key, subject=owner) + post = await call( + app, + 'content.post_create', + {'parent': folder, 'body': 'one private leaf'}, + key=key, + subject=owner, + ) + sibling = await call( + app, + 'content.post_create', + {'parent': folder, 'body': 'hidden sibling'}, + key=key, + subject=owner, + ) rid = post.resources[0].id - await call(app, 'content.chmod', {'id':folder,'mode':'0700'}, key=key, subject=owner, - expected=((folder,topic.data['generation']),)) + await call( + app, + 'content.chmod', + {'id': folder, 'mode': '0700'}, + key=key, + subject=owner, + expected=((folder, topic.data['generation']),), + ) token, verifier = proof() - args = {'resource':rid,'verifier':verifier,'expires_at':wire(NOW+timedelta(hours=1))} + args = {'resource': rid, 'verifier': verifier, 'expires_at': wire(NOW + timedelta(hours=1))} off = await call(app, 'sharing.link_create', args, key=key, subject=owner) assert off.error.code == 'share_links_disabled' async with app.metadata.transaction(write=True) as tx: @@ -79,62 +117,93 @@ async def test_share_link_is_explicit_bounded_and_revocable(installed): assert token not in str(created.data) and verifier not in str(created.data) assert created.data['endpoint'] == '/-/p/sharing.link_read' async with app.metadata.transaction(write=False) as tx: - before = (tx.one('SELECT COUNT(*) FROM events')[0], - tx.one('SELECT COUNT(*) FROM audit')[0], - tx.setting('authorization_epoch',0)) - assert tx.one('SELECT verifier FROM share_links WHERE id=?',(link['id'],))[0] == verifier - wrong = await call(app, 'sharing.link_read', {'link_id':link['id'], - 'token':b64(os.urandom(32))}) - missing = await call(app, 'sharing.link_read', {'link_id':'link_missing','token':token}) - for denied in (wrong,missing): - assert denied.status=='error' and denied.error.code=='share_link_unavailable' - read = await call(app, 'sharing.link_read', {'link_id':link['id'],'token':token}) - assert read.status=='ok', (read.error.code if read.error else None, wire(read)) - assert read.data['id']==rid and 'hidden sibling' not in str(read.data) - assert set(read.data)=={'id','revision','media_type','data'} + before = ( + tx.one('SELECT COUNT(*) FROM events')[0], + tx.one('SELECT COUNT(*) FROM audit')[0], + tx.setting('authorization_epoch', 0), + ) + assert tx.one('SELECT verifier FROM share_links WHERE id=?', (link['id'],))[0] == verifier + wrong = await call( + app, 'sharing.link_read', {'link_id': link['id'], 'token': b64(os.urandom(32))} + ) + missing = await call(app, 'sharing.link_read', {'link_id': 'link_missing', 'token': token}) + for denied in (wrong, missing): + assert denied.status == 'error' and denied.error.code == 'share_link_unavailable' + read = await call(app, 'sharing.link_read', {'link_id': link['id'], 'token': token}) + assert read.status == 'ok', (read.error.code if read.error else None, wire(read)) + assert read.data['id'] == rid and 'hidden sibling' not in str(read.data) + assert set(read.data) == {'id', 'revision', 'media_type', 'data'} assert 'one private leaf' in unb64(read.data['data']).decode() - anonymous_folder = await call(app, 'discovery.list', {'parent':folder}) - anonymous_sibling = await call(app, 'discovery.get', {'id':sibling.resources[0].id}) - assert anonymous_folder.status==anonymous_sibling.status=='error' + anonymous_folder = await call(app, 'discovery.list', {'parent': folder}) + anonymous_sibling = await call(app, 'discovery.get', {'id': sibling.resources[0].id}) + assert anonymous_folder.status == anonymous_sibling.status == 'error' async with app.metadata.transaction(write=False) as tx: - assert (tx.one('SELECT COUNT(*) FROM events')[0], - tx.one('SELECT COUNT(*) FROM audit')[0], - tx.setting('authorization_epoch',0)) == before - revoked = await call(app, 'sharing.link_revoke', {'link_id':link['id']}, - key=key, subject=owner) - assert revoked.status=='ok' - after = await call(app, 'sharing.link_read', {'link_id':link['id'],'token':token}) - assert after.error.code=='share_link_unavailable' + assert ( + tx.one('SELECT COUNT(*) FROM events')[0], + tx.one('SELECT COUNT(*) FROM audit')[0], + tx.setting('authorization_epoch', 0), + ) == before + revoked = await call( + app, 'sharing.link_revoke', {'link_id': link['id']}, key=key, subject=owner + ) + assert revoked.status == 'ok' + after = await call(app, 'sharing.link_read', {'link_id': link['id'], 'token': token}) + assert after.error.code == 'share_link_unavailable' @pytest.mark.asyncio async def test_share_link_rechecks_owner_acl_expiry_and_global_switch(installed): app, _ = installed key, owner, _ = await register(app, 'link-policy-owner') - post = await call(app, 'content.post_create', {'parent':'/main','body':'policy body'}, - key=key, subject=owner) - rid=post.resources[0].id + post = await call( + app, + 'content.post_create', + {'parent': '/main', 'body': 'policy body'}, + key=key, + subject=owner, + ) + rid = post.resources[0].id async with app.metadata.transaction(write=True) as tx: tx.set_setting('share_links_enabled', True) - token, verifier=proof() - created=await call(app,'sharing.link_create',{'resource':rid,'verifier':verifier, - 'expires_at':wire(NOW+timedelta(seconds=2))},key=key,subject=owner) - assert created.status=='ok',wire(created) - lid=created.data['link']['id'] - read=lambda:call(app,'sharing.link_read',{'link_id':lid,'token':token}) - first=await read() - assert first.status=='ok', (first.error.code if first.error else None,wire(first)) - hidden=await call(app,'content.chmod',{'id':rid,'mode':'0000'},key=key,subject=owner, - expected=((rid,post.data['generation']),)) - assert hidden.status=='ok',wire(hidden) - assert (await read()).error.code=='share_link_unavailable' - restored=await call(app,'content.chmod',{'id':rid,'mode':'0600'},key=key,subject=owner, - expected=((rid,hidden.data['generation']),)) - assert restored.status=='ok',wire(restored) - assert (await read()).status=='ok' - app.executor.clock=lambda:NOW+timedelta(seconds=3) - assert (await read()).error.code=='share_link_unavailable' - app.executor.clock=lambda:NOW + token, verifier = proof() + created = await call( + app, + 'sharing.link_create', + {'resource': rid, 'verifier': verifier, 'expires_at': wire(NOW + timedelta(seconds=2))}, + key=key, + subject=owner, + ) + assert created.status == 'ok', wire(created) + lid = created.data['link']['id'] + + def read(): + return call(app, 'sharing.link_read', {'link_id': lid, 'token': token}) + + first = await read() + assert first.status == 'ok', (first.error.code if first.error else None, wire(first)) + hidden = await call( + app, + 'content.chmod', + {'id': rid, 'mode': '0000'}, + key=key, + subject=owner, + expected=((rid, post.data['generation']),), + ) + assert hidden.status == 'ok', wire(hidden) + assert (await read()).error.code == 'share_link_unavailable' + restored = await call( + app, + 'content.chmod', + {'id': rid, 'mode': '0600'}, + key=key, + subject=owner, + expected=((rid, hidden.data['generation']),), + ) + assert restored.status == 'ok', wire(restored) + assert (await read()).status == 'ok' + app.executor.clock = lambda: NOW + timedelta(seconds=3) + assert (await read()).error.code == 'share_link_unavailable' + app.executor.clock = lambda: NOW async with app.metadata.transaction(write=True) as tx: tx.set_setting('share_links_enabled', False) - assert (await read()).error.code=='share_links_disabled' + assert (await read()).error.code == 'share_links_disabled' diff --git a/tests/test_share_source_boundaries.py b/tests/test_share_source_boundaries.py index b8befb3f..3fcab2c1 100644 --- a/tests/test_share_source_boundaries.py +++ b/tests/test_share_source_boundaries.py @@ -1,28 +1,53 @@ """Every consumer of a ShareGrant must recheck the same current target boundary.""" + from dataclasses import replace from datetime import timedelta import pytest - -from msg.core.codec import canonical, wire from test_service import NOW, register from test_share_grants_v2 import invoke, ok +from msg.core.codec import canonical, wire + @pytest.mark.asyncio async def test_named_source_revalidates_target_and_persisted_constraints(installed): app, _ = installed owner = await register(app, 'boundary-owner') reader = await register(app, 'boundary-reader') - post = ok(await invoke(app, owner, 'content.post_create', - {'parent': '/main', 'body': 'private source evidence'})) + post = ok( + await invoke( + app, + owner, + 'content.post_create', + {'parent': '/main', 'body': 'private source evidence'}, + ) + ) rid = post.resources[0].id - ok(await invoke(app, owner, 'content.chmod', {'id': rid, 'mode': '0600'}, - expected=((rid, post.data['generation']),))) - grant = ok(await invoke(app, owner, 'sharing.grant', - {'resource': rid, 'grantee': reader[1], 'grantee_kind': 'user', - 'operations': ['read'], 'expires_at': wire(NOW + timedelta(days=2))}, - version=2)).data['grant']['id'] + ok( + await invoke( + app, + owner, + 'content.chmod', + {'id': rid, 'mode': '0600'}, + expected=((rid, post.data['generation']),), + ) + ) + grant = ok( + await invoke( + app, + owner, + 'sharing.grant', + { + 'resource': rid, + 'grantee': reader[1], + 'grantee_kind': 'user', + 'operations': ['read'], + 'expires_at': wire(NOW + timedelta(days=2)), + }, + version=2, + ) + ).data['grant']['id'] async with app.metadata.transaction(write=True) as tx: resource = await tx.resource(rid) assert await app.authorizer.share_source_active(resource, grant, reader[1], NOW, tx) @@ -36,11 +61,18 @@ async def test_named_source_revalidates_target_and_persisted_constraints(install ] for column, value in cases: original = tx.one(f'SELECT {column} FROM share_grants_v2 WHERE id=?', (grant,))[0] - tx.execute(f'UPDATE share_grants_v2 SET {column}=? WHERE id=?', (value, grant), write=True) - assert not await app.authorizer.share_source_active(resource, grant, reader[1], NOW, tx), column - tx.execute(f'UPDATE share_grants_v2 SET {column}=? WHERE id=?', (original, grant), write=True) + tx.execute( + f'UPDATE share_grants_v2 SET {column}=? WHERE id=?', (value, grant), write=True + ) + assert not await app.authorizer.share_source_active( + resource, grant, reader[1], NOW, tx + ), column + tx.execute( + f'UPDATE share_grants_v2 SET {column}=? WHERE id=?', (original, grant), write=True + ) assert not await app.authorizer.share_source_active( - replace(resource, state='archived'), grant, reader[1], NOW, tx) + replace(resource, state='archived'), grant, reader[1], NOW, tx + ) tx.set_setting('hosting_preview_file:' + rid, {'owner': owner[1]}) assert not await app.authorizer.share_source_active(resource, grant, reader[1], NOW, tx) @@ -51,25 +83,65 @@ async def test_child_cannot_outlive_source_even_after_restore(installed): owner = await register(app, 'duration-owner') middle = await register(app, 'duration-middle') reader = await register(app, 'duration-reader') - post = ok(await invoke(app, owner, 'content.post_create', - {'parent': '/main', 'body': 'duration'})) + post = ok( + await invoke(app, owner, 'content.post_create', {'parent': '/main', 'body': 'duration'}) + ) rid = post.resources[0].id - source = ok(await invoke(app, owner, 'sharing.grant', - {'resource': rid, 'grantee': middle[1], 'grantee_kind': 'user', - 'operations': ['read'], 'expires_at': wire(NOW + timedelta(days=2)), - 'allow_reshare': True}, version=2)).data['grant']['id'] - child = ok(await invoke(app, middle, 'sharing.grant', - {'resource': rid, 'grantee': reader[1], 'grantee_kind': 'user', - 'operations': ['read'], 'expires_at': wire(NOW + timedelta(days=1)), - 'source_grant_id': source}, version=2)).data['grant']['id'] + source = ok( + await invoke( + app, + owner, + 'sharing.grant', + { + 'resource': rid, + 'grantee': middle[1], + 'grantee_kind': 'user', + 'operations': ['read'], + 'expires_at': wire(NOW + timedelta(days=2)), + 'allow_reshare': True, + }, + version=2, + ) + ).data['grant']['id'] + child = ok( + await invoke( + app, + middle, + 'sharing.grant', + { + 'resource': rid, + 'grantee': reader[1], + 'grantee_kind': 'user', + 'operations': ['read'], + 'expires_at': wire(NOW + timedelta(days=1)), + 'source_grant_id': source, + }, + version=2, + ) + ).data['grant']['id'] async with app.metadata.transaction(write=True) as tx: resource = await tx.resource(rid) assert await app.authorizer.share_source_active(resource, child, reader[1], NOW, tx) - tx.execute('UPDATE share_grants_v2 SET expires_at=? WHERE id=?', - (wire(NOW + timedelta(days=3)), child), write=True) + tx.execute( + 'UPDATE share_grants_v2 SET expires_at=? WHERE id=?', + (wire(NOW + timedelta(days=3)), child), + write=True, + ) assert not await app.authorizer.share_source_active(resource, child, reader[1], NOW, tx) # An independent owner grant is not vetoed by this malformed source. - ok(await invoke(app, owner, 'sharing.grant', - {'resource': rid, 'grantee': reader[1], 'grantee_kind': 'user', - 'operations': ['read'], 'expires_at': wire(NOW + timedelta(hours=1))}, version=2)) + ok( + await invoke( + app, + owner, + 'sharing.grant', + { + 'resource': rid, + 'grantee': reader[1], + 'grantee_kind': 'user', + 'operations': ['read'], + 'expires_at': wire(NOW + timedelta(hours=1)), + }, + version=2, + ) + ) ok(await invoke(app, reader, 'discovery.get', {'id': rid})) diff --git a/tests/test_share_transport_matrix.py b/tests/test_share_transport_matrix.py index caecde78..d1b75b81 100644 --- a/tests/test_share_transport_matrix.py +++ b/tests/test_share_transport_matrix.py @@ -1,11 +1,12 @@ """Live share revocation crosses real adapters, historical reads, search and Sync.""" + import asyncio -from contextlib import asynccontextmanager -from datetime import timedelta import os -from pathlib import Path import socket import sys +from contextlib import asynccontextmanager +from datetime import timedelta +from pathlib import Path from uuid import uuid4 import pytest @@ -17,7 +18,7 @@ from msg.config import write_example from msg.core.codec import canonical, digest, loads, wire from msg.core.executor import result_wire -from msg.transports.client import HTTPTransport, TRANSPORTS +from msg.transports.client import TRANSPORTS, HTTPTransport from msg.transports.http import create_app @@ -33,21 +34,21 @@ async def live_adapters(tmp_path, pg_dsn, mode): sock.bind(('127.0.0.1', 0)) sock.setblocking(False) url = 'http://127.0.0.1:' + str(sock.getsockname()[1]) - app = Application(write_example(tmp_path / 'etc', tmp_path / 'data', url, - postgres_dsn=pg_dsn)) + app = Application(write_example(tmp_path / 'etc', tmp_path / 'data', url, postgres_dsn=pg_dsn)) server = None serving = process = None transports = [] try: csr, root = await _provision(app, 'share-matrix-test-passphrase') await _approve_csr(app, csr, root, expected_digest=None, operator='matrix-fixture') - server = uvicorn.Server(uvicorn.Config(create_app(app), log_level='critical', - access_log=False, ws='none')) + server = uvicorn.Server( + uvicorn.Config(create_app(app), log_level='critical', access_log=False, ws='none') + ) serving = asyncio.create_task(server.serve(sockets=[sock])) for _ in range(100): if server.started: break - await asyncio.sleep(.02) + await asyncio.sleep(0.02) assert server.started peers = [] for name in ('owner', 'middle', 'reader'): @@ -61,23 +62,50 @@ async def live_adapters(tmp_path, pg_dsn, mode): transports.append(transport) client = MsgClient(reader.state, transport) environment = {**os.environ, 'PYTHONPATH': str(Path(__file__).parents[1] / 'src')} - command = [sys.executable, '-m', 'msg.cli', '--config-dir', - str(reader.state.directory), '--server', url] + command = [ + sys.executable, + '-m', + 'msg.cli', + '--config-dir', + str(reader.state.directory), + '--server', + url, + ] if mode == 'mcp_stdio': - process = await asyncio.create_subprocess_exec(*command, 'mcp', env=environment, - stdin=asyncio.subprocess.PIPE, stdout=asyncio.subprocess.PIPE, - stderr=asyncio.subprocess.PIPE) - process.stdin.write(canonical({'jsonrpc': '2.0', 'id': 'init', - 'method': 'initialize', 'params': {'protocolVersion': '2025-11-25'}}) + b'\n') + process = await asyncio.create_subprocess_exec( + *command, + 'mcp', + env=environment, + stdin=asyncio.subprocess.PIPE, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + process.stdin.write( + canonical({ + 'jsonrpc': '2.0', + 'id': 'init', + 'method': 'initialize', + 'params': {'protocolVersion': '2025-11-25'}, + }) + + b'\n' + ) await process.stdin.drain() assert 'result' in loads(await asyncio.wait_for(process.stdout.readline(), 20)) async def invoke(operation, arguments): request_id = uuid4().hex if mode == 'cli': - child = await asyncio.create_subprocess_exec(*command, 'call', operation, - canonical(arguments).decode(), '--request-id', request_id, env=environment, - stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE) + child = await asyncio.create_subprocess_exec( + *command, + 'call', + operation, + canonical(arguments).decode(), + '--request-id', + request_id, + env=environment, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) try: stdout, stderr = await asyncio.wait_for(child.communicate(), 30) except BaseException: @@ -88,9 +116,19 @@ async def invoke(operation, arguments): assert child.returncode in {0, 1}, stderr.decode() return loads(stdout) if mode == 'mcp_stdio': - process.stdin.write(canonical({'jsonrpc': '2.0', 'id': request_id, - 'method': 'tools/call', 'params': {'name': operation, 'arguments': arguments, - '_meta': {'msg/request_id': request_id}}}) + b'\n') + process.stdin.write( + canonical({ + 'jsonrpc': '2.0', + 'id': request_id, + 'method': 'tools/call', + 'params': { + 'name': operation, + 'arguments': arguments, + '_meta': {'msg/request_id': request_id}, + }, + }) + + b'\n' + ) await process.stdin.drain() response = loads(await asyncio.wait_for(process.stdout.readline(), 20)) assert 'result' in response, response @@ -126,16 +164,41 @@ async def test_live_group_reshare_revocation_across_read_surfaces(tmp_path, pg_d ok(await middle.call('group.join', {'group': group})) post = ok(await owner.call('content.post_create', {'parent': '/main', 'body': marker})) rid, original = post['resources'][0]['id'], post['resources'][0]['revision'] - private = ok(await owner.call('content.chmod', {'id': rid, 'mode': '0600'}, - expected=((rid, post['data']['generation']),))) - ok(await owner.call('content.post_edit', {'id': rid, 'expected_revision': original, - 'body': marker + '\nnew revision'}, expected=((rid, private['data']['generation']),))) + private = ok( + await owner.call( + 'content.chmod', + {'id': rid, 'mode': '0600'}, + expected=((rid, post['data']['generation']),), + ) + ) + ok( + await owner.call( + 'content.post_edit', + {'id': rid, 'expected_revision': original, 'body': marker + '\nnew revision'}, + expected=((rid, private['data']['generation']),), + ) + ) expiry = wire(app.clock() + timedelta(minutes=30)) terms = {'resource': rid, 'operations': ['read'], 'expires_at': expiry} - parent = ok(await owner.call('sharing.grant', {**terms, 'grantee': group, - 'grantee_kind': 'group', 'allow_reshare': True}, contract_version=2))['data']['grant']['id'] - ok(await middle.call('sharing.grant', {**terms, 'grantee': reader.state.subject, - 'grantee_kind': 'user', 'source_grant_id': parent}, contract_version=2)) + parent = ok( + await owner.call( + 'sharing.grant', + {**terms, 'grantee': group, 'grantee_kind': 'group', 'allow_reshare': True}, + contract_version=2, + ) + )['data']['grant']['id'] + ok( + await middle.call( + 'sharing.grant', + { + **terms, + 'grantee': reader.state.subject, + 'grantee_kind': 'user', + 'source_grant_id': parent, + }, + contract_version=2, + ) + ) current = ok(await invoke('discovery.get', {'id': rid})) assert marker in canonical(current).decode() historical = ok(await invoke('discovery.get', {'id': rid, 'revision': original})) @@ -158,22 +221,38 @@ async def test_live_group_reshare_revocation_across_read_surfaces(tmp_path, pg_d # Reuse the same clients, credential, cursor and cache digest after the # source member leaves. Child grants are not independent authority. ok(await middle.call('group.leave', {'group': group})) - for query in ({'id': rid}, {'id': rid, 'revision': original}, - {'id': rid, 'view': 'history'}, cached): + for query in ( + {'id': rid}, + {'id': rid, 'revision': original}, + {'id': rid, 'view': 'history'}, + cached, + ): denied = await invoke('discovery.get', query) - assert denied['status'] == 'error' and denied['error']['code'] == 'permission_denied', denied + assert denied['status'] == 'error' and denied['error']['code'] == 'permission_denied', ( + denied + ) assert marker not in canonical(denied).decode() assert not denied.get('data', {}).get('not_modified', False) hidden = ok(await invoke('discovery.lexical_search', search)) - assert rid not in canonical(hidden).decode() and marker not in canonical(hidden['data']).decode() + assert ( + rid not in canonical(hidden).decode() + and marker not in canonical(hidden['data']).decode() + ) delta = ok(await invoke('communication.sync', {'cursor': cursor, 'limit': 100}))['data'] assert delta['resync_required'] is True - assert any(item['kind'] == 'revoked' and item['ref']['id'] == rid for item in delta['items']) + assert any( + item['kind'] == 'revoked' and item['ref']['id'] == rid for item in delta['items'] + ) assert marker not in canonical(delta).decode() # An invalid old source must not veto an independent live owner grant. - direct = ok(await owner.call('sharing.grant', {**terms, 'grantee': reader.state.subject, - 'grantee_kind': 'user'}, contract_version=2))['data']['grant']['id'] + direct = ok( + await owner.call( + 'sharing.grant', + {**terms, 'grantee': reader.state.subject, 'grantee_kind': 'user'}, + contract_version=2, + ) + )['data']['grant']['id'] ok(await invoke('discovery.get', {'id': rid, 'revision': original})) ok(await owner.call('sharing.revoke', {'grant_id': parent}, contract_version=2)) ok(await invoke('discovery.get', {'id': rid})) diff --git a/tests/test_single_mailbox_delivery.py b/tests/test_single_mailbox_delivery.py index 5274187d..e717ff9b 100644 --- a/tests/test_single_mailbox_delivery.py +++ b/tests/test_single_mailbox_delivery.py @@ -3,10 +3,11 @@ SMTP uses the real stdlib send_message parser and serializer with only sendmail replaced by a recorder. No socket, SMTP server or external address is contacted. """ + +import smtplib from datetime import UTC, datetime from email import policy from types import SimpleNamespace -import smtplib import pytest @@ -16,20 +17,32 @@ from msg.market.delivery_targets import email_binding, validate_address from msg.workers import mail - AMBIGUOUS = ( - 'alias,buyer@example.test', 'buyer@example.test,alias', - ',buyer@example.test', 'buyer@example.test,', - 'buyer@example.test;', 'group:buyer@example.test;', - '', 'buyer(comment)@example.test', - 'buyer@example.test(comment)', 'buyer@exam(comment)ple.test', - '"buyer"@example.test', 'buyer..alias@example.test', - 'buyer@example..test', 'buyer@example.test\x7f', + 'alias,buyer@example.test', + 'buyer@example.test,alias', + ',buyer@example.test', + 'buyer@example.test,', + 'buyer@example.test;', + 'group:buyer@example.test;', + '', + 'buyer(comment)@example.test', + 'buyer@example.test(comment)', + 'buyer@exam(comment)ple.test', + '"buyer"@example.test', + 'buyer..alias@example.test', + 'buyer@example..test', + 'buyer@example.test\x7f', ) VALID = ( - 'buyer@example.test', 'Buyer+tag@Example.TEST', 'first.last@example.test', - 'a_b-c@example.test', "o'hara@example.test", 'buyer@xn--bcher-kva.test', - 'buyer@例子.测试', '用户@example.test', '"a,b"@example.test', + 'buyer@example.test', + 'Buyer+tag@Example.TEST', + 'first.last@example.test', + 'a_b-c@example.test', + "o'hara@example.test", + 'buyer@xn--bcher-kva.test', + 'buyer@例子.测试', + '用户@example.test', + '"a,b"@example.test', ) @@ -37,8 +50,11 @@ def test_market_rejects_non_exact_single_mailbox_without_leaking_it(address): with pytest.raises(Failure) as caught: validate_address(address) - assert caught.value.as_dict() == {'code': 'invalid_email', 'retryable': False, - 'message': 'Enter one valid email address.'} + assert caught.value.as_dict() == { + 'code': 'invalid_email', + 'retryable': False, + 'message': 'Enter one valid email address.', + } assert address not in str(caught.value) @@ -48,8 +64,10 @@ def test_market_rejects_controls_and_whitespace(codepoint): validate_address('buyer' + chr(codepoint) + '@example.test') -@pytest.mark.parametrize('address', [None, 1, [], {}, '', 'buyer', '@example.test', - 'buyer@localhost', 'x' * 243 + '@example.test']) +@pytest.mark.parametrize( + 'address', + [None, 1, [], {}, '', 'buyer', '@example.test', 'buyer@localhost', 'x' * 243 + '@example.test'], +) def test_market_rejects_non_addresses(address): with pytest.raises(Failure, match='invalid_email'): validate_address(address) @@ -65,8 +83,9 @@ def test_valid_mailbox_bytes_are_not_rewritten(address): @pytest.mark.parametrize('address', AMBIGUOUS) def test_legacy_verified_row_cannot_authorize_an_ambiguous_recipient(address): - email = EmailSettings(subject_id='u_buyer', address=address, - verified_at=datetime(2026, 1, 1, tzinfo=UTC)) + email = EmailSettings( + subject_id='u_buyer', address=address, verified_at=datetime(2026, 1, 1, tzinfo=UTC) + ) tx = SimpleNamespace(one=lambda *_: (3, canonical(email))) with pytest.raises(Failure, match='invalid_email'): email_binding(tx, 'u_buyer', address) @@ -74,8 +93,9 @@ def test_legacy_verified_row_cannot_authorize_an_ambiguous_recipient(address): @pytest.fixture def smtp(monkeypatch): - seen = SimpleNamespace(connections=0, envelopes=[], messages=[], closed=0, - fail_after_data=False, refused={}) + seen = SimpleNamespace( + connections=0, envelopes=[], messages=[], closed=0, fail_after_data=False, refused={} + ) real_smtp = smtplib.SMTP class RecordingSMTP(real_smtp): @@ -92,8 +112,9 @@ def ehlo(self, name=''): def starttls(self, *, context=None): return 220, b'test' - def send_message(self, msg, from_addr=None, to_addrs=None, - mail_options=(), rcpt_options=()): + def send_message( + self, msg, from_addr=None, to_addrs=None, mail_options=(), rcpt_options=() + ): seen.envelopes.append(to_addrs) return super().send_message(msg, from_addr, to_addrs, mail_options, rcpt_options) @@ -112,16 +133,29 @@ def close(self): def sender(tls='starttls'): - return mail.SmtpSender(SimpleNamespace(enabled=True, host='smtp.invalid', port=587, - tls=tls, sender='msg@example.test', credential_file=None)) + return mail.SmtpSender( + SimpleNamespace( + enabled=True, + host='smtp.invalid', + port=587, + tls=tls, + sender='msg@example.test', + credential_file=None, + ) + ) def job(recipient, verification=False): - return SimpleNamespace(id='job_mailbox', arguments={ - 'recipient': recipient, 'recipient_subject': 'u_buyer', 'verification': verification, - 'subject': 'Verify msg email' if verification else 'msg order ready', - 'text': 'test-only notification', - }) + return SimpleNamespace( + id='job_mailbox', + arguments={ + 'recipient': recipient, + 'recipient_subject': 'u_buyer', + 'verification': verification, + 'subject': 'Verify msg email' if verification else 'msg order ready', + 'text': 'test-only notification', + }, + ) @pytest.mark.parametrize('address', AMBIGUOUS) diff --git a/tests/test_smtp_socket_acceptance.py b/tests/test_smtp_socket_acceptance.py index 4c1a2e47..dc294e8b 100644 --- a/tests/test_smtp_socket_acceptance.py +++ b/tests/test_smtp_socket_acceptance.py @@ -1,4 +1,5 @@ """Exercise SMTP over a real, certificate-verified loopback TLS socket.""" + import socket import ssl import subprocess @@ -17,10 +18,28 @@ @pytest.fixture def tls_contexts(tmp_path): cert, key = tmp_path / 'cert.pem', tmp_path / 'key.pem' - subprocess.run(['openssl', 'req', '-x509', '-newkey', 'rsa:2048', '-nodes', - '-keyout', str(key), '-out', str(cert), '-days', '1', - '-subj', '/CN=localhost', '-addext', 'subjectAltName=IP:127.0.0.1'], - check=True, capture_output=True) + subprocess.run( + [ + 'openssl', + 'req', + '-x509', + '-newkey', + 'rsa:2048', + '-nodes', + '-keyout', + str(key), + '-out', + str(cert), + '-days', + '1', + '-subj', + '/CN=localhost', + '-addext', + 'subjectAltName=IP:127.0.0.1', + ], + check=True, + capture_output=True, + ) server = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) server.load_cert_chain(cert, key) client = ssl.create_default_context(cafile=str(cert)) @@ -41,9 +60,11 @@ def serve(): with context.wrap_socket(conn, server_side=True) as conn: conn.settimeout(5) with conn.makefile('rwb') as wire: + def reply(value): wire.write(value + b'\r\n') wire.flush() + reply(b'220 localhost isolated test sink') while line := wire.readline(): commands.append(line.strip()) @@ -53,7 +74,11 @@ def reply(value): elif verb == b'MAIL': reply(b'250 sender accepted') elif verb == b'RCPT': - reply(b'550 recipient refused' if outcome == 'refused' else b'250 recipient accepted') + reply( + b'550 recipient refused' + if outcome == 'refused' + else b'250 recipient accepted' + ) elif verb == b'RSET': reply(b'250 reset') elif verb == b'DATA': @@ -83,20 +108,37 @@ def reply(value): def sender(port): - return SmtpSender(SimpleNamespace(enabled=True, host='127.0.0.1', port=port, - sender='sender@example.test', tls='tls', credential_file=None)) + return SmtpSender( + SimpleNamespace( + enabled=True, + host='127.0.0.1', + port=port, + sender='sender@example.test', + tls='tls', + credential_file=None, + ) + ) def job(): - return SimpleNamespace(id='isolated-smtp-acceptance', arguments={ - 'recipient': 'reader@example.test', 'subject': 'minimal notice', - 'text': 'Open your inbox.\n.dot-stuffed line'}) + return SimpleNamespace( + id='isolated-smtp-acceptance', + arguments={ + 'recipient': 'reader@example.test', + 'subject': 'minimal notice', + 'text': 'Open your inbox.\n.dot-stuffed line', + }, + ) @pytest.mark.asyncio -@pytest.mark.parametrize(('outcome', 'expected'), [ - ('accepted', 'sent'), ('disconnect', 'uncertain'), ('refused', 'uncertain')]) -async def test_real_tls_smtp_success_refusal_and_lost_ack(tls_contexts, monkeypatch, outcome, expected): +@pytest.mark.parametrize( + ('outcome', 'expected'), + [('accepted', 'sent'), ('disconnect', 'uncertain'), ('refused', 'uncertain')], +) +async def test_real_tls_smtp_success_refusal_and_lost_ack( + tls_contexts, monkeypatch, outcome, expected +): server, client = tls_contexts monkeypatch.setattr(ssl, 'create_default_context', lambda: client) # Keep EHLO hostname discovery independent of the host machine's DNS. @@ -105,7 +147,8 @@ async def test_real_tls_smtp_success_refusal_and_lost_ack(tls_contexts, monkeypa assert await sender(port).send(job()) == expected assert sum(command.upper().startswith(b'MAIL ') for command in commands) == 1 assert [command.lower() for command in commands if command.upper().startswith(b'RCPT ')] == [ - b'rcpt to:'] + b'rcpt to:' + ] assert len(messages) == (0 if outcome == 'refused' else 1) if messages: message = BytesParser(policy=policy.default).parsebytes(messages[0]) diff --git a/tests/test_sqlite_connection_lifecycle.py b/tests/test_sqlite_connection_lifecycle.py new file mode 100644 index 00000000..b9555656 --- /dev/null +++ b/tests/test_sqlite_connection_lifecycle.py @@ -0,0 +1,84 @@ +"""SQLite setup and backup failures must release every acquired connection.""" + +import sqlite3 + +import pytest + +from msg.storage.sqlite import SqliteMetadataStore + + +def assert_closed(connection): + with pytest.raises(sqlite3.ProgrammingError, match='closed database'): + connection.execute('SELECT 1') + + +@pytest.mark.parametrize('statement', ['PRAGMA foreign_keys=ON', 'PRAGMA synchronous=FULL']) +def test_connection_setup_failure_closes_connection(tmp_path, monkeypatch, statement): + store = SqliteMetadataStore(tmp_path / 'metadata.db') + connect = sqlite3.connect + acquired = [] + + class BrokenConnection(sqlite3.Connection): + def execute(self, query, *args, **kwargs): + if query == statement: + raise sqlite3.OperationalError('injected setup failure') + return super().execute(query, *args, **kwargs) + + def acquire(*args, **kwargs): + connection = connect(*args, **kwargs, factory=BrokenConnection) + acquired.append(connection) + return connection + + monkeypatch.setattr(sqlite3, 'connect', acquire) + with pytest.raises(sqlite3.OperationalError, match='injected setup failure'): + store._connect() + assert len(acquired) == 1 + assert_closed(acquired[0]) + + +def test_backup_target_open_failure_closes_source(tmp_path, monkeypatch): + store = SqliteMetadataStore(tmp_path / 'metadata.db') + target = tmp_path / 'backup.db' + connect = sqlite3.connect + acquired = [] + + def acquire(path, *args, **kwargs): + if path == target: + raise sqlite3.OperationalError('injected target open failure') + connection = connect(path, *args, **kwargs) + acquired.append(connection) + return connection + + monkeypatch.setattr(sqlite3, 'connect', acquire) + with pytest.raises(sqlite3.OperationalError, match='injected target open failure'): + store.backup(target) + assert len(acquired) == 1 + assert_closed(acquired[0]) + + +@pytest.mark.parametrize('fail', [False, True]) +def test_backup_closes_source_and_target(tmp_path, monkeypatch, fail): + store = SqliteMetadataStore(tmp_path / 'metadata.db') + connect = sqlite3.connect + acquired = [] + + class BackupConnection(sqlite3.Connection): + def backup(self, target, *args, **kwargs): + if fail: + raise sqlite3.OperationalError('injected copy failure') + return super().backup(target, *args, **kwargs) + + def acquire(*args, **kwargs): + connection = connect(*args, **kwargs, factory=BackupConnection) + acquired.append(connection) + return connection + + monkeypatch.setattr(sqlite3, 'connect', acquire) + if fail: + with pytest.raises(sqlite3.OperationalError, match='injected copy failure'): + store.backup(tmp_path / 'backup.db') + else: + store.backup(tmp_path / 'backup.db') + assert len(acquired) == 2 + for connection in acquired: + assert_closed(connection) diff --git a/tests/test_ssh_process_identity.py b/tests/test_ssh_process_identity.py index fc44905c..72e61e8e 100644 --- a/tests/test_ssh_process_identity.py +++ b/tests/test_ssh_process_identity.py @@ -1,8 +1,9 @@ """Process names and root-owned interpreters are not an SSH authentication channel.""" + import os -from pathlib import Path import subprocess import sys +from pathlib import Path from types import SimpleNamespace import pytest @@ -32,8 +33,14 @@ def test_real_unprivileged_process_cannot_spoof_an_sshd_ancestor(name): # normally already runs as an unprivileged account; no service/user changes. identity = {'user': 65534, 'group': 65534, 'extra_groups': []} if os.geteuid() == 0 else {} environment = {**os.environ, 'PYTHONPATH': str(Path(__file__).resolve().parents[1] / 'src')} - result = subprocess.run([sys.executable, '-c', parent], env=environment, - capture_output=True, text=True, timeout=15, **identity) + result = subprocess.run( + [sys.executable, '-c', parent], + env=environment, + capture_output=True, + text=True, + timeout=15, + **identity, + ) assert result.returncode == 0, result.stdout + result.stderr assert result.stdout.strip() == 'ssh_os_isolation_required' @@ -43,15 +50,18 @@ def proc_tree(monkeypatch, tmp_path, *, owner, real_uid, effective_uid, parent=1 directory.mkdir() (directory / 'comm').write_text('sshd\n') (directory / 'status').write_text( - f'Uid:\t{real_uid}\t{effective_uid}\t{effective_uid}\t{effective_uid}\nPPid:\t{parent}\n') + f'Uid:\t{real_uid}\t{effective_uid}\t{effective_uid}\t{effective_uid}\nPPid:\t{parent}\n' + ) monkeypatch.setattr(ssh, 'Path', lambda root: tmp_path) monkeypatch.setattr(ssh.os, 'geteuid', lambda: 1000) monkeypatch.setattr(ssh.os, 'getppid', lambda: 123) original_stat = Path.stat + def stat(path, *args, **kwargs): if path == directory: return SimpleNamespace(st_uid=owner) return original_stat(path, *args, **kwargs) + monkeypatch.setattr(Path, 'stat', stat) return directory @@ -62,8 +72,9 @@ def test_privileged_sshd_ancestor_does_not_require_readable_proc_exe(monkeypatch @pytest.mark.parametrize('owner,real_uid,effective_uid', [(1000, 0, 0), (0, 1000, 0), (0, 0, 1000)]) -def test_inconsistent_or_unprivileged_sshd_identity_fails_closed(monkeypatch, tmp_path, - owner, real_uid, effective_uid): +def test_inconsistent_or_unprivileged_sshd_identity_fails_closed( + monkeypatch, tmp_path, owner, real_uid, effective_uid +): proc_tree(monkeypatch, tmp_path, owner=owner, real_uid=real_uid, effective_uid=effective_uid) with pytest.raises(Failure, match='ssh_os_isolation_required'): ssh.require_sshd_process() diff --git a/tests/test_ssh_rss.py b/tests/test_ssh_rss.py index b124bc78..6545b891 100644 --- a/tests/test_ssh_rss.py +++ b/tests/test_ssh_rss.py @@ -1,133 +1,256 @@ +from datetime import timedelta + import pytest from cryptography.hazmat.primitives import serialization from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey -from msg.core.codec import canonical,wire +from test_service import NOW, call, register + +from msg.core.codec import canonical, wire from msg.core.errors import Failure from msg.core.executor import OperationExecutor from msg.core.requests import request_for -from msg.extensions.ssh import SSHAuthenticator,parse_command,public_key +from msg.extensions.ssh import SSHAuthenticator, parse_command from msg.security.crypto import Ed25519Signer -from test_service import register,call,NOW -from datetime import timedelta def test_ssh_command_allowlist_has_no_shell_or_root_proxy(): - assert parse_command("git-upload-pack '/@alice/code.git'")==('git-upload-pack','/@alice/code.git') - assert parse_command("msg call discovery.get '{\"id\":\"/main\"}'")[0]=='call' - for command in ('','sh','bash -c id','msgd cert issue a','git-upload-pack /etc/passwd', - 'git-upload-pack /@a/x.git;id','msg call identity.register {} extra','scp -t .'): - with pytest.raises(Failure):parse_command(command) + assert parse_command("git-upload-pack '/@alice/code.git'") == ( + 'git-upload-pack', + '/@alice/code.git', + ) + assert parse_command('msg call discovery.get \'{"id":"/main"}\'')[0] == 'call' + for command in ( + '', + 'sh', + 'bash -c id', + 'msgd cert issue a', + 'git-upload-pack /etc/passwd', + 'git-upload-pack /@a/x.git;id', + 'msg call identity.register {} extra', + 'scp -t .', + ): + with pytest.raises(Failure): + parse_command(command) @pytest.mark.asyncio async def test_registered_ssh_key_binds_identity_and_ceiling(installed): - app,_=installed - key,uid,_=await register(app,'ssh-agent') - ssh=Ed25519Signer.generate() - line=Ed25519PrivateKey.from_private_bytes(ssh.private_bytes()).public_key().public_bytes( - serialization.Encoding.OpenSSH,serialization.PublicFormat.OpenSSH).decode() - pop=ssh.sign(canonical({'subject_id':uid,'public_key':line}),purpose='ssh-key-add') - grant=next(g for g in app.base_grants() if g.capability=='discovery.basic') - added=await call(app,'identity.ssh_key_add',{'public_key':line,'proof':wire(pop),'ceiling':[wire(grant)]},key=key,subject=uid) - assert added.status=='ok',wire(added) - authenticator=SSHAuthenticator(app,ssh.key_id) - executor=OperationExecutor(app.registry,app.metadata,app.contents,authenticator,app.authorizer,app.clock,app.receipt_signer) - packet=request_for('discovery.get',{'id':'/main'},app.settings.service_url,subject=uid,expires_at=NOW+timedelta(seconds=60)) - read=await executor.execute(packet) - assert read.status=='ok' and read.actor==uid,wire(read) - bad=await executor.execute(request_for('content.post_create',{'parent':'/main','body':'not authorized'}, - app.settings.service_url,subject=uid,expires_at=NOW+timedelta(seconds=60))) - assert bad.status=='error' - revoked=await call(app,'identity.ssh_key_revoke',{'key_id':ssh.key_id},key=key,subject=uid) - assert revoked.status=='ok',wire(revoked) - assert (await executor.execute(packet)).error.code=='credential_revoked' + app, _ = installed + key, uid, _ = await register(app, 'ssh-agent') + ssh = Ed25519Signer.generate() + line = ( + Ed25519PrivateKey + .from_private_bytes(ssh.private_bytes()) + .public_key() + .public_bytes(serialization.Encoding.OpenSSH, serialization.PublicFormat.OpenSSH) + .decode() + ) + pop = ssh.sign(canonical({'subject_id': uid, 'public_key': line}), purpose='ssh-key-add') + grant = next(g for g in app.base_grants() if g.capability == 'discovery.basic') + added = await call( + app, + 'identity.ssh_key_add', + {'public_key': line, 'proof': wire(pop), 'ceiling': [wire(grant)]}, + key=key, + subject=uid, + ) + assert added.status == 'ok', wire(added) + authenticator = SSHAuthenticator(app, ssh.key_id) + executor = OperationExecutor( + app.registry, + app.metadata, + app.contents, + authenticator, + app.authorizer, + app.clock, + app.receipt_signer, + ) + packet = request_for( + 'discovery.get', + {'id': '/main'}, + app.settings.service_url, + subject=uid, + expires_at=NOW + timedelta(seconds=60), + ) + read = await executor.execute(packet) + assert read.status == 'ok' and read.actor == uid, wire(read) + bad = await executor.execute( + request_for( + 'content.post_create', + {'parent': '/main', 'body': 'not authorized'}, + app.settings.service_url, + subject=uid, + expires_at=NOW + timedelta(seconds=60), + ) + ) + assert bad.status == 'error' + revoked = await call( + app, 'identity.ssh_key_revoke', {'key_id': ssh.key_id}, key=key, subject=uid + ) + assert revoked.status == 'ok', wire(revoked) + assert (await executor.execute(packet)).error.code == 'credential_revoked' + @pytest.mark.asyncio -async def test_git_reference_guard_real_commit_and_revoked_key(installed,tmp_path): - from msg.extensions.ssh_git import ReferenceGuard,guarded_command +async def test_git_reference_guard_real_commit_and_revoked_key(installed, tmp_path): from msg.extensions.repositories import NativeGitStore - app,_=installed - key,uid,_=await register(app,'ssh-git-owner') - ssh=Ed25519Signer.generate() - line=Ed25519PrivateKey.from_private_bytes(ssh.private_bytes()).public_key().public_bytes( - serialization.Encoding.OpenSSH,serialization.PublicFormat.OpenSSH).decode() - pop=ssh.sign(canonical({'subject_id':uid,'public_key':line}),purpose='ssh-key-add') - grant=next(g for g in app.base_grants() if g.capability=='git.basic') - added=await call(app,'identity.ssh_key_add',{'public_key':line,'proof':wire(pop),'ceiling':[wire(grant)]},key=key,subject=uid) - assert added.status=='ok',wire(added) - repo=await call(app,'git.create',{'parent':'/@ssh-git-owner','name':'code.git'},key=key,subject=uid) - rid=repo.resources[0].id - store=NativeGitStore(app) - import asyncio,subprocess + from msg.extensions.ssh_git import ReferenceGuard, guarded_command + + app, _ = installed + key, uid, _ = await register(app, 'ssh-git-owner') + ssh = Ed25519Signer.generate() + line = ( + Ed25519PrivateKey + .from_private_bytes(ssh.private_bytes()) + .public_key() + .public_bytes(serialization.Encoding.OpenSSH, serialization.PublicFormat.OpenSSH) + .decode() + ) + pop = ssh.sign(canonical({'subject_id': uid, 'public_key': line}), purpose='ssh-key-add') + grant = next(g for g in app.base_grants() if g.capability == 'git.basic') + added = await call( + app, + 'identity.ssh_key_add', + {'public_key': line, 'proof': wire(pop), 'ceiling': [wire(grant)]}, + key=key, + subject=uid, + ) + assert added.status == 'ok', wire(added) + repo = await call( + app, 'git.create', {'parent': '/@ssh-git-owner', 'name': 'code.git'}, key=key, subject=uid + ) + rid = repo.resources[0].id + store = NativeGitStore(app) + import asyncio + import subprocess + # Real Git objects with an empty tree; neither arbitrary shell nor fake refs. - tree=(await asyncio.to_thread(store._run,rid,'mktree',input=b'')).decode().strip() - env={**store.env,'GIT_AUTHOR_NAME':'Test','GIT_AUTHOR_EMAIL':'test@example.invalid'} - commit=subprocess.run(['git','--git-dir',str(store.path(rid)),'commit-tree',tree],input=b'initial\n',capture_output=True,env=env,check=True).stdout.decode().strip() - auth=SSHAuthenticator(app,ssh.key_id) - executor=OperationExecutor(app.registry,app.metadata,app.contents,auth,app.authorizer,app.clock,app.receipt_signer) + tree = (await asyncio.to_thread(store._run, rid, 'mktree', input=b'')).decode().strip() + env = {**store.env, 'GIT_AUTHOR_NAME': 'Test', 'GIT_AUTHOR_EMAIL': 'test@example.invalid'} + commit = ( + subprocess + .run( + ['git', '--git-dir', str(store.path(rid)), 'commit-tree', tree], + input=b'initial\n', + capture_output=True, + env=env, + check=True, + ) + .stdout.decode() + .strip() + ) + auth = SSHAuthenticator(app, ssh.key_id) + executor = OperationExecutor( + app.registry, + app.metadata, + app.contents, + auth, + app.authorizer, + app.clock, + app.receipt_signer, + ) + async def job(): - packet=request_for('git.receive',{'id':rid},app.settings.service_url,subject=uid,expires_at=NOW+timedelta(seconds=60)) - accepted=await executor.execute(packet) - assert accepted.status=='accepted',wire(accepted) - async with app.metadata.transaction(write=False) as tx:return await tx.job(accepted.data['job_id']) - initial=await job() + packet = request_for( + 'git.receive', + {'id': rid}, + app.settings.service_url, + subject=uid, + expires_at=NOW + timedelta(seconds=60), + ) + accepted = await executor.execute(packet) + assert accepted.status == 'accepted', wire(accepted) + async with app.metadata.transaction(write=False) as tx: + return await tx.job(accepted.data['job_id']) + + initial = await job() # Git 2.54+ invokes reference-transaction with a pre-lock `preparing` # phase. Exercise it explicitly so older local Git versions do not hide # compatibility regressions. - probe=ReferenceGuard(app,initial) - probe_task=asyncio.create_task(probe.run()) - changes=f"{'0'*40} {commit} refs/heads/main" - assert (await probe.message({'state':'preparing','changes':changes}))['ok'] - await probe.message({'state':'stop'}) + probe = ReferenceGuard(app, initial) + probe_task = asyncio.create_task(probe.run()) + changes = f'{"0" * 40} {commit} refs/heads/main' + assert (await probe.message({'state': 'preparing', 'changes': changes}))['ok'] + await probe.message({'state': 'stop'}) await probe_task - code=await guarded_command(app,initial,lambda _:['update-ref','refs/heads/main',commit,'0'*40]) - assert code==0 + code = await guarded_command( + app, initial, lambda _: ['update-ref', 'refs/heads/main', commit, '0' * 40] + ) + assert code == 0 async with app.metadata.transaction(write=False) as tx: - assert (await tx.job(initial.id)).state=='done' - assert (await tx.resource(rid)).generation==repo.data['generation']+1 - assert tx.one('SELECT COUNT(*) FROM audit')[0]>0 + assert (await tx.job(initial.id)).state == 'done' + assert (await tx.resource(rid)).generation == repo.data['generation'] + 1 + assert tx.one('SELECT COUNT(*) FROM audit')[0] > 0 # A real Git ref transaction that loses stdin after prepare must abort # every staged ref. This covers EOF, not host power loss or disk durability. - interrupted=await job() - before_refs,_=await store.refs(rid) + interrupted = await job() + before_refs, _ = await store.refs(rid) async with app.metadata.transaction(write=False) as tx: - before_generation=(await tx.resource(rid)).generation - commands=(f'start\ncreate refs/heads/second {commit}\n' - f'create refs/heads/third {commit}\nprepare\n').encode() - code,output,changed=await guarded_command(app,interrupted, - lambda _:['update-ref','--stdin'],input_data=commands,capture_output=True) - assert code==0 and b'prepare: ok' in output and not changed - assert (await store.refs(rid))[0]==before_refs + before_generation = (await tx.resource(rid)).generation + commands = ( + f'start\ncreate refs/heads/second {commit}\ncreate refs/heads/third {commit}\nprepare\n' + ).encode() + code, output, changed = await guarded_command( + app, + interrupted, + lambda _: ['update-ref', '--stdin'], + input_data=commands, + capture_output=True, + ) + assert code == 0 and b'prepare: ok' in output and not changed + assert (await store.refs(rid))[0] == before_refs async with app.metadata.transaction(write=False) as tx: - assert (await tx.resource(rid)).generation==before_generation - pending=await job() - await call(app,'identity.ssh_key_revoke',{'key_id':ssh.key_id},key=key,subject=uid) - denied=await guarded_command(app,pending,lambda _:['update-ref','refs/heads/other',commit,'0'*40]) - assert denied!=0 - refs,_=await store.refs(rid) - assert [r['name'] for r in refs]==['refs/heads/main'] + assert (await tx.resource(rid)).generation == before_generation + pending = await job() + await call(app, 'identity.ssh_key_revoke', {'key_id': ssh.key_id}, key=key, subject=uid) + denied = await guarded_command( + app, pending, lambda _: ['update-ref', 'refs/heads/other', commit, '0' * 40] + ) + assert denied != 0 + refs, _ = await store.refs(rid) + assert [r['name'] for r in refs] == ['refs/heads/main'] async with app.metadata.transaction(write=False) as tx: - assert (await tx.job(pending.id)).state=='failed' + assert (await tx.job(pending.id)).state == 'failed' + @pytest.mark.asyncio async def test_rss_is_read_only_acl_filtered_and_escapes_titles(installed): - import httpx from xml.etree import ElementTree as ET + + import httpx + from msg.transports.http import create_app - app,_=installed - key,uid,_=await register(app,'feed-author') - post=await call(app,'content.post_create',{'parent':'/main','body':'public'},key=key,subject=uid) - secret=await call(app,'content.post_create',{'parent':'/@feed-author/files','body':'private'},key=key,subject=uid) - async with app.metadata.transaction(write=False) as tx:before=tx.one('SELECT COUNT(*) FROM events')[0] - async with httpx.AsyncClient(transport=httpx.ASGITransport(app=create_app(app)),base_url=app.settings.service_url) as http: - feed=await http.get('/rss') - assert feed.status_code==200 - ids=[x.text for x in ET.fromstring(feed.content).findall('./channel/item/guid')] + + app, _ = installed + key, uid, _ = await register(app, 'feed-author') + post = await call( + app, 'content.post_create', {'parent': '/main', 'body': 'public'}, key=key, subject=uid + ) + secret = await call( + app, + 'content.post_create', + {'parent': '/@feed-author/files', 'body': 'private'}, + key=key, + subject=uid, + ) + async with app.metadata.transaction(write=False) as tx: + before = tx.one('SELECT COUNT(*) FROM events')[0] + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=create_app(app)), base_url=app.settings.service_url + ) as http: + feed = await http.get('/rss') + assert feed.status_code == 200 + ids = [x.text for x in ET.fromstring(feed.content).findall('./channel/item/guid')] assert post.resources[0].id in ids and secret.resources[0].id not in ids - latest=await http.get('/latest/post') - assert latest.status_code==200 - raw=await http.get('/_id/'+post.resources[0].id+'/raw',headers={'Range':'bytes=1-3'}) - assert raw.status_code==206 and raw.content==b'ubl' - bad=await http.get('/_id/'+post.resources[0].id+'/raw',headers={'Range':'bytes=100-105'}) - assert bad.status_code==416 - async with app.metadata.transaction(write=False) as tx:assert tx.one('SELECT COUNT(*) FROM events')[0]==before \ No newline at end of file + latest = await http.get('/latest/post') + assert latest.status_code == 200 + raw = await http.get( + '/_id/' + post.resources[0].id + '/raw', headers={'Range': 'bytes=1-3'} + ) + assert raw.status_code == 206 and raw.content == b'ubl' + bad = await http.get( + '/_id/' + post.resources[0].id + '/raw', headers={'Range': 'bytes=100-105'} + ) + assert bad.status_code == 416 + async with app.metadata.transaction(write=False) as tx: + assert tx.one('SELECT COUNT(*) FROM events')[0] == before diff --git a/tests/test_stable_resource_views.py b/tests/test_stable_resource_views.py index 90e9bd51..10325f50 100644 --- a/tests/test_stable_resource_views.py +++ b/tests/test_stable_resource_views.py @@ -1,4 +1,5 @@ """Stable resource-ID projections keep authorization and canonical paths separate.""" + from datetime import timedelta import httpx @@ -14,22 +15,34 @@ async def test_stable_id_views_survive_move_and_reuse_existing_projections(installed): app, _ = installed key, uid, cert = await register(app, 'stable-view-owner') - created = await call(app, 'content.post_create', {'parent': '/main', 'body': 'stable body'}, - key=key, subject=uid, certs=(cert,)) + created = await call( + app, + 'content.post_create', + {'parent': '/main', 'body': 'stable body'}, + key=key, + subject=uid, + certs=(cert,), + ) rid, revision = created.resources[0].id, created.resources[0].revision base = f'/_r/{rid}' - async with httpx.AsyncClient(transport=httpx.ASGITransport(app=create_app(app)), - base_url='http://testserver') as http: + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=create_app(app)), base_url='http://testserver' + ) as http: first = await http.get(base + '/json') assert first.status_code == 200, first.text assert first.json()['id'] == rid assert (await http.get(base + '/raw')).content == b'stable body' assert (await http.get(base + '/rev/' + revision)).status_code == 200 - moved = await call(app, 'content.move', {'id': rid, 'parent': '/intro', - 'name': 'renamed.md'}, - key=key, subject=uid, certs=(cert,), - expected=((rid, created.data['generation']),)) + moved = await call( + app, + 'content.move', + {'id': rid, 'parent': '/intro', 'name': 'renamed.md'}, + key=key, + subject=uid, + certs=(cert,), + expected=((rid, created.data['generation']),), + ) assert moved.status == 'ok' async with app.metadata.transaction(write=False) as tx: events_before = tx.one('SELECT COUNT(*) FROM events')[0] @@ -57,14 +70,28 @@ async def test_stable_id_views_survive_move_and_reuse_existing_projections(insta async def test_private_stable_views_do_not_disclose_via_head_etag_or_status(installed): app, _ = installed key, uid, cert = await register(app, 'stable-private-owner') - created = await call(app, 'content.post_create', {'parent': '/main', 'body': 'hidden body'}, - key=key, subject=uid, certs=(cert,)) + created = await call( + app, + 'content.post_create', + {'parent': '/main', 'body': 'hidden body'}, + key=key, + subject=uid, + certs=(cert,), + ) rid, revision = created.resources[0].id, created.resources[0].revision - await call(app, 'content.chmod', {'id': rid, 'mode': '0600'}, key=key, subject=uid, - certs=(cert,), expected=((rid, created.data['generation']),)) + await call( + app, + 'content.chmod', + {'id': rid, 'mode': '0600'}, + key=key, + subject=uid, + certs=(cert,), + expected=((rid, created.data['generation']),), + ) base = f'/_r/{rid}' - async with httpx.AsyncClient(transport=httpx.ASGITransport(app=create_app(app)), - base_url='http://testserver') as http: + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=create_app(app)), base_url='http://testserver' + ) as http: for suffix in ('/json', '/meta', '/raw', '/history', '/rev/' + revision): for method in (http.get, http.head): response = await method(base + suffix, headers={'If-None-Match': '*'}) @@ -75,9 +102,14 @@ async def test_private_stable_views_do_not_disclose_via_head_etag_or_status(inst assert missing.status_code == 404 assert 'etag' not in missing.headers - signed = request_for('discovery.get', {'id': rid, 'view': 'meta'}, - app.settings.service_url, signer=key, subject=uid, - expires_at=NOW + timedelta(seconds=120)) + signed = request_for( + 'discovery.get', + {'id': rid, 'view': 'meta'}, + app.settings.service_url, + signer=key, + subject=uid, + expires_at=NOW + timedelta(seconds=120), + ) headers = {'X-Msg-Request': b64(canonical(signed))} permitted = await http.get(base + '/meta', headers=headers) assert permitted.status_code == 200, permitted.text diff --git a/tests/test_stdio_cli.py b/tests/test_stdio_cli.py index 6cf19971..37de9bdb 100644 --- a/tests/test_stdio_cli.py +++ b/tests/test_stdio_cli.py @@ -3,106 +3,171 @@ import os import socket import sys -import tempfile from pathlib import Path import httpx import pytest import uvicorn +from test_service import NOW +from msg.admin.root import _approve_csr, _provision from msg.application import Application -from msg.admin.root import _provision,_approve_csr -from msg.client import ClientState,MsgClient +from msg.client import ClientState, MsgClient from msg.config import write_example -from msg.core.codec import canonical,loads +from msg.core.codec import canonical, loads from msg.transports.client import HTTPTransport from msg.transports.http import create_app from msg.transports.mcp import MCPServer from msg.transports.stdio import serve_stdio -from test_service import NOW @pytest.mark.asyncio -async def test_local_stdio_signs_tools_and_never_exposes_root(installed,tmp_path): - app,_=installed - async with httpx.AsyncClient(transport=httpx.ASGITransport(app=create_app(app)),base_url=app.settings.service_url) as http: - client=MsgClient(ClientState(tmp_path/'cli',server=app.settings.service_url), - HTTPTransport(app.settings.service_url,http=http),clock=lambda:NOW) +async def test_local_stdio_signs_tools_and_never_exposes_root(installed, tmp_path): + app, _ = installed + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=create_app(app)), base_url=app.settings.service_url + ) as http: + client = MsgClient( + ClientState(tmp_path / 'cli', server=app.settings.service_url), + HTTPTransport(app.settings.service_url, http=http), + clock=lambda: NOW, + ) await client.register('stdio-agent') - registry=await client.contract_registry() - assert registry.operation('identity.register',1).version==1 - newer=registry.operation('identity.register',2) + registry = await client.contract_registry() + assert registry.operation('identity.register', 1).version == 1 + newer = registry.operation('identity.register', 2) await registry.load_schemas((newer,)) - assert tuple(registry.schema(newer.input_schema)['required'])==( - 'handle','public_key','encryption_recipient') - mcp=MCPServer(app) - listed=[] - cursor=None + assert tuple(registry.schema(newer.input_schema)['required']) == ( + 'handle', + 'public_key', + 'encryption_recipient', + ) + mcp = MCPServer(app) + listed = [] + cursor = None while True: - page=await mcp.tools(cursor) + page = await mcp.tools(cursor) listed.extend(tool['name'] for tool in page['tools']) - cursor=page.get('nextCursor') + cursor = page.get('nextCursor') if cursor is None: break assert 'identity.register' in listed and 'identity.register@2' in listed - assert len(listed)==len(set(listed)) - request={'jsonrpc':'2.0','id':3,'method':'tools/call','params':{ - 'name':'content.post_create','arguments':{'parent':'/main','body':'stdio message'}}} - messages=[{'jsonrpc':'2.0','id':1,'method':'initialize','params':{'protocolVersion':'2025-11-25'}}, - {'jsonrpc':'2.0','id':2,'method':'tools/list'},request,request, - {'jsonrpc':'2.0','id':4,'method':'tools/call','params':{'name':'root.cert.issue','arguments':{}}}] - source=io.BytesIO(b''.join(canonical(m)+b'\n' for m in messages));target=io.BytesIO() - await serve_stdio(client,source,target) - replies=[loads(line) for line in target.getvalue().splitlines()] - assert len(replies)==5 - assert replies[2]['result']['structuredContent']['status']=='ok' + assert len(listed) == len(set(listed)) + request = { + 'jsonrpc': '2.0', + 'id': 3, + 'method': 'tools/call', + 'params': { + 'name': 'content.post_create', + 'arguments': {'parent': '/main', 'body': 'stdio message'}, + }, + } + messages = [ + { + 'jsonrpc': '2.0', + 'id': 1, + 'method': 'initialize', + 'params': {'protocolVersion': '2025-11-25'}, + }, + {'jsonrpc': '2.0', 'id': 2, 'method': 'tools/list'}, + request, + request, + { + 'jsonrpc': '2.0', + 'id': 4, + 'method': 'tools/call', + 'params': {'name': 'root.cert.issue', 'arguments': {}}, + }, + ] + source = io.BytesIO(b''.join(canonical(m) + b'\n' for m in messages)) + target = io.BytesIO() + await serve_stdio(client, source, target) + replies = [loads(line) for line in target.getvalue().splitlines()] + assert len(replies) == 5 + assert replies[2]['result']['structuredContent']['status'] == 'ok' assert replies[3]['result']['structuredContent']['replayed'] is True assert 'error' in replies[4] for tool in replies[1]['result']['tools']: assert 'packet' not in tool['inputSchema']['properties'] async with app.metadata.transaction(write=False) as tx: - assert tx.one("SELECT COUNT(*) FROM resources WHERE type='post'")[0]==1 + assert tx.one("SELECT COUNT(*) FROM resources WHERE type='post'")[0] == 1 @pytest.mark.asyncio -async def test_real_cli_process_and_stdio_over_loopback(tmp_path,pg_dsn): - sock=socket.socket();sock.bind(('127.0.0.1',0));port=sock.getsockname()[1] +async def test_real_cli_process_and_stdio_over_loopback(tmp_path, pg_dsn): + sock = socket.socket() + sock.bind(('127.0.0.1', 0)) + port = sock.getsockname()[1] sock.setblocking(False) - url=f'http://127.0.0.1:{port}' - app=Application(write_example(tmp_path/'server-etc',tmp_path/'server-data',url, - postgres_dsn=pg_dsn)) - csr,root=await _provision(app,'a-long-test-passphrase') - await _approve_csr(app,csr,root,expected_digest=None,operator='test') - server=uvicorn.Server(uvicorn.Config(create_app(app),log_level='critical',access_log=False)) - task=asyncio.create_task(server.serve(sockets=[sock])) + url = f'http://127.0.0.1:{port}' + app = Application( + write_example(tmp_path / 'server-etc', tmp_path / 'server-data', url, postgres_dsn=pg_dsn) + ) + csr, root = await _provision(app, 'a-long-test-passphrase') + await _approve_csr(app, csr, root, expected_digest=None, operator='test') + server = uvicorn.Server(uvicorn.Config(create_app(app), log_level='critical', access_log=False)) + task = asyncio.create_task(server.serve(sockets=[sock])) for _ in range(100): - if server.started:break - await asyncio.sleep(.02) + if server.started: + break + await asyncio.sleep(0.02) assert server.started - env={**os.environ,'PYTHONPATH':str(Path(__file__).parents[1]/'src')} - base=[sys.executable,'-m','msg.cli','--config-dir',str(tmp_path/'client'),'--server',url] - async def cli(*args,input=None): - process=await asyncio.create_subprocess_exec(*base,*args,env=env, - stdin=asyncio.subprocess.PIPE,stdout=asyncio.subprocess.PIPE,stderr=asyncio.subprocess.PIPE) - out,err=await asyncio.wait_for(process.communicate(input),20) - assert process.returncode==0,err.decode()+out.decode() + env = {**os.environ, 'PYTHONPATH': str(Path(__file__).parents[1] / 'src')} + base = [ + sys.executable, + '-m', + 'msg.cli', + '--config-dir', + str(tmp_path / 'client'), + '--server', + url, + ] + + async def cli(*args, input=None): + process = await asyncio.create_subprocess_exec( + *base, + *args, + env=env, + stdin=asyncio.subprocess.PIPE, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + out, err = await asyncio.wait_for(process.communicate(input), 20) + assert process.returncode == 0, err.decode() + out.decode() return [loads(line) for line in out.splitlines()] + try: - registration=(await cli('identity','new','real-cli'))[0] - assert registration['status']=='ok' - posted=(await cli('--transport','path_get','post','/main','--text','from subprocess'))[0] - assert posted['status']=='ok' - rid=posted['resources'][0]['id'] - reply=(await cli('reply',rid,'--text','reply subprocess'))[0] - assert reply['status']=='ok' - read=(await cli('read',rid,'--ack'))[0] - assert read['read']['status']=='ok' and read['ack']['status']=='ok',read - messages=[{'jsonrpc':'2.0','id':1,'method':'initialize','params':{'protocolVersion':'2025-11-25'}}, - {'jsonrpc':'2.0','id':2,'method':'tools/call','params':{'name':'content.post_create', - 'arguments':{'parent':'/main','body':'real stdio'}}}] - responses=await cli('mcp',input=b''.join(canonical(m)+b'\n' for m in messages)) - assert responses[1]['result']['structuredContent']['status']=='ok' + registration = (await cli('identity', 'new', 'real-cli'))[0] + assert registration['status'] == 'ok' + posted = ( + await cli('--transport', 'path_get', 'post', '/main', '--text', 'from subprocess') + )[0] + assert posted['status'] == 'ok' + rid = posted['resources'][0]['id'] + reply = (await cli('reply', rid, '--text', 'reply subprocess'))[0] + assert reply['status'] == 'ok' + read = (await cli('read', rid, '--ack'))[0] + assert read['read']['status'] == 'ok' and read['ack']['status'] == 'ok', read + messages = [ + { + 'jsonrpc': '2.0', + 'id': 1, + 'method': 'initialize', + 'params': {'protocolVersion': '2025-11-25'}, + }, + { + 'jsonrpc': '2.0', + 'id': 2, + 'method': 'tools/call', + 'params': { + 'name': 'content.post_create', + 'arguments': {'parent': '/main', 'body': 'real stdio'}, + }, + }, + ] + responses = await cli('mcp', input=b''.join(canonical(m) + b'\n' for m in messages)) + assert responses[1]['result']['structuredContent']['status'] == 'ok' finally: - server.should_exit=True + server.should_exit = True await task sock.close() diff --git a/tests/test_storage_commit_faults.py b/tests/test_storage_commit_faults.py index a21ae46d..e09dde8f 100644 --- a/tests/test_storage_commit_faults.py +++ b/tests/test_storage_commit_faults.py @@ -1,4 +1,5 @@ """Real PostgreSQL COMMIT rejection after durable Git/CAS publication.""" + import pytest from test_service import call, register @@ -9,12 +10,23 @@ @pytest.mark.asyncio @pytest.mark.parametrize('media', ['text/plain', 'application/octet-stream']) -async def test_deferred_commit_failure_leaves_collectable_orphan_and_retry(installed, monkeypatch, media): +async def test_deferred_commit_failure_leaves_collectable_orphan_and_retry( + installed, monkeypatch, media +): app, _ = installed key, subject, _ = await register(app, 'commit-fault') - initial = await call(app, 'file.create', {'parent': '/@commit-fault/files', - 'name': 'retained.dat', 'data': b64(b'retained bytes'), 'media_type': media}, - key=key, subject=subject) + initial = await call( + app, + 'file.create', + { + 'parent': '/@commit-fault/files', + 'name': 'retained.dat', + 'data': b64(b'retained bytes'), + 'media_type': media, + }, + key=key, + subject=subject, + ) assert initial.status == 'ok', initial.error rid = initial.resources[0].id tables = ('resources', 'revisions', 'events', 'results', 'audit', 'jobs') @@ -23,24 +35,39 @@ async def test_deferred_commit_failure_leaves_collectable_orphan_and_retry(insta before = tuple(tx.one('SELECT COUNT(*) FROM ' + table)[0] for table in tables) captured = [] commit_revision = app.contents.commit_revision + async def observe(topic, revision): result = await commit_revision(topic, revision) captured.append(revision) return result + monkeypatch.setattr(app.contents, 'commit_revision', observe) # A real deferred PostgreSQL trigger fails inside connection.commit(), after # handler, Event and idempotency rows have been prepared. No mocked principal # or fabricated connection failure can short-circuit that boundary. async with app.metadata.transaction(write=True) as tx: - tx.execute("""CREATE FUNCTION reject_test_revision_commit() RETURNS trigger AS $$ + tx.execute( + """CREATE FUNCTION reject_test_revision_commit() RETURNS trigger AS $$ BEGIN RAISE EXCEPTION 'injected_deferred_commit_failure'; END; - $$ LANGUAGE plpgsql""", write=True) - tx.execute('''CREATE CONSTRAINT TRIGGER reject_test_revision_commit + $$ LANGUAGE plpgsql""", + write=True, + ) + tx.execute( + """CREATE CONSTRAINT TRIGGER reject_test_revision_commit AFTER INSERT ON revisions DEFERRABLE INITIALLY DEFERRED - FOR EACH ROW EXECUTE FUNCTION reject_test_revision_commit()''', write=True) + FOR EACH ROW EXECUTE FUNCTION reject_test_revision_commit()""", + write=True, + ) args = {'id': rid, 'base_revision': old.id, 'data': b64(b'uncommitted bytes')} - failed = await call(app, 'file.write', args, key=key, subject=subject, - expected=((rid, initial.data['generation']),), rid='commit-fault-request') + failed = await call( + app, + 'file.write', + args, + key=key, + subject=subject, + expected=((rid, initial.data['generation']),), + rid='commit-fault-request', + ) assert failed.status == 'error', failed assert len(captured) == 1 # durable publication ran before COMMIT rejected it orphan = captured[0] @@ -61,12 +88,25 @@ async def observe(topic, revision): assert not (app.contents.binary / orphan.content.digest[7:]).exists() assert await app.contents.read_bytes(old.content) == b'retained bytes' assert await app.contents.pinned(old.content, old.id) - retry = await call(app, 'file.write', args, key=key, subject=subject, - expected=((rid, initial.data['generation']),), rid='commit-fault-request') + retry = await call( + app, + 'file.write', + args, + key=key, + subject=subject, + expected=((rid, initial.data['generation']),), + rid='commit-fault-request', + ) assert retry.status == 'ok' and not retry.replayed, retry.error # Old immutable bytes remain retained after a new revision and ordinary archive. - archived = await call(app, 'file.delete', {'id': rid}, key=key, subject=subject, - expected=((rid, retry.data['generation']),)) + archived = await call( + app, + 'file.delete', + {'id': rid}, + key=key, + subject=subject, + expected=((rid, retry.data['generation']),), + ) assert archived.status == 'ok', archived.error async with app.metadata.transaction(write=True) as tx: assert (await _collect(app, tx, grace_seconds=0))['unreferenced_contents_removed'] == 0 diff --git a/tests/test_storage_layout.py b/tests/test_storage_layout.py index dbbebe49..dd3ce0b4 100644 --- a/tests/test_storage_layout.py +++ b/tests/test_storage_layout.py @@ -17,8 +17,9 @@ async def test_new_installation_uses_documented_persistent_layout(tmp_path): assert server.staging_dir == data / 'transfers' / 'staging' assert server.service_keys_dir == data / 'service' - store = GitContentStore(server.content_dir, binary_dir=server.blob_dir, - staging_dir=server.staging_dir) + store = GitContentStore( + server.content_dir, binary_dir=server.blob_dir, staging_dir=server.staging_dir + ) binary = await store.put_bytes(b'\x00\x01', 'application/octet-stream') assert (server.blob_dir / binary.digest[7:]).read_bytes() == b'\x00\x01' assert not (server.content_dir / 'binary').exists() @@ -30,9 +31,14 @@ async def test_new_installation_uses_documented_persistent_layout(tmp_path): assert NativeGitStore(app).path('example') == server.repositories_dir / 'example.git' config = tmp_path / 'etc' / 'msgd.toml' - config.write_text('\n'.join(line for line in config.read_text().splitlines() - if not line.startswith(('repositories =', 'blobs =', - 'service_keys ='))) + '\n') + config.write_text( + '\n'.join( + line + for line in config.read_text().splitlines() + if not line.startswith(('repositories =', 'blobs =', 'service_keys =')) + ) + + '\n' + ) defaults = load_settings(tmp_path / 'etc').server assert defaults.repositories_dir == server.repositories_dir assert defaults.blob_dir == server.blob_dir diff --git a/tests/test_store_foundation.py b/tests/test_store_foundation.py index 66521a5d..202724c7 100644 --- a/tests/test_store_foundation.py +++ b/tests/test_store_foundation.py @@ -1,173 +1,299 @@ """The catalog has immutable facts, while buying and settlement remain absent.""" -import pytest + from types import SimpleNamespace +import pytest +from test_service import call, register + from msg.core.codec import b64, canonical, wire from msg.plugins.store import _body -from test_service import call, register def listing_args(**changes): - return {'name':'sample.bin', 'item_kind':'file', 'price_minor':1250000, - 'currency_id':'primary', 'quantity':2, 'delivery_mode':'managed_instant', - 'escrow_policy':'escrow-v1', 'dispute_policy':'dispute-v1', - 'terms':'The file is delivered after settlement.', **changes} + return { + 'name': 'sample.bin', + 'item_kind': 'file', + 'price_minor': 1250000, + 'currency_id': 'primary', + 'quantity': 2, + 'delivery_mode': 'managed_instant', + 'escrow_policy': 'escrow-v1', + 'dispute_policy': 'dispute-v1', + 'terms': 'The file is delivered after settlement.', + **changes, + } @pytest.mark.asyncio async def test_store_starts_empty_and_listing_requires_seller_signature(installed): app, _ = installed - anonymous=await call(app,'discovery.get',{'id':'/store'}) - assert anonymous.status=='ok',wire(anonymous) + anonymous = await call(app, 'discovery.get', {'id': '/store'}) + assert anonymous.status == 'ok', wire(anonymous) async with app.metadata.transaction(write=False) as tx: - assert tx.one("SELECT COUNT(*) FROM resources WHERE parent='t_store'")[0]==0 - assert tx.one('SELECT COUNT(*) FROM store_packages')[0]==0 - unsigned=await call(app,'store.listing_create',listing_args()) - assert unsigned.status=='error' - key,seller,_=await register(app,'catalog-seller') - created=await call(app,'store.listing_create',listing_args(),key=key,subject=seller) - assert created.status=='ok',wire(created) - listing=created.data['listing'] - assert listing['seller']==seller and listing['state']=='draft' and listing['mode']=='sale' + assert tx.one("SELECT COUNT(*) FROM resources WHERE parent='t_store'")[0] == 0 + assert tx.one('SELECT COUNT(*) FROM store_packages')[0] == 0 + unsigned = await call(app, 'store.listing_create', listing_args()) + assert unsigned.status == 'error' + key, seller, _ = await register(app, 'catalog-seller') + created = await call(app, 'store.listing_create', listing_args(), key=key, subject=seller) + assert created.status == 'ok', wire(created) + listing = created.data['listing'] + assert listing['seller'] == seller and listing['state'] == 'draft' and listing['mode'] == 'sale' assert listing['package_ref'] is None - assert listing['listing_revision']==listing['terms_revision'] - assert (await call(app,'store.listing_get',{'id':listing['listing_id']})).status=='ok' - public_resource=await call(app,'discovery.get',{'id':listing['listing_id']}) - assert public_resource.status=='ok' and listing['listing_id'] in str(public_resource.data) + assert listing['listing_revision'] == listing['terms_revision'] + assert (await call(app, 'store.listing_get', {'id': listing['listing_id']})).status == 'ok' + public_resource = await call(app, 'discovery.get', {'id': listing['listing_id']}) + assert public_resource.status == 'ok' and listing['listing_id'] in str(public_resource.data) # A draft catalog entry is not a purchase API. - assert not any(s.name in {'store.buy','store.order_create','store.escrow_fund'} - for s in app.registry.operations()) - explicit=await call(app,'store.listing_create',listing_args(mode='sale',name='sale.bin'), - key=key,subject=seller) - assert explicit.status=='ok' and explicit.data['listing']['mode']=='sale' - unsupported=await call(app,'store.listing_create',listing_args(mode='bounty',name='bounty.bin'), - key=key,subject=seller) - assert unsupported.error.code=='listing_mode_unsupported' + assert not any( + s.name in {'store.buy', 'store.order_create', 'store.escrow_fund'} + for s in app.registry.operations() + ) + explicit = await call( + app, + 'store.listing_create', + listing_args(mode='sale', name='sale.bin'), + key=key, + subject=seller, + ) + assert explicit.status == 'ok' and explicit.data['listing']['mode'] == 'sale' + unsupported = await call( + app, + 'store.listing_create', + listing_args(mode='bounty', name='bounty.bin'), + key=key, + subject=seller, + ) + assert unsupported.error.code == 'listing_mode_unsupported' async with app.metadata.transaction(write=False) as tx: - revision=await tx.revision(created.resources[0]) - persisted=await app.contents.read_bytes(revision.content) + revision = await tx.revision(created.resources[0]) + persisted = await app.contents.read_bytes(revision.content) assert b'"mode":"sale"' in persisted - assert tx.one("SELECT COUNT(*) FROM resources WHERE parent='t_store'")[0]==2 + assert tx.one("SELECT COUNT(*) FROM resources WHERE parent='t_store'")[0] == 2 @pytest.mark.asyncio async def test_legacy_listing_body_without_mode_reads_as_sale(): - old={'currency_id':'primary','price_minor':1,'quantity':1,'item_kind':'file', - 'delivery_mode':'managed_instant','package_ref':None,'escrow_policy':'v1', - 'dispute_policy':'v1','terms':'old','expires_at':None,'state':'draft'} - revision=SimpleNamespace(id='v_legacy',content=object()) + old = { + 'currency_id': 'primary', + 'price_minor': 1, + 'quantity': 1, + 'item_kind': 'file', + 'delivery_mode': 'managed_instant', + 'package_ref': None, + 'escrow_policy': 'v1', + 'dispute_policy': 'v1', + 'terms': 'old', + 'expires_at': None, + 'state': 'draft', + } + revision = SimpleNamespace(id='v_legacy', content=object()) + class LegacyTx: async def revision(self, ref): - assert ref.id=='r_legacy' + assert ref.id == 'r_legacy' return revision + class LegacyContent: async def read_bytes(self, blob): assert blob is revision.content return canonical(old) - body,found=await _body(SimpleNamespace(contents=LegacyContent()),LegacyTx(), - SimpleNamespace(id='r_legacy')) - assert found is revision and body=={'mode':'sale',**old} + + body, found = await _body( + SimpleNamespace(contents=LegacyContent()), LegacyTx(), SimpleNamespace(id='r_legacy') + ) + assert found is revision and body == {'mode': 'sale', **old} @pytest.mark.asyncio async def test_deposit_is_immutable_and_only_seller_can_read_payload(installed): - app,_=installed - key,seller,_=await register(app,'deposit-seller') - other_key,other,_=await register(app,'deposit-other') - listing=await call(app,'store.listing_create',listing_args(),key=key,subject=seller) - rid=listing.resources[0].id - version=listing.resources[0].revision - file=await call(app,'content.file_put',{'parent':'/@deposit-seller/files', - 'name':'private.bin','data':b64(b'package secret'), - 'media_type':'application/octet-stream'},key=key,subject=seller) - assert file.status=='ok',wire(file) - payload={'id':file.resources[0].id,'revision':file.resources[0].revision} - deposit=await call(app,'store.package_deposit',{'listing_id':rid, - 'listing_revision':version,'manifest':{'format':1,'name':'private.bin'}, - 'payload_refs':[payload]},key=key,subject=seller,rid='deposit-once') - assert deposit.status=='ok',wire(deposit) - replay=await call(app,'store.package_deposit',{'listing_id':rid, - 'listing_revision':version,'manifest':{'format':1,'name':'private.bin'}, - 'payload_refs':[payload]},key=key,subject=seller,rid='deposit-once') - assert replay.status=='ok' and replay.replayed - package_id=deposit.data['package']['id'] - first=await call(app,'store.package_get',{'id':package_id},key=key,subject=seller) - assert first.status=='ok' and first.data['package']['payload_refs'][0]['id']==payload['id'] - hidden=await call(app,'store.package_get',{'id':package_id},key=other_key,subject=other) - absent=await call(app,'store.package_get',{'id':'pkg_does_not_exist'}, - key=other_key,subject=other) - assert hidden.error.code==absent.error.code=='package_not_found' + app, _ = installed + key, seller, _ = await register(app, 'deposit-seller') + other_key, other, _ = await register(app, 'deposit-other') + listing = await call(app, 'store.listing_create', listing_args(), key=key, subject=seller) + rid = listing.resources[0].id + version = listing.resources[0].revision + file = await call( + app, + 'content.file_put', + { + 'parent': '/@deposit-seller/files', + 'name': 'private.bin', + 'data': b64(b'package secret'), + 'media_type': 'application/octet-stream', + }, + key=key, + subject=seller, + ) + assert file.status == 'ok', wire(file) + payload = {'id': file.resources[0].id, 'revision': file.resources[0].revision} + deposit = await call( + app, + 'store.package_deposit', + { + 'listing_id': rid, + 'listing_revision': version, + 'manifest': {'format': 1, 'name': 'private.bin'}, + 'payload_refs': [payload], + }, + key=key, + subject=seller, + rid='deposit-once', + ) + assert deposit.status == 'ok', wire(deposit) + replay = await call( + app, + 'store.package_deposit', + { + 'listing_id': rid, + 'listing_revision': version, + 'manifest': {'format': 1, 'name': 'private.bin'}, + 'payload_refs': [payload], + }, + key=key, + subject=seller, + rid='deposit-once', + ) + assert replay.status == 'ok' and replay.replayed + package_id = deposit.data['package']['id'] + first = await call(app, 'store.package_get', {'id': package_id}, key=key, subject=seller) + assert first.status == 'ok' and first.data['package']['payload_refs'][0]['id'] == payload['id'] + hidden = await call(app, 'store.package_get', {'id': package_id}, key=other_key, subject=other) + absent = await call( + app, 'store.package_get', {'id': 'pkg_does_not_exist'}, key=other_key, subject=other + ) + assert hidden.error.code == absent.error.code == 'package_not_found' assert 'package secret' not in str(hidden.data) - update=await call(app,'store.listing_update',{'id':rid,'package_ref':package_id, - 'state':'active'},key=key,subject=seller, - expected=((rid,listing.data['generation']),)) - assert update.status=='ok',wire(update) - old=await call(app,'store.listing_get',{'id':rid,'revision':version}) - assert old.status=='ok' and old.data['listing']['package_ref'] is None - assert old.data['listing']['price_minor']==1250000 - newer=await call(app,'store.listing_update',{'id':rid,'price_minor':1500000, - 'terms':'Revised terms.'},key=key,subject=seller, - expected=((rid,update.data['generation']),)) - assert newer.status=='ok',wire(newer) - again=await call(app,'store.package_get',{'id':package_id},key=key,subject=seller) - assert again.data['package']['digest']==first.data['package']['digest'] - assert again.data['package']['listing_revision']==version + update = await call( + app, + 'store.listing_update', + {'id': rid, 'package_ref': package_id, 'state': 'active'}, + key=key, + subject=seller, + expected=((rid, listing.data['generation']),), + ) + assert update.status == 'ok', wire(update) + old = await call(app, 'store.listing_get', {'id': rid, 'revision': version}) + assert old.status == 'ok' and old.data['listing']['package_ref'] is None + assert old.data['listing']['price_minor'] == 1250000 + newer = await call( + app, + 'store.listing_update', + {'id': rid, 'price_minor': 1500000, 'terms': 'Revised terms.'}, + key=key, + subject=seller, + expected=((rid, update.data['generation']),), + ) + assert newer.status == 'ok', wire(newer) + again = await call(app, 'store.package_get', {'id': package_id}, key=key, subject=seller) + assert again.data['package']['digest'] == first.data['package']['digest'] + assert again.data['package']['listing_revision'] == version assert old.data['listing']['terms_digest'] != newer.data['listing']['terms_digest'] @pytest.mark.asyncio async def test_listing_cas_payload_ownership_and_no_secret_leak(installed): - app,_=installed - seller_key,seller,_=await register(app,'catalog-owner') - other_key,other,_=await register(app,'catalog-intruder') - listing=await call(app,'store.listing_create',listing_args(),key=seller_key,subject=seller) - rid=listing.resources[0].id - generation=listing.data['generation'] - stale=await call(app,'store.listing_update',{'id':rid,'price_minor':2}, - key=seller_key,subject=seller,expected=((rid,generation+1),)) - assert stale.error.code=='generation_conflict' - forbidden=await call(app,'store.listing_update',{'id':rid,'price_minor':2}, - key=other_key,subject=other,expected=((rid,generation),)) - missing=await call(app,'store.listing_update',{'id':'r_absent','price_minor':2}, - key=other_key,subject=other) - assert forbidden.error.code==missing.error.code=='listing_not_found' - foreign=await call(app,'content.file_put',{'parent':'/@catalog-intruder/files', - 'name':'private.txt','data':b64(b'not yours')},key=other_key,subject=other) - deposit=await call(app,'store.package_deposit',{'listing_id':rid, - 'listing_revision':listing.resources[0].revision,'manifest':{}, - 'payload_refs':[{'id':foreign.resources[0].id, - 'revision':foreign.resources[0].revision}]}, - key=seller_key,subject=seller) - assert deposit.error.code=='payload_not_owned' + app, _ = installed + seller_key, seller, _ = await register(app, 'catalog-owner') + other_key, other, _ = await register(app, 'catalog-intruder') + listing = await call( + app, 'store.listing_create', listing_args(), key=seller_key, subject=seller + ) + rid = listing.resources[0].id + generation = listing.data['generation'] + stale = await call( + app, + 'store.listing_update', + {'id': rid, 'price_minor': 2}, + key=seller_key, + subject=seller, + expected=((rid, generation + 1),), + ) + assert stale.error.code == 'generation_conflict' + forbidden = await call( + app, + 'store.listing_update', + {'id': rid, 'price_minor': 2}, + key=other_key, + subject=other, + expected=((rid, generation),), + ) + missing = await call( + app, + 'store.listing_update', + {'id': 'r_absent', 'price_minor': 2}, + key=other_key, + subject=other, + ) + assert forbidden.error.code == missing.error.code == 'listing_not_found' + foreign = await call( + app, + 'content.file_put', + {'parent': '/@catalog-intruder/files', 'name': 'private.txt', 'data': b64(b'not yours')}, + key=other_key, + subject=other, + ) + deposit = await call( + app, + 'store.package_deposit', + { + 'listing_id': rid, + 'listing_revision': listing.resources[0].revision, + 'manifest': {}, + 'payload_refs': [ + {'id': foreign.resources[0].id, 'revision': foreign.resources[0].revision} + ], + }, + key=seller_key, + subject=seller, + ) + assert deposit.error.code == 'payload_not_owned' async with app.metadata.transaction(write=False) as tx: - assert tx.one('SELECT COUNT(*) FROM store_packages')[0]==0 + assert tx.one('SELECT COUNT(*) FROM store_packages')[0] == 0 @pytest.mark.asyncio async def test_store_namespace_and_listing_metadata_need_store_operations(installed): - app,_=installed - key,seller,_=await register(app,'catalog-boundary') - for operation,args in ( - ('content.post_create',{'parent':'/store','body':'unrelated post'}), - ('content.file_put',{'parent':'/store','name':'unrelated.bin','data':b64(b'x')}), - ('content.topic_create',{'parent':'/store','name':'unrelated'}), + app, _ = installed + key, seller, _ = await register(app, 'catalog-boundary') + for operation, args in ( + ('content.post_create', {'parent': '/store', 'body': 'unrelated post'}), + ('content.file_put', {'parent': '/store', 'name': 'unrelated.bin', 'data': b64(b'x')}), + ('content.topic_create', {'parent': '/store', 'name': 'unrelated'}), ): - denied=await call(app,operation,args,key=key,subject=seller) - assert denied.error.code=='store_controlled_resource',wire(denied) - ordinary=await call(app,'content.post_create',{'parent':'/main','body':'ordinary'}, - key=key,subject=seller) - assert ordinary.status=='ok',wire(ordinary) - moved=await call(app,'content.move',{'id':ordinary.resources[0].id, - 'parent':'/store'},key=key,subject=seller, - expected=((ordinary.resources[0].id,ordinary.data['generation']),)) - assert moved.error.code=='store_controlled_resource' - listing=await call(app,'store.listing_create',listing_args(),key=key,subject=seller) - assert listing.status=='ok',wire(listing) - rid=listing.resources[0].id - changed=await call(app,'content.chmod',{'id':rid,'mode':'0777'}, - key=key,subject=seller, - expected=((rid,listing.data['generation']),)) - assert changed.error.code=='controlled_resource' - changed_group=await call(app,'content.chgrp',{'id':rid,'group':'/&public'}, - key=key,subject=seller,expected=((rid,listing.data['generation']),)) - assert changed_group.error.code=='controlled_resource' + denied = await call(app, operation, args, key=key, subject=seller) + assert denied.error.code == 'store_controlled_resource', wire(denied) + ordinary = await call( + app, 'content.post_create', {'parent': '/main', 'body': 'ordinary'}, key=key, subject=seller + ) + assert ordinary.status == 'ok', wire(ordinary) + moved = await call( + app, + 'content.move', + {'id': ordinary.resources[0].id, 'parent': '/store'}, + key=key, + subject=seller, + expected=((ordinary.resources[0].id, ordinary.data['generation']),), + ) + assert moved.error.code == 'store_controlled_resource' + listing = await call(app, 'store.listing_create', listing_args(), key=key, subject=seller) + assert listing.status == 'ok', wire(listing) + rid = listing.resources[0].id + changed = await call( + app, + 'content.chmod', + {'id': rid, 'mode': '0777'}, + key=key, + subject=seller, + expected=((rid, listing.data['generation']),), + ) + assert changed.error.code == 'controlled_resource' + changed_group = await call( + app, + 'content.chgrp', + {'id': rid, 'group': '/&public'}, + key=key, + subject=seller, + expected=((rid, listing.data['generation']),), + ) + assert changed_group.error.code == 'controlled_resource' diff --git a/tests/test_subject_read_aliases.py b/tests/test_subject_read_aliases.py index 87573064..529b6ce1 100644 --- a/tests/test_subject_read_aliases.py +++ b/tests/test_subject_read_aliases.py @@ -1,98 +1,137 @@ """Short and long subject paths share live read authorization and content.""" + from datetime import timedelta import httpx import pytest from cryptography.hazmat.primitives import serialization from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey +from test_service import NOW, call, register -from msg.core.codec import b64, canonical +from msg.core.codec import b64, canonical, wire from msg.core.requests import request_for -from msg.transports.http import create_app -from msg.security.crypto import Ed25519Signer from msg.security.capabilities import grant_for -from msg.core.codec import wire -from test_service import NOW, call, register +from msg.security.crypto import Ed25519Signer +from msg.transports.http import create_app @pytest.mark.asyncio async def test_resource_backed_subject_aliases_share_content_cache_and_permissions(installed): app, _ = installed key, user, certificate_id = await register(app, 'alias-owner') - ssh=Ed25519Signer.generate() - public=Ed25519PrivateKey.from_private_bytes(ssh.private_bytes()).public_key().public_bytes( - serialization.Encoding.OpenSSH,serialization.PublicFormat.OpenSSH).decode() - proof=ssh.sign(canonical({'subject_id':user,'public_key':public}),purpose='ssh-key-add') - added=await call(app,'identity.ssh_key_add',{'public_key':public,'proof':wire(proof), - 'ceiling':wire((grant_for(app.registry.capability('git.basic')),))},key=key,subject=user) - assert added.status=='ok',wire(added) - stored=await call(app,'keystore.put',{'name':'sealed','format':'age', - 'ciphertext':b64(b'age-encryption.org/v1\npayload')},key=key,subject=user) - assert stored.status=='ok',wire(stored) - before=None + ssh = Ed25519Signer.generate() + public = ( + Ed25519PrivateKey + .from_private_bytes(ssh.private_bytes()) + .public_key() + .public_bytes(serialization.Encoding.OpenSSH, serialization.PublicFormat.OpenSSH) + .decode() + ) + proof = ssh.sign(canonical({'subject_id': user, 'public_key': public}), purpose='ssh-key-add') + added = await call( + app, + 'identity.ssh_key_add', + { + 'public_key': public, + 'proof': wire(proof), + 'ceiling': wire((grant_for(app.registry.capability('git.basic')),)), + }, + key=key, + subject=user, + ) + assert added.status == 'ok', wire(added) + stored = await call( + app, + 'keystore.put', + {'name': 'sealed', 'format': 'age', 'ciphertext': b64(b'age-encryption.org/v1\npayload')}, + key=key, + subject=user, + ) + assert stored.status == 'ok', wire(stored) + before = None async with app.metadata.transaction(write=False) as tx: - before=(tx.one('SELECT COUNT(*) FROM events')[0], - tx.one('SELECT SUM(generation) FROM resources')[0]) - keystore=await tx.resolve('/@alias-owner/keystore') - signed=request_for('discovery.get',{'id':keystore},app.settings.service_url, - signer=key,subject=user,expires_at=NOW+timedelta(seconds=120)) - headers={'X-Msg-Request':b64(canonical(signed))} - async with httpx.AsyncClient(transport=httpx.ASGITransport(app=create_app(app)), - base_url='http://testserver') as http: - for short,long in (('cert','certificates'),('ssh','ssh-keys')): - left=await http.get('/@alias-owner/'+short+'/json') - right=await http.get('/@alias-owner/'+long+'/json') - assert left.status_code==right.status_code==200 - assert left.content==right.content - assert left.headers['etag']==right.headers['etag'] + before = ( + tx.one('SELECT COUNT(*) FROM events')[0], + tx.one('SELECT SUM(generation) FROM resources')[0], + ) + keystore = await tx.resolve('/@alias-owner/keystore') + signed = request_for( + 'discovery.get', + {'id': keystore}, + app.settings.service_url, + signer=key, + subject=user, + expires_at=NOW + timedelta(seconds=120), + ) + headers = {'X-Msg-Request': b64(canonical(signed))} + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=create_app(app)), base_url='http://testserver' + ) as http: + for short, long in (('cert', 'certificates'), ('ssh', 'ssh-keys')): + left = await http.get('/@alias-owner/' + short + '/json') + right = await http.get('/@alias-owner/' + long + '/json') + assert left.status_code == right.status_code == 200 + assert left.content == right.content + assert left.headers['etag'] == right.headers['etag'] assert not left.is_redirect and not right.is_redirect - cached_short=await http.get('/@alias-owner/'+short+'/json', - headers={'If-None-Match':left.headers['etag']}) - cached_long=await http.get('/@alias-owner/'+long+'/json', - headers={'If-None-Match':right.headers['etag']}) - assert cached_short.status_code==cached_long.status_code==304 - if short=='cert': - assert left.json()['path']=='/@alias-owner/'+short - detail=await http.get('/@alias-owner/cert/'+certificate_id) - legacy=await http.get('/@alias-owner/certificates/'+certificate_id) - assert detail.status_code==legacy.status_code==200 - assert detail.content==legacy.content + cached_short = await http.get( + '/@alias-owner/' + short + '/json', headers={'If-None-Match': left.headers['etag']} + ) + cached_long = await http.get( + '/@alias-owner/' + long + '/json', headers={'If-None-Match': right.headers['etag']} + ) + assert cached_short.status_code == cached_long.status_code == 304 + if short == 'cert': + assert left.json()['path'] == '/@alias-owner/' + short + detail = await http.get('/@alias-owner/cert/' + certificate_id) + legacy = await http.get('/@alias-owner/certificates/' + certificate_id) + assert detail.status_code == legacy.status_code == 200 + assert detail.content == legacy.content else: - assert [item['key_id'] for item in left.json()['keys']]==[added.data['key_id']] - detail=await http.get('/@alias-owner/ssh/'+added.data['key_id']) - legacy=await http.get('/@alias-owner/ssh-keys/'+added.data['key_id']) - assert detail.status_code==legacy.status_code==200 - assert detail.content==legacy.content - for short,long in (('ks','keystore'),): - denied_short=await http.get('/@alias-owner/'+short+'/json') - denied_long=await http.get('/@alias-owner/'+long+'/json') - assert denied_short.status_code==denied_long.status_code==403 - assert denied_short.json()['error']['code']==denied_long.json()['error']['code'] - left=await http.get('/@alias-owner/'+short+'/json',headers=headers) - right=await http.get('/@alias-owner/'+long+'/json',headers=headers) - assert left.status_code==right.status_code==200 - assert left.content==right.content - assert left.headers['etag']==right.headers['etag'] - assert left.json()['path']=='/@alias-owner/ks' - cached=await http.get('/@alias-owner/keystore/json', - headers={**headers,'If-None-Match':left.headers['etag']}) - assert cached.status_code==304 - head=await http.head('/@alias-owner/ks/json',headers=headers) - assert head.status_code==200 and head.headers['etag']==left.headers['etag'] - entry_packet=request_for('discovery.get',{'id':stored.resources[0].id}, - app.settings.service_url,signer=key,subject=user, - expires_at=NOW+timedelta(seconds=120)) - entry_headers={'X-Msg-Request':b64(canonical(entry_packet))} - entry_short=await http.get('/@alias-owner/ks/sealed/json',headers=entry_headers) - entry_long=await http.get('/@alias-owner/keystore/sealed/json',headers=entry_headers) - assert entry_short.status_code==entry_long.status_code==200 - assert entry_short.content==entry_long.content - assert entry_short.json()['path']=='/@alias-owner/ks/sealed' - anonymous=await http.get('/@alias-owner/ks/sealed/meta') - assert anonymous.status_code==403 + assert [item['key_id'] for item in left.json()['keys']] == [added.data['key_id']] + detail = await http.get('/@alias-owner/ssh/' + added.data['key_id']) + legacy = await http.get('/@alias-owner/ssh-keys/' + added.data['key_id']) + assert detail.status_code == legacy.status_code == 200 + assert detail.content == legacy.content + for short, long in (('ks', 'keystore'),): + denied_short = await http.get('/@alias-owner/' + short + '/json') + denied_long = await http.get('/@alias-owner/' + long + '/json') + assert denied_short.status_code == denied_long.status_code == 403 + assert denied_short.json()['error']['code'] == denied_long.json()['error']['code'] + left = await http.get('/@alias-owner/' + short + '/json', headers=headers) + right = await http.get('/@alias-owner/' + long + '/json', headers=headers) + assert left.status_code == right.status_code == 200 + assert left.content == right.content + assert left.headers['etag'] == right.headers['etag'] + assert left.json()['path'] == '/@alias-owner/ks' + cached = await http.get( + '/@alias-owner/keystore/json', + headers={**headers, 'If-None-Match': left.headers['etag']}, + ) + assert cached.status_code == 304 + head = await http.head('/@alias-owner/ks/json', headers=headers) + assert head.status_code == 200 and head.headers['etag'] == left.headers['etag'] + entry_packet = request_for( + 'discovery.get', + {'id': stored.resources[0].id}, + app.settings.service_url, + signer=key, + subject=user, + expires_at=NOW + timedelta(seconds=120), + ) + entry_headers = {'X-Msg-Request': b64(canonical(entry_packet))} + entry_short = await http.get('/@alias-owner/ks/sealed/json', headers=entry_headers) + entry_long = await http.get('/@alias-owner/keystore/sealed/json', headers=entry_headers) + assert entry_short.status_code == entry_long.status_code == 200 + assert entry_short.content == entry_long.content + assert entry_short.json()['path'] == '/@alias-owner/ks/sealed' + anonymous = await http.get('/@alias-owner/ks/sealed/meta') + assert anonymous.status_code == 403 async with app.metadata.transaction(write=False) as tx: - assert before==(tx.one('SELECT COUNT(*) FROM events')[0], - tx.one('SELECT SUM(generation) FROM resources')[0]) + assert before == ( + tx.one('SELECT COUNT(*) FROM events')[0], + tx.one('SELECT SUM(generation) FROM resources')[0], + ) @pytest.mark.asyncio @@ -101,77 +140,117 @@ async def test_operation_backed_subject_aliases_use_same_live_handler(installed) key, user, _ = await register(app, 'mailbox-alias') other_key, other, _ = await register(app, 'mailbox-other') for index in range(2): - post=await call(app,'content.post_create',{'parent':'/main','body':f'notice {index}'}, - key=other_key,subject=other) - sent=await call(app,'communication.send', - {'recipient':user,'resource':wire(post.resources[0])}, - key=other_key,subject=other) - assert sent.status=='ok',wire(sent) + post = await call( + app, + 'content.post_create', + {'parent': '/main', 'body': f'notice {index}'}, + key=other_key, + subject=other, + ) + sent = await call( + app, + 'communication.send', + {'recipient': user, 'resource': wire(post.resources[0])}, + key=other_key, + subject=other, + ) + assert sent.status == 'ok', wire(sent) async with app.metadata.transaction(write=False) as tx: - before=(tx.one('SELECT COUNT(*) FROM events')[0], - tx.one('SELECT COUNT(*) FROM jobs')[0], - tx.one('SELECT COUNT(*) FROM revisions')[0], - tx.one('SELECT SUM(generation) FROM resources')[0]) - async with httpx.AsyncClient(transport=httpx.ASGITransport(app=create_app(app)), - base_url='http://testserver') as http: - for short,long,operation in (('in','inbox','communication.inbox'), - ('out','outbox','communication.outbox'), - ('dm','dm','communication.dm_list')): - packet=request_for(operation,{},app.settings.service_url, - signer=key,subject=user,expires_at=NOW+timedelta(seconds=120)) - headers={'X-Msg-Request':b64(canonical(packet))} - left=await http.get('/@mailbox-alias/'+short,headers=headers) - right=await http.get('/@mailbox-alias/'+long,headers=headers) - assert left.status_code==right.status_code==200,(left.text,right.text) - assert left.content==right.content - assert left.headers['etag']==right.headers['etag'] + before = ( + tx.one('SELECT COUNT(*) FROM events')[0], + tx.one('SELECT COUNT(*) FROM jobs')[0], + tx.one('SELECT COUNT(*) FROM revisions')[0], + tx.one('SELECT SUM(generation) FROM resources')[0], + ) + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=create_app(app)), base_url='http://testserver' + ) as http: + for short, long, operation in ( + ('in', 'inbox', 'communication.inbox'), + ('out', 'outbox', 'communication.outbox'), + ('dm', 'dm', 'communication.dm_list'), + ): + packet = request_for( + operation, + {}, + app.settings.service_url, + signer=key, + subject=user, + expires_at=NOW + timedelta(seconds=120), + ) + headers = {'X-Msg-Request': b64(canonical(packet))} + left = await http.get('/@mailbox-alias/' + short, headers=headers) + right = await http.get('/@mailbox-alias/' + long, headers=headers) + assert left.status_code == right.status_code == 200, (left.text, right.text) + assert left.content == right.content + assert left.headers['etag'] == right.headers['etag'] assert not left.is_redirect and not right.is_redirect - cached=await http.get('/@mailbox-alias/'+long, - headers={**headers,'If-None-Match':left.headers['etag']}) - assert cached.status_code==304 - denied=await http.get('/@mailbox-other/'+short,headers=headers) - assert denied.status_code==403 - inbox=request_for('communication.inbox',{'limit':1},app.settings.service_url, - signer=key,subject=user,expires_at=NOW+timedelta(seconds=120)) - paged_headers={'X-Msg-Request':b64(canonical(inbox))} - first=await http.get('/@mailbox-alias/in?limit=1',headers=paged_headers) - old=await http.get('/@mailbox-alias/inbox?limit=1',headers=paged_headers) - assert first.status_code==old.status_code==200 - assert first.content==old.content and first.json().get('cursor') - for short,long in (('pk','pubkey'),('k','keys'),('ek','encryption-key'), - ('e','encryption-keys'),('ach','achievements')): - public_short=await http.get('/@mailbox-alias/'+short) - public_long=await http.get('/@mailbox-alias/'+long) - assert public_short.status_code==public_long.status_code==200 - assert public_short.content==public_long.content - assert public_short.headers['etag']==public_long.headers['etag'] + cached = await http.get( + '/@mailbox-alias/' + long, + headers={**headers, 'If-None-Match': left.headers['etag']}, + ) + assert cached.status_code == 304 + denied = await http.get('/@mailbox-other/' + short, headers=headers) + assert denied.status_code == 403 + inbox = request_for( + 'communication.inbox', + {'limit': 1}, + app.settings.service_url, + signer=key, + subject=user, + expires_at=NOW + timedelta(seconds=120), + ) + paged_headers = {'X-Msg-Request': b64(canonical(inbox))} + first = await http.get('/@mailbox-alias/in?limit=1', headers=paged_headers) + old = await http.get('/@mailbox-alias/inbox?limit=1', headers=paged_headers) + assert first.status_code == old.status_code == 200 + assert first.content == old.content and first.json().get('cursor') + for short, long in ( + ('pk', 'pubkey'), + ('k', 'keys'), + ('ek', 'encryption-key'), + ('e', 'encryption-keys'), + ('ach', 'achievements'), + ): + public_short = await http.get('/@mailbox-alias/' + short) + public_long = await http.get('/@mailbox-alias/' + long) + assert public_short.status_code == public_long.status_code == 200 + assert public_short.content == public_long.content + assert public_short.headers['etag'] == public_long.headers['etag'] assert not public_short.is_redirect and not public_long.is_redirect - assert public_short.json()['path']=='/@mailbox-alias/'+short - cached=await http.get('/@mailbox-alias/'+long, - headers={'If-None-Match':public_short.headers['etag']}) - assert cached.status_code==304 - if short in {'pk','ek','k','e'}: - key_id=(public_short.json()['key_id'] if short in {'pk','ek'} - else public_short.json()['keys'][0]['key_id']) - by_id=await http.get('/@mailbox-alias/'+short+'/'+key_id) - old_by_id=await http.get('/@mailbox-alias/'+long+'/'+key_id) - assert by_id.status_code==old_by_id.status_code==200 - assert by_id.content==old_by_id.content + assert public_short.json()['path'] == '/@mailbox-alias/' + short + cached = await http.get( + '/@mailbox-alias/' + long, headers={'If-None-Match': public_short.headers['etag']} + ) + assert cached.status_code == 304 + if short in {'pk', 'ek', 'k', 'e'}: + key_id = ( + public_short.json()['key_id'] + if short in {'pk', 'ek'} + else public_short.json()['keys'][0]['key_id'] + ) + by_id = await http.get('/@mailbox-alias/' + short + '/' + key_id) + old_by_id = await http.get('/@mailbox-alias/' + long + '/' + key_id) + assert by_id.status_code == old_by_id.status_code == 200 + assert by_id.content == old_by_id.content async with app.metadata.transaction(write=False) as tx: - assert before==(tx.one('SELECT COUNT(*) FROM events')[0], - tx.one('SELECT COUNT(*) FROM jobs')[0], - tx.one('SELECT COUNT(*) FROM revisions')[0], - tx.one('SELECT SUM(generation) FROM resources')[0]) + assert before == ( + tx.one('SELECT COUNT(*) FROM events')[0], + tx.one('SELECT COUNT(*) FROM jobs')[0], + tx.one('SELECT COUNT(*) FROM revisions')[0], + tx.one('SELECT SUM(generation) FROM resources')[0], + ) @pytest.mark.asyncio async def test_key_aliases_fail_closed_for_missing_key_id(installed): app, _ = installed await register(app, 'missing-key-view') - async with httpx.AsyncClient(transport=httpx.ASGITransport(app=create_app(app)), - base_url='http://testserver') as http: - for short,long in (('pk','pubkey'),('ek','encryption-key')): - left=await http.get('/@missing-key-view/'+short+'/k_missing') - right=await http.get('/@missing-key-view/'+long+'/k_missing') - assert left.status_code==right.status_code - assert left.json()['error']['code']==right.json()['error']['code'] + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=create_app(app)), base_url='http://testserver' + ) as http: + for short, long in (('pk', 'pubkey'), ('ek', 'encryption-key')): + left = await http.get('/@missing-key-view/' + short + '/k_missing') + right = await http.get('/@missing-key-view/' + long + '/k_missing') + assert left.status_code == right.status_code + assert left.json()['error']['code'] == right.json()['error']['code'] diff --git a/tests/test_subject_receipts.py b/tests/test_subject_receipts.py index 7d28c0fe..09cf4b1c 100644 --- a/tests/test_subject_receipts.py +++ b/tests/test_subject_receipts.py @@ -1,193 +1,263 @@ """/@user/receipts projects the caller's persisted OperationResults read-only.""" + from datetime import timedelta import httpx import pytest +from test_service import NOW, call, register from msg.core.codec import b64, canonical, digest, wire from msg.core.requests import request_for from msg.transports.http import create_app -from test_service import NOW, call, register PROJECTION = {'request_id', 'operation', 'status', 'committed_at', 'resources', 'signed'} def _headers(app, operation, args, key, subject): - packet=request_for(operation,args,app.settings.service_url,signer=key, - subject=subject,expires_at=NOW+timedelta(seconds=120)) - return {'X-Msg-Request':b64(canonical(packet))} + packet = request_for( + operation, + args, + app.settings.service_url, + signer=key, + subject=subject, + expires_at=NOW + timedelta(seconds=120), + ) + return {'X-Msg-Request': b64(canonical(packet))} def _state(tx): - counts=tuple(tx.one(f'SELECT COUNT(*) FROM {table}')[0] for table in - ('results','audit','jobs','events','messages','revisions')) - results=digest([list(row) for row in tx.rows( - 'SELECT subject,request_id,digest,body FROM results ORDER BY subject,request_id')]) - generations=tx.one('SELECT SUM(generation) FROM resources')[0] - return counts,results,generations + counts = tuple( + tx.one(f'SELECT COUNT(*) FROM {table}')[0] + for table in ('results', 'audit', 'jobs', 'events', 'messages', 'revisions') + ) + results = digest([ + list(row) + for row in tx.rows( + 'SELECT subject,request_id,digest,body FROM results ORDER BY subject,request_id' + ) + ]) + generations = tx.one('SELECT SUM(generation) FROM resources')[0] + return counts, results, generations @pytest.mark.asyncio async def test_subject_receipts_are_owner_only_read_only_projection(installed): - app,_=installed - alice_key,alice,_=await register(app,'receipt-alice') - bob_key,bob,_=await register(app,'receipt-bob') - written=[] + app, _ = installed + alice_key, alice, _ = await register(app, 'receipt-alice') + bob_key, bob, _ = await register(app, 'receipt-bob') + written = [] for index in range(3): - post=await call(app,'content.post_create',{'parent':'/main','body':f'receipt {index}'}, - key=alice_key,subject=alice) - assert post.status=='ok',wire(post) + post = await call( + app, + 'content.post_create', + {'parent': '/main', 'body': f'receipt {index}'}, + key=alice_key, + subject=alice, + ) + assert post.status == 'ok', wire(post) written.append(post) - bob_post=await call(app,'content.post_create',{'parent':'/main','body':'bob receipt'}, - key=bob_key,subject=bob) - assert bob_post.status=='ok',wire(bob_post) + bob_post = await call( + app, + 'content.post_create', + {'parent': '/main', 'body': 'bob receipt'}, + key=bob_key, + subject=bob, + ) + assert bob_post.status == 'ok', wire(bob_post) async with app.metadata.transaction(write=False) as tx: - expected=[row[0] for row in tx.rows( - 'SELECT request_id FROM results WHERE subject=? ORDER BY request_id',(alice,))] - before=_state(tx) - assert {post.request_id for post in written}<=set(expected) and len(expected)>=4 + expected = [ + row[0] + for row in tx.rows( + 'SELECT request_id FROM results WHERE subject=? ORDER BY request_id', (alice,) + ) + ] + before = _state(tx) + assert {post.request_id for post in written} <= set(expected) and len(expected) >= 4 assert bob_post.request_id not in expected - async with httpx.AsyncClient(transport=httpx.ASGITransport(app=create_app(app)), - base_url=app.settings.service_url) as http: - list_headers=_headers(app,'communication.receipt_list',{},alice_key,alice) - listed=await http.get('/@receipt-alice/receipts',headers=list_headers) - assert listed.status_code==200,listed.text - body=listed.json() - assert body['path']=='/@receipt-alice/receipts' - assert [item['request_id'] for item in body['items']]==expected + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=create_app(app)), base_url=app.settings.service_url + ) as http: + list_headers = _headers(app, 'communication.receipt_list', {}, alice_key, alice) + listed = await http.get('/@receipt-alice/receipts', headers=list_headers) + assert listed.status_code == 200, listed.text + body = listed.json() + assert body['path'] == '/@receipt-alice/receipts' + assert [item['request_id'] for item in body['items']] == expected assert 'cursor' not in body for item in body['items']: - assert set(item)==PROJECTION - assert item['signed'] is True and item['status']=='ok' - slashed=await http.get('/@receipt-alice/receipts/',headers=list_headers) - suffixed=await http.get('/@receipt-alice/receipts/json',headers=list_headers) - assert slashed.content==suffixed.content==listed.content - cached=await http.get('/@receipt-alice/receipts', - headers={**list_headers,'If-None-Match':listed.headers['etag']}) - assert cached.status_code==304 - - target=written[1] - item_headers=_headers(app,'communication.receipt_get', - {'request_id':target.request_id},alice_key,alice) - single=await http.get(f'/@receipt-alice/receipts/{target.request_id}',headers=item_headers) - assert single.status_code==200,single.text - receipt=single.json()['receipt'] - assert single.json()['path']==f'/@receipt-alice/receipts/{target.request_id}' - assert receipt==next(item for item in body['items'] - if item['request_id']==target.request_id) - assert receipt['operation']=='content.post_create' - assert receipt['resources']==[wire(ref) for ref in target.resources] - assert receipt['committed_at']==wire(target.committed_at) - for secret in ('data','output','receipt','cli_url','prefer_cli','actor','subject'): + assert set(item) == PROJECTION + assert item['signed'] is True and item['status'] == 'ok' + slashed = await http.get('/@receipt-alice/receipts/', headers=list_headers) + suffixed = await http.get('/@receipt-alice/receipts/json', headers=list_headers) + assert slashed.content == suffixed.content == listed.content + cached = await http.get( + '/@receipt-alice/receipts', + headers={**list_headers, 'If-None-Match': listed.headers['etag']}, + ) + assert cached.status_code == 304 + + target = written[1] + item_headers = _headers( + app, 'communication.receipt_get', {'request_id': target.request_id}, alice_key, alice + ) + single = await http.get( + f'/@receipt-alice/receipts/{target.request_id}', headers=item_headers + ) + assert single.status_code == 200, single.text + receipt = single.json()['receipt'] + assert single.json()['path'] == f'/@receipt-alice/receipts/{target.request_id}' + assert receipt == next( + item for item in body['items'] if item['request_id'] == target.request_id + ) + assert receipt['operation'] == 'content.post_create' + assert receipt['resources'] == [wire(ref) for ref in target.resources] + assert receipt['committed_at'] == wire(target.committed_at) + for secret in ('data', 'output', 'receipt', 'cli_url', 'prefer_cli', 'actor', 'subject'): assert secret not in receipt - suffixed=await http.get(f'/@receipt-alice/receipts/{target.request_id}/json', - headers=item_headers) - assert suffixed.content==single.content - head=await http.head(f'/@receipt-alice/receipts/{target.request_id}',headers=item_headers) - assert head.status_code==200 and head.content==b'' - assert head.headers['etag']==single.headers['etag'] - - missing_headers=_headers(app,'communication.receipt_get', - {'request_id':'r_missing_receipt'},alice_key,alice) - missing=await http.get('/@receipt-alice/receipts/r_missing_receipt',headers=missing_headers) - assert missing.status_code==404 and missing.json()['error']['code']=='not_found' - - bob_list=_headers(app,'communication.receipt_list',{},bob_key,bob) - cross=await http.get('/@receipt-alice/receipts',headers=bob_list) - assert cross.status_code==403 and cross.json()['error']['code']=='permission_denied' + suffixed = await http.get( + f'/@receipt-alice/receipts/{target.request_id}/json', headers=item_headers + ) + assert suffixed.content == single.content + head = await http.head( + f'/@receipt-alice/receipts/{target.request_id}', headers=item_headers + ) + assert head.status_code == 200 and head.content == b'' + assert head.headers['etag'] == single.headers['etag'] + + missing_headers = _headers( + app, 'communication.receipt_get', {'request_id': 'r_missing_receipt'}, alice_key, alice + ) + missing = await http.get( + '/@receipt-alice/receipts/r_missing_receipt', headers=missing_headers + ) + assert missing.status_code == 404 and missing.json()['error']['code'] == 'not_found' + + bob_list = _headers(app, 'communication.receipt_list', {}, bob_key, bob) + cross = await http.get('/@receipt-alice/receipts', headers=bob_list) + assert cross.status_code == 403 and cross.json()['error']['code'] == 'permission_denied' assert target.request_id not in cross.text - for request_id in (target.request_id,bob_post.request_id,'r_missing_receipt'): - bob_get=_headers(app,'communication.receipt_get',{'request_id':request_id},bob_key,bob) - denied=await http.get(f'/@receipt-alice/receipts/{request_id}',headers=bob_get) - assert denied.status_code==403,denied.text - assert denied.json()['error']['code']=='permission_denied' - own=await http.get('/@receipt-bob/receipts',headers=bob_list) - assert own.status_code==200 + for request_id in (target.request_id, bob_post.request_id, 'r_missing_receipt'): + bob_get = _headers( + app, 'communication.receipt_get', {'request_id': request_id}, bob_key, bob + ) + denied = await http.get(f'/@receipt-alice/receipts/{request_id}', headers=bob_get) + assert denied.status_code == 403, denied.text + assert denied.json()['error']['code'] == 'permission_denied' + own = await http.get('/@receipt-bob/receipts', headers=bob_list) + assert own.status_code == 200 assert bob_post.request_id in [item['request_id'] for item in own.json()['items']] - assert not set(expected)&{item['request_id'] for item in own.json()['items']} + assert not set(expected) & {item['request_id'] for item in own.json()['items']} - for path in ('/@receipt-alice/receipts',f'/@receipt-alice/receipts/{target.request_id}'): - anonymous=await http.get(path) - assert anonymous.status_code==401 - assert anonymous.json()['error']['code']=='authentication_required' + for path in ('/@receipt-alice/receipts', f'/@receipt-alice/receipts/{target.request_id}'): + anonymous = await http.get(path) + assert anonymous.status_code == 401 + assert anonymous.json()['error']['code'] == 'authentication_required' assert 'items' not in anonymous.json() and 'receipt' not in anonymous.json() - unknown=await http.get('/@receipt-alice/receipts?after=x',headers=list_headers) - assert unknown.status_code==400 - assert unknown.json()['error']['code']=='unknown_query_parameter' - detail_query=await http.get(f'/@receipt-alice/receipts/{target.request_id}?limit=1', - headers=item_headers) - assert detail_query.json()['error']['code']=='unknown_query_parameter' - for bad in (0,201): - bad_headers=_headers(app,'communication.receipt_list',{'limit':bad},alice_key,alice) - rejected=await http.get(f'/@receipt-alice/receipts?limit={bad}',headers=bad_headers) - assert rejected.status_code==400,rejected.text - for path in ('/@receipt-alice/receipts',f'/@receipt-alice/receipts/{target.request_id}'): - written_path=await http.post(path,json={}) - assert written_path.status_code in {404,405} - - pages=[] - cursor=None + unknown = await http.get('/@receipt-alice/receipts?after=x', headers=list_headers) + assert unknown.status_code == 400 + assert unknown.json()['error']['code'] == 'unknown_query_parameter' + detail_query = await http.get( + f'/@receipt-alice/receipts/{target.request_id}?limit=1', headers=item_headers + ) + assert detail_query.json()['error']['code'] == 'unknown_query_parameter' + for bad in (0, 201): + bad_headers = _headers( + app, 'communication.receipt_list', {'limit': bad}, alice_key, alice + ) + rejected = await http.get(f'/@receipt-alice/receipts?limit={bad}', headers=bad_headers) + assert rejected.status_code == 400, rejected.text + for path in ('/@receipt-alice/receipts', f'/@receipt-alice/receipts/{target.request_id}'): + written_path = await http.post(path, json={}) + assert written_path.status_code in {404, 405} + + pages = [] + cursor = None while True: - args={'limit':1} if cursor is None else {'limit':1,'cursor':cursor} - page=await http.get('/@receipt-alice/receipts',params=args, - headers=_headers(app,'communication.receipt_list',args, - alice_key,alice)) - assert page.status_code==200,page.text - data=page.json() - assert len(data['items'])==1 + args = {'limit': 1} if cursor is None else {'limit': 1, 'cursor': cursor} + page = await http.get( + '/@receipt-alice/receipts', + params=args, + headers=_headers(app, 'communication.receipt_list', args, alice_key, alice), + ) + assert page.status_code == 200, page.text + data = page.json() + assert len(data['items']) == 1 pages.append(data['items'][0]['request_id']) - cursor=data.get('cursor') + cursor = data.get('cursor') if cursor is None: break assert data['next_requires_auth'] is True assert data['next'].startswith('/-/g/communication.receipt_list/j/') - stolen_args={'limit':1,'cursor':cursor} - stolen=await http.get('/@receipt-bob/receipts',params=stolen_args, - headers=_headers(app,'communication.receipt_list', - stolen_args,bob_key,bob)) - assert stolen.status_code==400 - assert stolen.json()['error']['code']=='cursor_query_mismatch' - assert pages==expected + stolen_args = {'limit': 1, 'cursor': cursor} + stolen = await http.get( + '/@receipt-bob/receipts', + params=stolen_args, + headers=_headers(app, 'communication.receipt_list', stolen_args, bob_key, bob), + ) + assert stolen.status_code == 400 + assert stolen.json()['error']['code'] == 'cursor_query_mismatch' + assert pages == expected async with app.metadata.transaction(write=False) as tx: - assert _state(tx)==before + assert _state(tx) == before @pytest.mark.asyncio async def test_receipt_projection_hides_resources_no_longer_readable(installed): - app,_=installed - alice_key,alice,_=await register(app,'receipt-share-alice') - bob_key,bob,_=await register(app,'receipt-share-bob') - note=await call(app,'identity.note_put',{'name':'receipt-note','body':'Private'}, - key=alice_key,subject=alice) - assert note.status=='ok',wire(note) - rid=note.resources[0].id - grant=await call(app,'sharing.grant',{'resource':rid,'grantee':bob, - 'expires_at':wire(NOW+timedelta(days=1))}, - key=alice_key,subject=alice) - assert grant.status=='ok',wire(grant) - sent=await call(app,'communication.send', - {'recipient':alice,'resource':wire(note.resources[0])}, - key=bob_key,subject=bob) - assert sent.status=='ok',wire(sent) + app, _ = installed + alice_key, alice, _ = await register(app, 'receipt-share-alice') + bob_key, bob, _ = await register(app, 'receipt-share-bob') + note = await call( + app, + 'identity.note_put', + {'name': 'receipt-note', 'body': 'Private'}, + key=alice_key, + subject=alice, + ) + assert note.status == 'ok', wire(note) + rid = note.resources[0].id + grant = await call( + app, + 'sharing.grant', + {'resource': rid, 'grantee': bob, 'expires_at': wire(NOW + timedelta(days=1))}, + key=alice_key, + subject=alice, + ) + assert grant.status == 'ok', wire(grant) + sent = await call( + app, + 'communication.send', + {'recipient': alice, 'resource': wire(note.resources[0])}, + key=bob_key, + subject=bob, + ) + assert sent.status == 'ok', wire(sent) assert rid in {ref.id for ref in sent.resources} - shown=await call(app,'communication.receipt_get',{'request_id':sent.request_id}, - key=bob_key,subject=bob) - assert shown.status=='ok',wire(shown) + shown = await call( + app, 'communication.receipt_get', {'request_id': sent.request_id}, key=bob_key, subject=bob + ) + assert shown.status == 'ok', wire(shown) assert rid in {ref['id'] for ref in shown.data['receipt']['resources']} - revoked=await call(app,'sharing.revoke',{'grant_id':grant.data['grant']['id']}, - key=alice_key,subject=alice) - assert revoked.status=='ok',wire(revoked) + revoked = await call( + app, 'sharing.revoke', {'grant_id': grant.data['grant']['id']}, key=alice_key, subject=alice + ) + assert revoked.status == 'ok', wire(revoked) async with app.metadata.transaction(write=False) as tx: - stored=tx.one('SELECT body FROM results WHERE subject=? AND request_id=?', - (bob,sent.request_id))[0] - hidden=await call(app,'communication.receipt_get',{'request_id':sent.request_id}, - key=bob_key,subject=bob) - assert hidden.status=='ok',wire(hidden) + stored = tx.one( + 'SELECT body FROM results WHERE subject=? AND request_id=?', (bob, sent.request_id) + )[0] + hidden = await call( + app, 'communication.receipt_get', {'request_id': sent.request_id}, key=bob_key, subject=bob + ) + assert hidden.status == 'ok', wire(hidden) assert rid not in {ref['id'] for ref in hidden.data['receipt']['resources']} assert hidden.data['receipt']['signed'] is True async with app.metadata.transaction(write=False) as tx: - assert tx.one('SELECT body FROM results WHERE subject=? AND request_id=?', - (bob,sent.request_id))[0]==stored + assert ( + tx.one( + 'SELECT body FROM results WHERE subject=? AND request_id=?', (bob, sent.request_id) + )[0] + == stored + ) diff --git a/tests/test_sync_checkpoint.py b/tests/test_sync_checkpoint.py index 06230af5..08ae4870 100644 --- a/tests/test_sync_checkpoint.py +++ b/tests/test_sync_checkpoint.py @@ -1,146 +1,240 @@ """Durable SyncCursor advancement requires an explicit signed ACK.""" + from datetime import timedelta import httpx import pytest +from test_service import NOW, call, register from msg.core.codec import b64, canonical, wire from msg.core.requests import request_for from msg.transports.http import create_app -from test_service import NOW, call, register -def sync_headers(app,key,subject,cursor): - packet=request_for('communication.sync',{'cursor':cursor},app.settings.service_url, - signer=key,subject=subject,expires_at=NOW+timedelta(seconds=60)) - return {'x-msg-request':b64(canonical(packet))} +def sync_headers(app, key, subject, cursor): + packet = request_for( + 'communication.sync', + {'cursor': cursor}, + app.settings.service_url, + signer=key, + subject=subject, + expires_at=NOW + timedelta(seconds=60), + ) + return {'x-msg-request': b64(canonical(packet))} -async def read_sync(http,app,key,subject,cursor): - return await http.get('/_r/s/'+cursor,headers=sync_headers(app,key,subject,cursor)) +async def read_sync(http, app, key, subject, cursor): + return await http.get('/_r/s/' + cursor, headers=sync_headers(app, key, subject, cursor)) @pytest.mark.asyncio async def test_checkpoint_expands_seen_beyond_64_and_requires_ack(installed): - app,_=installed - key,subject,_=await register(app,'checkpoint-many') - other_key,other_subject,_=await register(app,'checkpoint-other') + app, _ = installed + key, subject, _ = await register(app, 'checkpoint-many') + other_key, other_subject, _ = await register(app, 'checkpoint-other') for index in range(70): - result=await call(app,'content.post_create',{'parent':'/main','body':f'post {index}'}, - key=key,subject=subject) - assert result.status=='ok',wire(result) - async with httpx.AsyncClient(transport=httpx.ASGITransport(app=create_app(app)), - base_url='http://testserver') as http: - packet=request_for('communication.sync',{},app.settings.service_url, - signer=key,subject=subject,expires_at=NOW+timedelta(seconds=60)) - start=await http.get('/_r/s/start',headers={'x-msg-request':b64(canonical(packet))}) - assert start.status_code==200,start.text - old=start.json()['sync_cursor'] - opened=await call(app,'communication.sync_checkpoint_open',{'cursor':old}, - key=key,subject=subject,rid='checkpoint-open') - assert opened.status=='ok',wire(opened) - replay=await call(app,'communication.sync_checkpoint_open',{'cursor':old}, - key=key,subject=subject,rid='checkpoint-open') - assert replay.status=='ok' and replay.replayed - cursor=opened.data['sync_cursor'] - forbidden=await read_sync(http,app,other_key,other_subject,cursor) - assert forbidden.status_code==403 - assert forbidden.json()['error']['code']=='cursor_principal_mismatch' - page=await read_sync(http,app,key,subject,cursor) - again=await read_sync(http,app,key,subject,cursor) - assert page.status_code==again.status_code==200 - assert page.json()==again.json() + result = await call( + app, + 'content.post_create', + {'parent': '/main', 'body': f'post {index}'}, + key=key, + subject=subject, + ) + assert result.status == 'ok', wire(result) + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=create_app(app)), base_url='http://testserver' + ) as http: + packet = request_for( + 'communication.sync', + {}, + app.settings.service_url, + signer=key, + subject=subject, + expires_at=NOW + timedelta(seconds=60), + ) + start = await http.get('/_r/s/start', headers={'x-msg-request': b64(canonical(packet))}) + assert start.status_code == 200, start.text + old = start.json()['sync_cursor'] + opened = await call( + app, + 'communication.sync_checkpoint_open', + {'cursor': old}, + key=key, + subject=subject, + rid='checkpoint-open', + ) + assert opened.status == 'ok', wire(opened) + replay = await call( + app, + 'communication.sync_checkpoint_open', + {'cursor': old}, + key=key, + subject=subject, + rid='checkpoint-open', + ) + assert replay.status == 'ok' and replay.replayed + cursor = opened.data['sync_cursor'] + forbidden = await read_sync(http, app, other_key, other_subject, cursor) + assert forbidden.status_code == 403 + assert forbidden.json()['error']['code'] == 'cursor_principal_mismatch' + page = await read_sync(http, app, key, subject, cursor) + again = await read_sync(http, app, key, subject, cursor) + assert page.status_code == again.status_code == 200 + assert page.json() == again.json() assert page.json()['items'] async with app.metadata.transaction(write=False) as tx: - version=tx.one('SELECT version FROM sync_checkpoints WHERE subject=?',(subject,))[0] - assert version==0 - ack=page.json()['checkpoint_ack'] - committed=await call(app,'communication.sync_checkpoint_ack',{'ack':ack}, - key=key,subject=subject,rid='checkpoint-ack') - assert committed.status=='ok',wire(committed) - retry=await call(app,'communication.sync_checkpoint_ack',{'ack':ack}, - key=key,subject=subject,rid='checkpoint-ack') - assert retry.status=='ok' and retry.replayed - conflict=await call(app,'communication.sync_checkpoint_ack',{'ack':ack}, - key=key,subject=subject,rid='checkpoint-concurrent') - assert conflict.error.code=='checkpoint_conflict' - stale_get=await read_sync(http,app,key,subject,cursor) - assert stale_get.status_code==400 - assert stale_get.json()['error']['code']=='checkpoint_conflict' - second=await read_sync(http,app,key,subject,committed.data['sync_cursor']) - assert second.status_code==200,second.text - next_ack=await call(app,'communication.sync_checkpoint_ack', - {'ack':second.json()['checkpoint_ack']},key=key,subject=subject) - assert next_ack.status=='ok',wire(next_ack) + version = tx.one('SELECT version FROM sync_checkpoints WHERE subject=?', (subject,))[0] + assert version == 0 + ack = page.json()['checkpoint_ack'] + committed = await call( + app, + 'communication.sync_checkpoint_ack', + {'ack': ack}, + key=key, + subject=subject, + rid='checkpoint-ack', + ) + assert committed.status == 'ok', wire(committed) + retry = await call( + app, + 'communication.sync_checkpoint_ack', + {'ack': ack}, + key=key, + subject=subject, + rid='checkpoint-ack', + ) + assert retry.status == 'ok' and retry.replayed + conflict = await call( + app, + 'communication.sync_checkpoint_ack', + {'ack': ack}, + key=key, + subject=subject, + rid='checkpoint-concurrent', + ) + assert conflict.error.code == 'checkpoint_conflict' + stale_get = await read_sync(http, app, key, subject, cursor) + assert stale_get.status_code == 400 + assert stale_get.json()['error']['code'] == 'checkpoint_conflict' + second = await read_sync(http, app, key, subject, committed.data['sync_cursor']) + assert second.status_code == 200, second.text + next_ack = await call( + app, + 'communication.sync_checkpoint_ack', + {'ack': second.json()['checkpoint_ack']}, + key=key, + subject=subject, + ) + assert next_ack.status == 'ok', wire(next_ack) async with app.metadata.transaction(write=False) as tx: - body=tx.one('SELECT body FROM sync_checkpoints WHERE subject=?',(subject,))[0] + body = tx.one('SELECT body FROM sync_checkpoints WHERE subject=?', (subject,))[0] from msg.core.codec import loads - assert len(loads(body)['seen'])>=70 + + assert len(loads(body)['seen']) >= 70 @pytest.mark.asyncio async def test_checkpoint_permission_gain_and_retention_fail_closed(installed): - app,_=installed - key,subject,_=await register(app,'checkpoint-stale') - created=await call(app,'content.post_create',{'parent':'/main','body':'sample'}, - key=key,subject=subject) - async with httpx.AsyncClient(transport=httpx.ASGITransport(app=create_app(app)), - base_url='http://testserver') as http: - packet=request_for('communication.sync',{},app.settings.service_url, - signer=key,subject=subject,expires_at=NOW+timedelta(seconds=60)) - start=await http.get('/_r/s/start',headers={'x-msg-request':b64(canonical(packet))}) - old=start.json()['sync_cursor'] - opened=await call(app,'communication.sync_checkpoint_open',{'cursor':old}, - key=key,subject=subject) - cursor=opened.data['sync_cursor'] + app, _ = installed + key, subject, _ = await register(app, 'checkpoint-stale') + created = await call( + app, 'content.post_create', {'parent': '/main', 'body': 'sample'}, key=key, subject=subject + ) + assert created.status == 'ok' + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=create_app(app)), base_url='http://testserver' + ) as http: + packet = request_for( + 'communication.sync', + {}, + app.settings.service_url, + signer=key, + subject=subject, + expires_at=NOW + timedelta(seconds=60), + ) + start = await http.get('/_r/s/start', headers={'x-msg-request': b64(canonical(packet))}) + old = start.json()['sync_cursor'] + opened = await call( + app, 'communication.sync_checkpoint_open', {'cursor': old}, key=key, subject=subject + ) + cursor = opened.data['sync_cursor'] async with app.metadata.transaction(write=True) as tx: - tx.set_setting('authorization_epoch',tx.setting('authorization_epoch',0)+1) - stale=await read_sync(http,app,key,subject,cursor) - assert stale.status_code==400 - assert stale.json()['error']['code']=='resync_required' + tx.set_setting('authorization_epoch', tx.setting('authorization_epoch', 0) + 1) + stale = await read_sync(http, app, key, subject, cursor) + assert stale.status_code == 400 + assert stale.json()['error']['code'] == 'resync_required' async with app.metadata.transaction(write=True) as tx: - tx.set_setting('authorization_epoch',tx.setting('authorization_epoch',0)-1) - tx.set_setting('sync_floor',10**9) - expired=await read_sync(http,app,key,subject,cursor) - assert expired.status_code==400 - assert expired.json()['error']['code']=='resync_required' + tx.set_setting('authorization_epoch', tx.setting('authorization_epoch', 0) - 1) + tx.set_setting('sync_floor', 10**9) + expired = await read_sync(http, app, key, subject, cursor) + assert expired.status_code == 400 + assert expired.json()['error']['code'] == 'resync_required' @pytest.mark.asyncio async def test_checkpoint_drains_known_revocations_one_ack_at_a_time(installed): - app,_=installed - key,subject,_=await register(app,'checkpoint-revokes') - created=[] + app, _ = installed + key, subject, _ = await register(app, 'checkpoint-revokes') + created = [] for index in range(3): - post=await call(app,'content.post_create',{'parent':'/main','body':f'known {index}'}, - key=key,subject=subject) + post = await call( + app, + 'content.post_create', + {'parent': '/main', 'body': f'known {index}'}, + key=key, + subject=subject, + ) created.append(post) - packet=request_for('communication.sync',{},app.settings.service_url, - signer=key,subject=subject,expires_at=NOW+timedelta(seconds=60)) - start=await app.executor.execute(packet) - opened=await call(app,'communication.sync_checkpoint_open', - {'cursor':start.data['sync_cursor']},key=key,subject=subject) - cursor=opened.data['sync_cursor'] + packet = request_for( + 'communication.sync', + {}, + app.settings.service_url, + signer=key, + subject=subject, + expires_at=NOW + timedelta(seconds=60), + ) + start = await app.executor.execute(packet) + opened = await call( + app, + 'communication.sync_checkpoint_open', + {'cursor': start.data['sync_cursor']}, + key=key, + subject=subject, + ) + cursor = opened.data['sync_cursor'] for post in created: - rid=post.resources[0].id - locked=await call(app,'content.chmod',{'id':rid,'mode':'0000'},key=key, - subject=subject,expected=((rid,post.data['generation']),)) - assert locked.status=='ok',wire(locked) - seen=[] + rid = post.resources[0].id + locked = await call( + app, + 'content.chmod', + {'id': rid, 'mode': '0000'}, + key=key, + subject=subject, + expected=((rid, post.data['generation']),), + ) + assert locked.status == 'ok', wire(locked) + seen = [] for _ in range(3): - page=await call(app,'communication.sync',{'cursor':cursor,'limit':1}, - key=key,subject=subject) - assert page.status=='ok',wire(page) + page = await call( + app, 'communication.sync', {'cursor': cursor, 'limit': 1}, key=key, subject=subject + ) + assert page.status == 'ok', wire(page) assert page.data['resync_required'] is True - assert len(page.data['items'])==1 - item=page.data['items'][0] - assert set(item)=={'kind','ref'} and item['kind']=='revoked' + assert len(page.data['items']) == 1 + item = page.data['items'][0] + assert set(item) == {'kind', 'ref'} and item['kind'] == 'revoked' seen.append(item['ref']['id']) - ack=await call(app,'communication.sync_checkpoint_ack', - {'ack':page.data['checkpoint_ack']},key=key,subject=subject) - assert ack.status=='ok',wire(ack) - cursor=ack.data['sync_cursor'] - assert set(seen)=={post.resources[0].id for post in created} - final=await call(app,'communication.sync',{'cursor':cursor,'limit':1}, - key=key,subject=subject) - assert final.error.code=='resync_required' + ack = await call( + app, + 'communication.sync_checkpoint_ack', + {'ack': page.data['checkpoint_ack']}, + key=key, + subject=subject, + ) + assert ack.status == 'ok', wire(ack) + cursor = ack.data['sync_cursor'] + assert set(seen) == {post.resources[0].id for post in created} + final = await call( + app, 'communication.sync', {'cursor': cursor, 'limit': 1}, key=key, subject=subject + ) + assert final.error.code == 'resync_required' diff --git a/tests/test_sync_cursor.py b/tests/test_sync_cursor.py index 29653646..9c7cfe4a 100644 --- a/tests/test_sync_cursor.py +++ b/tests/test_sync_cursor.py @@ -1,111 +1,151 @@ """Independent SyncCursor never doubles as a page cursor or read receipt.""" + from datetime import timedelta import httpx import pytest +from test_service import NOW, call, register from msg.core.codec import b64, canonical, wire from msg.core.errors import Failure from msg.core.requests import request_for from msg.plugins.communication import seal_sync_seen from msg.transports.http import create_app -from test_service import NOW, call, register -def signed_path(app,key,subject,path,args): - packet=request_for('communication.sync',args,app.settings.service_url, - signer=key,subject=subject, - expires_at=NOW+timedelta(seconds=60)) - return path+'/p/'+b64(canonical(packet)) +def signed_path(app, key, subject, path, args): + packet = request_for( + 'communication.sync', + args, + app.settings.service_url, + signer=key, + subject=subject, + expires_at=NOW + timedelta(seconds=60), + ) + return path + '/p/' + b64(canonical(packet)) -def signed_header(app,key,subject,args): - packet=request_for('communication.sync',args,app.settings.service_url, - signer=key,subject=subject, - expires_at=NOW+timedelta(seconds=60)) - return {'x-msg-request':b64(canonical(packet))} +def signed_header(app, key, subject, args): + packet = request_for( + 'communication.sync', + args, + app.settings.service_url, + signer=key, + subject=subject, + expires_at=NOW + timedelta(seconds=60), + ) + return {'x-msg-request': b64(canonical(packet))} async def state(app): async with app.metadata.transaction(write=False) as tx: - return (tx.one('SELECT COUNT(*) FROM events')[0], - tx.one('SELECT COUNT(*) FROM jobs')[0], - tx.one('SELECT COUNT(*) FROM reactions')[0], - tx.one('SELECT COUNT(*) FROM messages')[0], - tx.one('SELECT COUNT(*) FROM credentials')[0], - tx.one('SELECT COUNT(*) FROM revisions')[0], - tx.one('SELECT SUM(generation) FROM resources')[0], - tx.one('SELECT COUNT(*) FROM transfers')[0]) + return ( + tx.one('SELECT COUNT(*) FROM events')[0], + tx.one('SELECT COUNT(*) FROM jobs')[0], + tx.one('SELECT COUNT(*) FROM reactions')[0], + tx.one('SELECT COUNT(*) FROM messages')[0], + tx.one('SELECT COUNT(*) FROM credentials')[0], + tx.one('SELECT COUNT(*) FROM revisions')[0], + tx.one('SELECT SUM(generation) FROM resources')[0], + tx.one('SELECT COUNT(*) FROM transfers')[0], + ) @pytest.mark.asyncio async def test_sync_cursor_tracks_visible_create_modify_archive_and_known_revocation(installed): app, _ = installed - key,user,_=await register(app,'sync-owner') - first=await call(app,'content.post_create',{'parent':'/main','body':'first'}, - key=key,subject=user) - rid=first.resources[0].id - async with httpx.AsyncClient(transport=httpx.ASGITransport(app=create_app(app)), - base_url='http://testserver') as http: - opened=await http.get(signed_path(app,key,user,'/_read/s/start',{})) - assert opened.status_code==200,opened.text - opened_short=await http.get(signed_path(app,key,user,'/_r/s/start',{})) - assert opened_short.status_code==200 - assert opened.content==opened_short.content - assert opened.headers['etag']==opened_short.headers['etag'] + key, user, _ = await register(app, 'sync-owner') + first = await call( + app, 'content.post_create', {'parent': '/main', 'body': 'first'}, key=key, subject=user + ) + rid = first.resources[0].id + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=create_app(app)), base_url='http://testserver' + ) as http: + opened = await http.get(signed_path(app, key, user, '/_read/s/start', {})) + assert opened.status_code == 200, opened.text + opened_short = await http.get(signed_path(app, key, user, '/_r/s/start', {})) + assert opened_short.status_code == 200 + assert opened.content == opened_short.content + assert opened.headers['etag'] == opened_short.headers['etag'] assert opened.json()['next'].startswith('/_r/s/') - assert rid in {item['ref']['id'] for item in opened.json()['items'] - if item.get('ref')} - cursor=opened.json()['sync_cursor'] + assert rid in {item['ref']['id'] for item in opened.json()['items'] if item.get('ref')} + cursor = opened.json()['sync_cursor'] assert rid not in str(app.cursors.inspect(cursor)) - before=await state(app) - long=await http.get(signed_path(app,key,user,'/_read/s/'+cursor,{'cursor':cursor})) - short=await http.get(signed_path(app,key,user,'/_r/s/'+cursor,{'cursor':cursor})) - assert long.status_code==short.status_code==200 - assert long.content==short.content - assert long.headers['etag']==short.headers['etag'] - assert long.json()['sync_cursor']==short.json()['sync_cursor'] - long_cursor=app.cursors.inspect(long.json()['sync_cursor']) - short_cursor=app.cursors.inspect(short.json()['sync_cursor']) - assert long_cursor['position']['seq']==short_cursor['position']['seq'] - assert long_cursor['query']==short_cursor['query'] - assert long_cursor['position']['seen_ciphertext']==short_cursor['position']['seen_ciphertext'] - assert await state(app)==before - second=await call(app,'content.post_create',{'parent':'/main','body':'second'}, - key=key,subject=user) - new=await http.get(signed_path(app,key,user,opened.json()['next'],{'cursor':cursor})) - assert new.status_code==200 - assert any(item['kind']=='created' and item['ref']['id']==second.resources[0].id - for item in new.json()['items']) - cursor=new.json()['sync_cursor'] - edited=await call(app,'content.post_edit', - {'id':rid,'expected_revision':first.resources[0].revision,'body':'updated'}, - key=key,subject=user,expected=((rid,first.data['generation']),)) - changed=await http.get(signed_path(app,key,user,'/_r/s/'+cursor,{'cursor':cursor})) - assert any(item['kind']=='modified' and item['ref']['id']==rid - for item in changed.json()['items']) - cursor=changed.json()['sync_cursor'] - archived=await call(app,'content.archive',{'id':rid},key=key,subject=user, - expected=((rid,edited.data['generation']),)) - assert archived.status=='ok' - archive=await http.get(signed_path(app,key,user,'/_r/s/'+cursor,{'cursor':cursor})) - assert archive.status_code==400 - assert archive.json()['error']['code']=='resync_required' - restarted=await http.get(signed_path(app,key,user,'/_r/s/start',{})) - assert restarted.status_code==200 - assert any(item['kind']=='archived' and item['ref']['id']==rid - for item in restarted.json()['items']) - cursor=restarted.json()['sync_cursor'] + before = await state(app) + long = await http.get(signed_path(app, key, user, '/_read/s/' + cursor, {'cursor': cursor})) + short = await http.get(signed_path(app, key, user, '/_r/s/' + cursor, {'cursor': cursor})) + assert long.status_code == short.status_code == 200 + assert long.content == short.content + assert long.headers['etag'] == short.headers['etag'] + assert long.json()['sync_cursor'] == short.json()['sync_cursor'] + long_cursor = app.cursors.inspect(long.json()['sync_cursor']) + short_cursor = app.cursors.inspect(short.json()['sync_cursor']) + assert long_cursor['position']['seq'] == short_cursor['position']['seq'] + assert long_cursor['query'] == short_cursor['query'] + assert ( + long_cursor['position']['seen_ciphertext'] + == short_cursor['position']['seen_ciphertext'] + ) + assert await state(app) == before + second = await call( + app, 'content.post_create', {'parent': '/main', 'body': 'second'}, key=key, subject=user + ) + new = await http.get(signed_path(app, key, user, opened.json()['next'], {'cursor': cursor})) + assert new.status_code == 200 + assert any( + item['kind'] == 'created' and item['ref']['id'] == second.resources[0].id + for item in new.json()['items'] + ) + cursor = new.json()['sync_cursor'] + edited = await call( + app, + 'content.post_edit', + {'id': rid, 'expected_revision': first.resources[0].revision, 'body': 'updated'}, + key=key, + subject=user, + expected=((rid, first.data['generation']),), + ) + changed = await http.get(signed_path(app, key, user, '/_r/s/' + cursor, {'cursor': cursor})) + assert any( + item['kind'] == 'modified' and item['ref']['id'] == rid + for item in changed.json()['items'] + ) + cursor = changed.json()['sync_cursor'] + archived = await call( + app, + 'content.archive', + {'id': rid}, + key=key, + subject=user, + expected=((rid, edited.data['generation']),), + ) + assert archived.status == 'ok' + archive = await http.get(signed_path(app, key, user, '/_r/s/' + cursor, {'cursor': cursor})) + assert archive.status_code == 400 + assert archive.json()['error']['code'] == 'resync_required' + restarted = await http.get(signed_path(app, key, user, '/_r/s/start', {})) + assert restarted.status_code == 200 + assert any( + item['kind'] == 'archived' and item['ref']['id'] == rid + for item in restarted.json()['items'] + ) + cursor = restarted.json()['sync_cursor'] async with app.metadata.transaction(write=False) as tx: - generation=(await tx.resource(second.resources[0].id)).generation - locked=await call(app,'content.chmod', - {'id':second.resources[0].id,'mode':'0000'},key=key,subject=user, - expected=((second.resources[0].id,generation),)) - assert locked.status=='ok' - revoked=await http.get(signed_path(app,key,user,'/_r/s/'+cursor,{'cursor':cursor})) - assert revoked.status_code==200 - lost=[item for item in revoked.json()['items'] if item['kind']=='revoked'] - assert lost==[{'kind':'revoked','ref':{'id':second.resources[0].id}}] + generation = (await tx.resource(second.resources[0].id)).generation + locked = await call( + app, + 'content.chmod', + {'id': second.resources[0].id, 'mode': '0000'}, + key=key, + subject=user, + expected=((second.resources[0].id, generation),), + ) + assert locked.status == 'ok' + revoked = await http.get(signed_path(app, key, user, '/_r/s/' + cursor, {'cursor': cursor})) + assert revoked.status_code == 200 + lost = [item for item in revoked.json()['items'] if item['kind'] == 'revoked'] + assert lost == [{'kind': 'revoked', 'ref': {'id': second.resources[0].id}}] assert revoked.json()['resync_required'] is True assert 'sync_cursor' not in revoked.json() and 'next' not in revoked.json() assert 'second' not in revoked.text @@ -114,94 +154,135 @@ async def test_sync_cursor_tracks_visible_create_modify_archive_and_known_revoca @pytest.mark.asyncio async def test_sync_cursor_rejects_other_kinds_expiry_and_expired_window(installed): app, _ = installed - key,user,_=await register(app,'sync-window') - other_key,other,_=await register(app,'sync-other') - async with httpx.AsyncClient(transport=httpx.ASGITransport(app=create_app(app)), - base_url='http://testserver') as http: - opened=await http.get(signed_path(app,key,user,'/_r/s/start',{})) - assert opened.status_code==200 - token=opened.json()['sync_cursor'] - alien=await http.get(signed_path(app,other_key,other,'/_r/s/'+token,{'cursor':token})) - assert alien.status_code==403 - fake_page=app.cursors.encode('page','x','y') - kind=await http.get(signed_path(app,key,user,'/_r/s/'+fake_page,{'cursor':fake_page})) - assert kind.status_code==400 - assert kind.json()['error']['code']=='cursor_kind_mismatch' - decoded=app.cursors.inspect(token) - expired=app.cursors.encode('sync-v2',decoded['query'], - {**decoded['position'],'expires_at':wire(NOW-timedelta(seconds=1))}) - stale=await http.get(signed_path(app,key,user,'/_r/s/'+expired,{'cursor':expired})) - assert stale.status_code==400 - assert stale.json()['error']['code']=='resync_required' + key, user, _ = await register(app, 'sync-window') + other_key, other, _ = await register(app, 'sync-other') + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=create_app(app)), base_url='http://testserver' + ) as http: + opened = await http.get(signed_path(app, key, user, '/_r/s/start', {})) + assert opened.status_code == 200 + token = opened.json()['sync_cursor'] + alien = await http.get( + signed_path(app, other_key, other, '/_r/s/' + token, {'cursor': token}) + ) + assert alien.status_code == 403 + fake_page = app.cursors.encode('page', 'x', 'y') + kind = await http.get( + signed_path(app, key, user, '/_r/s/' + fake_page, {'cursor': fake_page}) + ) + assert kind.status_code == 400 + assert kind.json()['error']['code'] == 'cursor_kind_mismatch' + decoded = app.cursors.inspect(token) + expired = app.cursors.encode( + 'sync-v2', + decoded['query'], + {**decoded['position'], 'expires_at': wire(NOW - timedelta(seconds=1))}, + ) + stale = await http.get(signed_path(app, key, user, '/_r/s/' + expired, {'cursor': expired})) + assert stale.status_code == 400 + assert stale.json()['error']['code'] == 'resync_required' async with app.metadata.transaction(write=True) as tx: - tx.set_setting('sync_floor',decoded['position']['seq']+1) - window=await http.get(signed_path(app,key,user,'/_r/s/'+token,{'cursor':token})) - assert window.status_code==400 - assert window.json()['error']['code']=='resync_required' + tx.set_setting('sync_floor', decoded['position']['seq'] + 1) + window = await http.get(signed_path(app, key, user, '/_r/s/' + token, {'cursor': token})) + assert window.status_code == 400 + assert window.json()['error']['code'] == 'resync_required' @pytest.mark.asyncio async def test_sync_never_emits_unseen_private_reference(installed): app, _ = installed - observer,observer_id,_=await register(app,'sync-observer') - writer,writer_id,_=await register(app,'sync-writer') - async with httpx.AsyncClient(transport=httpx.ASGITransport(app=create_app(app)), - base_url='http://testserver') as http: - opened=await http.get(signed_path(app,observer,observer_id,'/_r/s/start',{})) - token=opened.json()['sync_cursor'] - post=await call(app,'content.post_create',{'parent':'/main','body':'unseen secret'}, - key=writer,subject=writer_id) - rid=post.resources[0].id - await call(app,'content.chmod',{'id':rid,'mode':'0600'},key=writer, - subject=writer_id,expected=((rid,post.data['generation']),)) - before=await state(app) - result=await http.get(signed_path(app,observer,observer_id,'/_r/s/'+token, - {'cursor':token})) - assert result.status_code==400 - assert result.json()['error']['code']=='resync_required' + observer, observer_id, _ = await register(app, 'sync-observer') + writer, writer_id, _ = await register(app, 'sync-writer') + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=create_app(app)), base_url='http://testserver' + ) as http: + opened = await http.get(signed_path(app, observer, observer_id, '/_r/s/start', {})) + token = opened.json()['sync_cursor'] + post = await call( + app, + 'content.post_create', + {'parent': '/main', 'body': 'unseen secret'}, + key=writer, + subject=writer_id, + ) + rid = post.resources[0].id + await call( + app, + 'content.chmod', + {'id': rid, 'mode': '0600'}, + key=writer, + subject=writer_id, + expected=((rid, post.data['generation']),), + ) + before = await state(app) + result = await http.get( + signed_path(app, observer, observer_id, '/_r/s/' + token, {'cursor': token}) + ) + assert result.status_code == 400 + assert result.json()['error']['code'] == 'resync_required' assert rid not in result.text and 'unseen secret' not in result.text - assert await state(app)==before + assert await state(app) == before @pytest.mark.asyncio async def test_sync_seen_window_has_a_hard_bounded_size(installed): app, _ = installed - with pytest.raises(Failure,match='resync_required'): - seal_sync_seen(app,[f'r_{index}' for index in range(65)]) + with pytest.raises(Failure, match='resync_required'): + seal_sync_seen(app, [f'r_{index}' for index in range(65)]) @pytest.mark.asyncio async def test_sync_permission_gain_requires_replay_of_old_events(installed): app, _ = installed - observer,observer_id,_=await register(app,'sync-gain-observer') - writer,writer_id,_=await register(app,'sync-gain-writer') - watched=await call(app,'communication.watch',{'id':'/main'}, - key=observer,subject=observer_id) - assert watched.status=='ok' - post=await call(app,'content.post_create',{'parent':'/main','body':'old private body'}, - key=writer,subject=writer_id) - rid=post.resources[0].id - private=await call(app,'content.chmod',{'id':rid,'mode':'0600'},key=writer, - subject=writer_id,expected=((rid,post.data['generation']),)) - assert private.status=='ok' - async with httpx.AsyncClient(transport=httpx.ASGITransport(app=create_app(app)), - base_url='http://testserver') as http: - opened=await http.get(signed_path(app,observer,observer_id,'/_r/s/start',{})) - assert opened.status_code==200 + observer, observer_id, _ = await register(app, 'sync-gain-observer') + writer, writer_id, _ = await register(app, 'sync-gain-writer') + watched = await call( + app, 'communication.watch', {'id': '/main'}, key=observer, subject=observer_id + ) + assert watched.status == 'ok' + post = await call( + app, + 'content.post_create', + {'parent': '/main', 'body': 'old private body'}, + key=writer, + subject=writer_id, + ) + rid = post.resources[0].id + private = await call( + app, + 'content.chmod', + {'id': rid, 'mode': '0600'}, + key=writer, + subject=writer_id, + expected=((rid, post.data['generation']),), + ) + assert private.status == 'ok' + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=create_app(app)), base_url='http://testserver' + ) as http: + opened = await http.get(signed_path(app, observer, observer_id, '/_r/s/start', {})) + assert opened.status_code == 200 assert rid not in opened.text - cursor=opened.json()['sync_cursor'] - public=await call(app,'content.chmod',{'id':rid,'mode':'0644'},key=writer, - subject=writer_id,expected=((rid,private.data['generation']),)) - assert public.status=='ok' - before=await state(app) - stale=await http.get(signed_path(app,observer,observer_id,'/_r/s/'+cursor, - {'cursor':cursor})) - assert stale.status_code==400 - assert stale.json()['error']['code']=='resync_required' + cursor = opened.json()['sync_cursor'] + public = await call( + app, + 'content.chmod', + {'id': rid, 'mode': '0644'}, + key=writer, + subject=writer_id, + expected=((rid, private.data['generation']),), + ) + assert public.status == 'ok' + before = await state(app) + stale = await http.get( + signed_path(app, observer, observer_id, '/_r/s/' + cursor, {'cursor': cursor}) + ) + assert stale.status_code == 400 + assert stale.json()['error']['code'] == 'resync_required' assert rid not in stale.text and 'old private body' not in stale.text - assert await state(app)==before - restarted=await http.get(signed_path(app,observer,observer_id,'/_r/s/start',{})) - assert restarted.status_code==200 + assert await state(app) == before + restarted = await http.get(signed_path(app, observer, observer_id, '/_r/s/start', {})) + assert restarted.status_code == 200 # A fresh bounded replay can discover the now-visible old event. assert rid in restarted.text @@ -209,56 +290,71 @@ async def test_sync_permission_gain_requires_replay_of_old_events(installed): @pytest.mark.asyncio async def test_sync_over_64_refs_fails_without_advancing_cursor(installed): app, _ = installed - key,user,_=await register(app,'sync-over-64') + key, user, _ = await register(app, 'sync-over-64') for index in range(65): - post=await call(app,'content.post_create',{'parent':'/main','body':f'item {index}'}, - key=key,subject=user) - assert post.status=='ok' - before=await state(app) - async with httpx.AsyncClient(transport=httpx.ASGITransport(app=create_app(app)), - base_url='http://testserver') as http: - first=await http.get(signed_path(app,key,user,'/_r/s/start',{})) - assert first.status_code==200 - cursor=first.json()['sync_cursor'] - over=await http.get('/_r/s/'+cursor, - headers=signed_header(app,key,user,{'cursor':cursor})) - assert over.status_code==400,over.text - assert over.json()['error']['code']=='resync_required' + post = await call( + app, + 'content.post_create', + {'parent': '/main', 'body': f'item {index}'}, + key=key, + subject=user, + ) + assert post.status == 'ok' + before = await state(app) + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=create_app(app)), base_url='http://testserver' + ) as http: + first = await http.get(signed_path(app, key, user, '/_r/s/start', {})) + assert first.status_code == 200 + cursor = first.json()['sync_cursor'] + over = await http.get( + '/_r/s/' + cursor, headers=signed_header(app, key, user, {'cursor': cursor}) + ) + assert over.status_code == 400, over.text + assert over.json()['error']['code'] == 'resync_required' assert 'sync_cursor' not in over.json() and 'next' not in over.json() # Replaying from the beginning is not a long-term recovery strategy: # the same exact seen-set boundary is reached again. Keep this as a # regression until a /-/ checkpoint operation can persist exact state. - restarted=await http.get(signed_path(app,key,user,'/_r/s/start',{})) - assert restarted.status_code==200 - replay_cursor=restarted.json()['sync_cursor'] - replay=await http.get('/_r/s/'+replay_cursor, - headers=signed_header(app,key,user,{'cursor':replay_cursor})) - assert replay.status_code==400 - assert replay.json()['error']['code']=='resync_required' + restarted = await http.get(signed_path(app, key, user, '/_r/s/start', {})) + assert restarted.status_code == 200 + replay_cursor = restarted.json()['sync_cursor'] + replay = await http.get( + '/_r/s/' + replay_cursor, + headers=signed_header(app, key, user, {'cursor': replay_cursor}), + ) + assert replay.status_code == 400 + assert replay.json()['error']['code'] == 'resync_required' assert 'sync_cursor' not in replay.json() and 'next' not in replay.json() - assert await state(app)==before + assert await state(app) == before @pytest.mark.asyncio async def test_sync_topic_membership_change_invalidates_old_cursor(installed): app, _ = installed - key,user,_=await register(app,'sync-topic-member') - creator,creator_id,_=await register(app,'sync-topic-creator') - topic=await call(app,'content.topic_create',{'parent':'/main','name':'sync-topic'}, - key=creator,subject=creator_id) - assert topic.status=='ok' - rid=topic.resources[0].id - async with httpx.AsyncClient(transport=httpx.ASGITransport(app=create_app(app)), - base_url='http://testserver') as http: - opened=await http.get(signed_path(app,key,user,'/_r/s/start',{})) - assert opened.status_code==200 - cursor=opened.json()['sync_cursor'] - joined=await call(app,'content.topic_join',{'id':rid},key=key,subject=user) - assert joined.status=='ok' - before=await state(app) - stale=await http.get(signed_path(app,key,user,'/_r/s/'+cursor,{'cursor':cursor})) - assert stale.status_code==400 - assert stale.json()['error']['code']=='resync_required' - assert await state(app)==before - restarted=await http.get(signed_path(app,key,user,'/_r/s/start',{})) - assert restarted.status_code==200 + key, user, _ = await register(app, 'sync-topic-member') + creator, creator_id, _ = await register(app, 'sync-topic-creator') + topic = await call( + app, + 'content.topic_create', + {'parent': '/main', 'name': 'sync-topic'}, + key=creator, + subject=creator_id, + ) + assert topic.status == 'ok' + rid = topic.resources[0].id + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=create_app(app)), base_url='http://testserver' + ) as http: + opened = await http.get(signed_path(app, key, user, '/_r/s/start', {})) + assert opened.status_code == 200 + cursor = opened.json()['sync_cursor'] + joined = await call(app, 'content.topic_join', {'id': rid}, key=key, subject=user) + assert joined.status == 'ok' + before = await state(app) + stale = await http.get(signed_path(app, key, user, '/_r/s/' + cursor, {'cursor': cursor})) + assert stale.status_code == 400 + assert stale.json()['error']['code'] == 'resync_required' + assert await state(app) == before + restarted = await http.get(signed_path(app, key, user, '/_r/s/start', {})) + assert restarted.status_code == 200 diff --git a/tests/test_system_rule_migration.py b/tests/test_system_rule_migration.py index 0923f8c3..c4124fa7 100644 --- a/tests/test_system_rule_migration.py +++ b/tests/test_system_rule_migration.py @@ -1,4 +1,5 @@ """Release rule identity survives an explicitly declared source relocation.""" + import shutil import pytest @@ -9,57 +10,64 @@ @pytest.mark.asyncio -@pytest.mark.parametrize("container", [tuple, list]) -async def test_source_move_preserves_public_resource_and_revision(installed,tmp_path,container): - app,_=installed - source=tmp_path/'system' - shutil.copytree(system_source_root(),source) - original=source/'rules'/'identity.md' - moved=source/'rules'/'moved'/'identity.md' +@pytest.mark.parametrize('container', [tuple, list]) +async def test_source_move_preserves_public_resource_and_revision(installed, tmp_path, container): + app, _ = installed + source = tmp_path / 'system' + shutil.copytree(system_source_root(), source) + original = source / 'rules' / 'identity.md' + moved = source / 'rules' / 'moved' / 'identity.md' moved.parent.mkdir() original.rename(moved) - paths={'msg.identity':'rules/moved/identity.md'} - declaration={'msg.identity':container(('rules/identity.md','rules/moved/identity.md'))} + paths = {'msg.identity': 'rules/moved/identity.md'} + declaration = {'msg.identity': container(('rules/identity.md', 'rules/moved/identity.md'))} async with app.metadata.transaction(write=False) as tx: - before=await tx.resource('r_rule_identity') - with pytest.raises(Failure,match='system_source_migration_required'): + before = await tx.resource('r_rule_identity') + with pytest.raises(Failure, match='system_source_migration_required'): async with app.metadata.transaction(write=True) as tx: - await sync_system_sources(tx,app.contents,NOW,source_root=source,source_paths=paths) + await sync_system_sources(tx, app.contents, NOW, source_root=source, source_paths=paths) async with app.metadata.transaction(write=True) as tx: - await sync_system_sources(tx,app.contents,NOW,source_root=source, - source_paths=paths,migrations=declaration) + await sync_system_sources( + tx, app.contents, NOW, source_root=source, source_paths=paths, migrations=declaration + ) async with app.metadata.transaction(write=False) as tx: - after=await tx.resource('r_rule_identity') - row=tx.one('SELECT source_path,rule_id,revision_id FROM system_sources WHERE resource_id=?', - ('r_rule_identity',)) - assert (after.id,after.revision,after.generation)==( - before.id,before.revision,before.generation) - assert row==('docs/system/rules/moved/identity.md','msg.identity',before.revision) + after = await tx.resource('r_rule_identity') + row = tx.one( + 'SELECT source_path,rule_id,revision_id FROM system_sources WHERE resource_id=?', + ('r_rule_identity',), + ) + assert (after.id, after.revision, after.generation) == ( + before.id, + before.revision, + before.generation, + ) + assert row == ('docs/system/rules/moved/identity.md', 'msg.identity', before.revision) # The same release declaration remains valid on every subsequent load. async with app.metadata.transaction(write=True) as tx: - await sync_system_sources(tx,app.contents,NOW,source_root=source, - source_paths=paths,migrations=declaration) + await sync_system_sources( + tx, app.contents, NOW, source_root=source, source_paths=paths, migrations=declaration + ) async with app.metadata.transaction(write=False) as tx: - assert (await tx.resource('r_rule_identity')).revision==before.revision + assert (await tx.resource('r_rule_identity')).revision == before.revision @pytest.mark.asyncio -async def test_unknown_duplicate_and_dangling_rule_sources_are_rejected(installed,tmp_path): - app,_=installed - source=tmp_path/'system' - shutil.copytree(system_source_root(),source) - extra=source/'rules'/'extra.md' +async def test_unknown_duplicate_and_dangling_rule_sources_are_rejected(installed, tmp_path): + app, _ = installed + source = tmp_path / 'system' + shutil.copytree(system_source_root(), source) + extra = source / 'rules' / 'extra.md' extra.write_text('\n') - with pytest.raises(Failure,match='unknown_rule_id'): + with pytest.raises(Failure, match='unknown_rule_id'): async with app.metadata.transaction(write=True) as tx: - await sync_system_sources(tx,app.contents,NOW,source_root=source) - extra.write_text((source/'rules'/'identity.md').read_text()) - with pytest.raises(Failure,match='duplicate_rule_id'): + await sync_system_sources(tx, app.contents, NOW, source_root=source) + extra.write_text((source / 'rules' / 'identity.md').read_text()) + with pytest.raises(Failure, match='duplicate_rule_id'): async with app.metadata.transaction(write=True) as tx: - await sync_system_sources(tx,app.contents,NOW,source_root=source) + await sync_system_sources(tx, app.contents, NOW, source_root=source) extra.unlink() - identity=source/'rules'/'identity.md' - identity.write_text(identity.read_text()+'\n\n') - with pytest.raises(Failure,match='dangling_requires_rules'): + identity = source / 'rules' / 'identity.md' + identity.write_text(identity.read_text() + '\n\n') + with pytest.raises(Failure, match='dangling_requires_rules'): async with app.metadata.transaction(write=True) as tx: - await sync_system_sources(tx,app.contents,NOW,source_root=source) + await sync_system_sources(tx, app.contents, NOW, source_root=source) diff --git a/tests/test_system_rule_retirement.py b/tests/test_system_rule_retirement.py index b067fb44..937452d0 100644 --- a/tests/test_system_rule_retirement.py +++ b/tests/test_system_rule_retirement.py @@ -1,15 +1,16 @@ """Release-only retirement preserves history and refuses dangling dependencies.""" -import shutil + import re +import shutil from dataclasses import replace import pytest +from test_service import NOW, call, register from msg.bootstrap import sync_system_sources, system_source_root from msg.core.codec import wire from msg.core.errors import Failure from msg.core.models import ResourceRef -from test_service import NOW, call, register async def retirement(app, tmp_path): @@ -19,19 +20,31 @@ async def retirement(app, tmp_path): for path in source.rglob('*.md'): text = path.read_text() if '/_rules/recovery' in text: - text = re.sub(r'version: (\d+)', lambda match: 'version: ' + str(int(match[1])+1), - text, count=1) - path.write_text('\n'.join(line for line in text.splitlines() - if '/_rules/recovery' not in line) + '\n') + text = re.sub( + r'version: (\d+)', lambda match: 'version: ' + str(int(match[1]) + 1), text, count=1 + ) + path.write_text( + '\n'.join(line for line in text.splitlines() if '/_rules/recovery' not in line) + + '\n' + ) async with app.metadata.transaction(write=False) as tx: - row = tx.one('SELECT source_path,source_version,source_digest FROM system_sources ' - 'WHERE rule_id=?', ('msg.recovery',)) - return source, {'msg.recovery': {'source_path': row[0].removeprefix('docs/system/'), - 'version': row[1], 'digest': row[2]}} + row = tx.one( + 'SELECT source_path,source_version,source_digest FROM system_sources WHERE rule_id=?', + ('msg.recovery',), + ) + return source, { + 'msg.recovery': { + 'source_path': row[0].removeprefix('docs/system/'), + 'version': row[1], + 'digest': row[2], + } + } @pytest.mark.asyncio -async def test_declared_retirement_preserves_history_is_idempotent_and_read_only(installed, tmp_path, monkeypatch): +async def test_declared_retirement_preserves_history_is_idempotent_and_read_only( + installed, tmp_path, monkeypatch +): app, _ = installed source, declaration = await retirement(app, tmp_path) async with app.metadata.transaction(write=False) as tx: @@ -40,16 +53,25 @@ async def test_declared_retirement_preserves_history_is_idempotent_and_read_only original = await app.contents.read_bytes(revision.content) for _ in range(2): async with app.metadata.transaction(write=True) as tx: - await sync_system_sources(tx, app.contents, NOW, source_root=source, - retirements=declaration, registry=app.registry) + await sync_system_sources( + tx, + app.contents, + NOW, + source_root=source, + retirements=declaration, + registry=app.registry, + ) async with app.metadata.transaction(write=False) as tx: after = await tx.resource(before.id) assert after.state == 'archived' assert (after.revision, after.generation) == (before.revision, before.generation + 1) - assert wire(await tx.revision(ResourceRef(id=before.id, revision=before.revision))) == wire(revision) + assert wire( + await tx.revision(ResourceRef(id=before.id, revision=before.revision)) + ) == wire(revision) assert await app.contents.read_bytes(revision.content) == original import msg.bootstrap as bootstrap from msg.application import Application + monkeypatch.setattr(bootstrap, 'SOURCE_RETIREMENTS', declaration) monkeypatch.setattr(bootstrap, 'system_source_root', lambda: source) reloaded = Application(app.settings, clock=lambda: NOW) @@ -65,7 +87,10 @@ async def test_declared_retirement_preserves_history_is_idempotent_and_read_only key, subject, _ = await register(app, 'retirement-writer') for operation, args in ( ('content.chmod', {'id': before.id, 'mode': '0777'}), - ('content.post_edit', {'id': before.id, 'expected_revision': before.revision, 'body': 'replace'}), + ( + 'content.post_edit', + {'id': before.id, 'expected_revision': before.revision, 'body': 'replace'}, + ), ): result = await call(app, operation, args, key=key, subject=subject) assert result.error.code == 'system_managed_resource', wire(result) @@ -76,12 +101,21 @@ async def test_declared_retirement_preserves_history_is_idempotent_and_read_only async def test_retirement_must_pin_current_release_metadata(installed, tmp_path, mismatch): app, _ = installed source, declaration = await retirement(app, tmp_path) - declaration['msg.recovery'][mismatch] = {'version': 999, 'digest': 'sha256:' + '0' * 64, - 'source_path': 'rules/wrong.md'}[mismatch] + declaration['msg.recovery'][mismatch] = { + 'version': 999, + 'digest': 'sha256:' + '0' * 64, + 'source_path': 'rules/wrong.md', + }[mismatch] with pytest.raises(Failure, match='^system_source_retirement_mismatch$'): async with app.metadata.transaction(write=True) as tx: - await sync_system_sources(tx, app.contents, NOW, source_root=source, - retirements=declaration, registry=app.registry) + await sync_system_sources( + tx, + app.contents, + NOW, + source_root=source, + retirements=declaration, + registry=app.registry, + ) async with app.metadata.transaction(write=False) as tx: assert (await tx.resource('r_rule_recovery')).state == 'active' @@ -94,7 +128,9 @@ async def test_retirement_rejects_active_dependencies(installed, tmp_path, depen registry = app.registry if dependency == 'operation': key = ('discovery.get', 1) - registry._operations[key] = replace(registry.operation(*key), requires_rules=('msg.recovery',)) + registry._operations[key] = replace( + registry.operation(*key), requires_rules=('msg.recovery',) + ) elif dependency == 'index': with (source / 'rules/_index.md').open('a') as handle: handle.write('\n[Recovery](/_rules/recovery)\n') @@ -105,14 +141,22 @@ async def test_retirement_rejects_active_dependencies(installed, tmp_path, depen registry = None with pytest.raises(Failure, match='^system_source_retirement_referenced$'): async with app.metadata.transaction(write=True) as tx: - await sync_system_sources(tx, app.contents, NOW, source_root=source, - retirements=declaration, registry=registry) + await sync_system_sources( + tx, + app.contents, + NOW, + source_root=source, + retirements=declaration, + registry=registry, + ) async with app.metadata.transaction(write=False) as tx: assert (await tx.resource('r_rule_recovery')).state == 'active' @pytest.mark.asyncio -async def test_fresh_install_does_not_invent_retired_history(installed, tmp_path, pg_dsn, monkeypatch): +async def test_fresh_install_does_not_invent_retired_history( + installed, tmp_path, pg_dsn, monkeypatch +): import msg.bootstrap as bootstrap from msg.admin.root import _provision from msg.application import Application @@ -121,22 +165,28 @@ async def test_fresh_install_does_not_invent_retired_history(installed, tmp_path source, declaration = await retirement(installed[0], tmp_path) monkeypatch.setattr(bootstrap, 'SOURCE_RETIREMENTS', declaration) monkeypatch.setattr(bootstrap, 'system_source_root', lambda: source) - app = Application(write_example(tmp_path / 'fresh-config', tmp_path / 'fresh-data', - postgres_dsn=pg_dsn), clock=lambda: NOW) + app = Application( + write_example(tmp_path / 'fresh-config', tmp_path / 'fresh-data', postgres_dsn=pg_dsn), + clock=lambda: NOW, + ) try: await _provision(app, 'test-retirement-passphrase') async with app.metadata.transaction(write=False) as tx: assert tx.one('SELECT id FROM resources WHERE id=?', ('r_rule_recovery',)) is None - assert tx.one('SELECT resource_id FROM system_sources WHERE rule_id=?', ('msg.recovery',)) is None + assert ( + tx.one('SELECT resource_id FROM system_sources WHERE rule_id=?', ('msg.recovery',)) + is None + ) assert tx.one('SELECT id FROM resources WHERE id=?', ('r_rule_security',)) is not None finally: await app.close() def test_registry_freeze_rejects_dependency_on_release_retired_rule(monkeypatch): - import msg.bootstrap as bootstrap from test_registry_operation_rules import assemble + import msg.bootstrap as bootstrap + monkeypatch.setattr(bootstrap, 'SOURCE_RETIREMENTS', {'msg.recovery': {}}) registry = assemble('recovery.custom', requires_rules=('msg.recovery',)) with pytest.raises(Failure, match='^dangling_requires_rules$'): @@ -149,26 +199,38 @@ async def test_retirement_cannot_implicitly_reactivate_archived_source(installed app, _ = installed source, declaration = await retirement(app, tmp_path) async with app.metadata.transaction(write=True) as tx: - await sync_system_sources(tx, app.contents, NOW, source_root=source, - retirements=declaration, registry=app.registry) + await sync_system_sources( + tx, + app.contents, + NOW, + source_root=source, + retirements=declaration, + registry=app.registry, + ) shutil.copyfile(system_source_root() / 'rules/recovery.md', source / 'rules/recovery.md') with pytest.raises(Failure, match='^system_source_retirement_required$'): async with app.metadata.transaction(write=True) as tx: - await sync_system_sources(tx, app.contents, NOW, source_root=source, - registry=app.registry) + await sync_system_sources( + tx, app.contents, NOW, source_root=source, registry=app.registry + ) async with app.metadata.transaction(write=False) as tx: assert (await tx.resource('r_rule_recovery')).state == 'archived' @pytest.mark.asyncio -@pytest.mark.parametrize(('change', 'code'), [ - ('unknown', 'system_source_invalid_retirement'), - ('mandatory', 'system_source_invalid_retirement'), - ('malformed', 'system_source_invalid_retirement'), - ('source_present', 'system_source_retirement_source_present'), - ('pointer_drift', 'system_source_pointer_drift'), -]) -async def test_retirement_refuses_invalid_declarations_or_source_drift(installed, tmp_path, change, code): +@pytest.mark.parametrize( + ('change', 'code'), + [ + ('unknown', 'system_source_invalid_retirement'), + ('mandatory', 'system_source_invalid_retirement'), + ('malformed', 'system_source_invalid_retirement'), + ('source_present', 'system_source_retirement_source_present'), + ('pointer_drift', 'system_source_pointer_drift'), + ], +) +async def test_retirement_refuses_invalid_declarations_or_source_drift( + installed, tmp_path, change, code +): app, _ = installed source, declaration = await retirement(app, tmp_path) if change == 'unknown': @@ -182,9 +244,17 @@ async def test_retirement_refuses_invalid_declarations_or_source_drift(installed elif change == 'pointer_drift': async with app.metadata.transaction(write=True) as tx: resource = await tx.resource('r_rule_recovery') - await tx.replace(replace(resource, revision=None, generation=resource.generation+1), - resource.generation) + await tx.replace( + replace(resource, revision=None, generation=resource.generation + 1), + resource.generation, + ) with pytest.raises(Failure, match='^' + code + '$'): async with app.metadata.transaction(write=True) as tx: - await sync_system_sources(tx, app.contents, NOW, source_root=source, - retirements=declaration, registry=app.registry) + await sync_system_sources( + tx, + app.contents, + NOW, + source_root=source, + retirements=declaration, + registry=app.registry, + ) diff --git a/tests/test_system_rules.py b/tests/test_system_rules.py index 94054ffd..06e35c89 100644 --- a/tests/test_system_rules.py +++ b/tests/test_system_rules.py @@ -1,27 +1,29 @@ """Release-owned rules and editable wiki have separate authority boundaries.""" -import shutil + import re +import shutil from dataclasses import replace +import httpx import pytest import tiktoken -import httpx +from test_service import NOW, call, register from msg.core.codec import digest, wire from msg.core.errors import Failure from msg.transports.http import create_app -from test_service import NOW, call, register def test_source_rule_token_budgets_and_index_links(): from msg.bootstrap import RULE_NAMES, system_source_root - root=system_source_root() - encode=tiktoken.get_encoding('cl100k_base').encode - assert len(encode((root/'AGENTS.md').read_text())) <= 200 - index=(root/'rules'/'_index.md').read_text() + + root = system_source_root() + encode = tiktoken.get_encoding('cl100k_base').encode + assert len(encode((root / 'AGENTS.md').read_text())) <= 200 + index = (root / 'rules' / '_index.md').read_text() assert len(encode(index)) <= 400 for name in RULE_NAMES: - source=(root/'rules'/(name+'.md')).read_text() + source = (root / 'rules' / (name + '.md')).read_text() assert len(encode(source)) <= 200 assert f'](/_rules/{name})' in index @@ -34,48 +36,86 @@ async def test_short_bootstrap_rule_index_shards_and_editable_wiki(installed): assert intro.status == index.status == 'ok', (wire(intro), wire(index)) assert '/_rules' in intro.data['content'] and '/wiki' in intro.data['content'] assert len(intro.data['content'].split()) < 150 - async with httpx.AsyncClient(transport=httpx.ASGITransport(app=create_app(app)), - base_url='http://testserver') as http: - index_path=await http.get('/_rules/_index.md') - index_default=await http.get('/_rules') - index_slash=await http.get('/_rules/') - assert index_path.status_code==index_default.status_code==index_slash.status_code==200 - assert index_path.content==index_default.content==index_slash.content - assert index_path.headers['etag']==index_default.headers['etag']==index_slash.headers['etag'] + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=create_app(app)), base_url='http://testserver' + ) as http: + index_path = await http.get('/_rules/_index.md') + index_default = await http.get('/_rules') + index_slash = await http.get('/_rules/') + assert index_path.status_code == index_default.status_code == index_slash.status_code == 200 + assert index_path.content == index_default.content == index_slash.content + assert ( + index_path.headers['etag'] + == index_default.headers['etag'] + == index_slash.headers['etag'] + ) assert 'location' not in index_default.headers - assert (await http.post('/_rules')).status_code==405 - seen=set() - for name in ('identity','read-write','auth','topics','files','recovery','security','protocol'): + assert (await http.post('/_rules')).status_code == 405 + seen = set() + for name in ( + 'identity', + 'read-write', + 'auth', + 'topics', + 'files', + 'recovery', + 'security', + 'protocol', + ): shard = await call(app, 'discovery.get', {'id': '/_rules/' + name}) assert shard.status == 'ok' and 'rule_id:' in shard.data['content'], name assert '/_rules/' + name in index.data['content'] - rule_id=re.search(r'rule_id: ([a-z][a-z0-9.\-]*)',shard.data['content']).group(1) + rule_id = re.search(r'rule_id: ([a-z][a-z0-9.\-]*)', shard.data['content']).group(1) assert rule_id not in seen seen.add(rule_id) key, subject, _ = await register(app, 'wiki-editor') before = digest(index.data['content']) - article = await call(app, 'content.post_create', {'parent': '/wiki', - 'body': 'Community explanation, not platform authority.'}, key=key, subject=subject) + article = await call( + app, + 'content.post_create', + {'parent': '/wiki', 'body': 'Community explanation, not platform authority.'}, + key=key, + subject=subject, + ) assert article.status == 'ok', wire(article) - edited = await call(app, 'content.post_edit', {'id': article.resources[0].id, - 'expected_revision': article.resources[0].revision, - 'body': 'A corrected community explanation.'}, key=key, subject=subject, - expected=((article.resources[0].id,article.data['generation']),)) + edited = await call( + app, + 'content.post_edit', + { + 'id': article.resources[0].id, + 'expected_revision': article.resources[0].revision, + 'body': 'A corrected community explanation.', + }, + key=key, + subject=subject, + expected=((article.resources[0].id, article.data['generation']),), + ) assert edited.status == 'ok', wire(edited) unchanged = await call(app, 'discovery.get', {'id': '/_rules/_index.md'}) assert digest(unchanged.data['content']) == before - for op, args in (('content.chmod', {'id': '/AGENTS.md', 'mode': '0777'}), - ('content.move', {'id': '/_rules/_index.md', 'parent': '/wiki'}), - ('content.post_edit', {'id': '/AGENTS.md', 'expected_revision': 'unused', 'body': 'replace'}), - ('content.post_create', {'parent': '/_rules', 'body': 'fake rule'})): - expected=(('r_rule_index',index.data['generation']),) if op=='content.move' else () + for op, args in ( + ('content.chmod', {'id': '/AGENTS.md', 'mode': '0777'}), + ('content.move', {'id': '/_rules/_index.md', 'parent': '/wiki'}), + ( + 'content.post_edit', + {'id': '/AGENTS.md', 'expected_revision': 'unused', 'body': 'replace'}, + ), + ('content.post_create', {'parent': '/_rules', 'body': 'fake rule'}), + ): + expected = (('r_rule_index', index.data['generation']),) if op == 'content.move' else () result = await call(app, op, args, key=key, subject=subject, expected=expected) - assert result.status == 'error' and result.error.code == 'system_managed_resource', (op, wire(result)) + assert result.status == 'error' and result.error.code == 'system_managed_resource', ( + op, + wire(result), + ) @pytest.mark.asyncio -async def test_per_file_upgrade_changes_only_edited_revision_and_rejects_deleted_source(installed,tmp_path): - from msg.bootstrap import system_source_root, sync_system_sources +async def test_per_file_upgrade_changes_only_edited_revision_and_rejects_deleted_source( + installed, tmp_path +): + from msg.bootstrap import sync_system_sources, system_source_root + app, _ = installed source = tmp_path / 'system' shutil.copytree(system_source_root(), source) @@ -83,62 +123,74 @@ async def test_per_file_upgrade_changes_only_edited_revision_and_rejects_deleted agents_before = (await tx.resource('r_agents')).revision identity_before = (await tx.resource('r_rule_identity')).revision identity = source / 'rules' / 'identity.md' - identity.write_text(identity.read_text().replace('version: 1', 'version: 2') + '\nNew release guidance.\n') + identity.write_text( + identity.read_text().replace('version: 1', 'version: 2') + '\nNew release guidance.\n' + ) async with app.metadata.transaction(write=True) as tx: - await sync_system_sources(tx,app.contents,NOW,source_root=source) + await sync_system_sources(tx, app.contents, NOW, source_root=source) async with app.metadata.transaction(write=False) as tx: assert (await tx.resource('r_agents')).revision == agents_before current = await tx.resource('r_rule_identity') assert current.revision != identity_before - assert (await tx.revision(__import__('msg.core.models',fromlist=['ResourceRef']).ResourceRef( - id=current.id,revision=current.revision))).parents == (identity_before,) - row=tx.one('SELECT source_kind,source_version,source_digest FROM system_sources WHERE resource_id=?', - ('r_rule_identity',)) + assert ( + await tx.revision( + __import__('msg.core.models', fromlist=['ResourceRef']).ResourceRef( + id=current.id, revision=current.revision + ) + ) + ).parents == (identity_before,) + row = tx.one( + 'SELECT source_kind,source_version,source_digest FROM system_sources WHERE resource_id=?', + ('r_rule_identity',), + ) assert row[0] == 'release' and row[1] == 2 and row[2].startswith('sha256:') identity.write_text(identity.read_text() + '\nChanged without a version bump.\n') with pytest.raises(Failure, match='system_source_version_required'): async with app.metadata.transaction(write=True) as tx: - await sync_system_sources(tx,app.contents,NOW,source_root=source) - identity.write_text(identity.read_text().replace('\nChanged without a version bump.\n','')) + await sync_system_sources(tx, app.contents, NOW, source_root=source) + identity.write_text(identity.read_text().replace('\nChanged without a version bump.\n', '')) (source / 'rules' / 'files.md').unlink() with pytest.raises(Failure, match='system_source_deleted_requires_migration'): async with app.metadata.transaction(write=True) as tx: - await sync_system_sources(tx,app.contents,NOW,source_root=source) + await sync_system_sources(tx, app.contents, NOW, source_root=source) @pytest.mark.asyncio -async def test_new_application_load_syncs_changed_packaged_release(installed,tmp_path,monkeypatch): +async def test_new_application_load_syncs_changed_packaged_release( + installed, tmp_path, monkeypatch +): import msg.bootstrap as bootstrap from msg.application import Application + app, _ = installed - source=tmp_path/'system' - shutil.copytree(bootstrap.system_source_root(),source) - rules=source/'rules'/'topics.md' - rules.write_text(rules.read_text().replace('version: 1','version: 2')+'\nRelease update.\n') - monkeypatch.setattr(bootstrap,'system_source_root',lambda:source) + source = tmp_path / 'system' + shutil.copytree(bootstrap.system_source_root(), source) + rules = source / 'rules' / 'topics.md' + rules.write_text(rules.read_text().replace('version: 1', 'version: 2') + '\nRelease update.\n') + monkeypatch.setattr(bootstrap, 'system_source_root', lambda: source) async with app.metadata.transaction(write=False) as tx: - before=(await tx.resource('r_rule_topics')).revision - next_app=Application(app.settings,clock=lambda:NOW) + before = (await tx.resource('r_rule_topics')).revision + next_app = Application(app.settings, clock=lambda: NOW) try: await next_app.load() async with next_app.metadata.transaction(write=False) as tx: - changed=(await tx.resource('r_rule_topics')).revision - assert changed!=before - count=tx.one('SELECT COUNT(*) FROM revisions')[0] + changed = (await tx.resource('r_rule_topics')).revision + assert changed != before + count = tx.one('SELECT COUNT(*) FROM revisions')[0] finally: await next_app.close() - again=Application(app.settings,clock=lambda:NOW) + again = Application(app.settings, clock=lambda: NOW) try: await again.load() async with again.metadata.transaction(write=False) as tx: - assert (await tx.resource('r_rule_topics')).revision==changed - assert tx.one('SELECT COUNT(*) FROM revisions')[0]==count + assert (await tx.resource('r_rule_topics')).revision == changed + assert tx.one('SELECT COUNT(*) FROM revisions')[0] == count finally: await again.close() - (source/'rules'/'files.md').unlink() - missing=Application(app.settings,clock=lambda:NOW) + (source / 'rules' / 'files.md').unlink() + missing = Application(app.settings, clock=lambda: NOW) try: - with pytest.raises(Failure,match='system_source_deleted_requires_migration'): + with pytest.raises(Failure, match='system_source_deleted_requires_migration'): await missing.load() finally: await missing.close() @@ -147,11 +199,13 @@ async def test_new_application_load_syncs_changed_packaged_release(installed,tmp @pytest.mark.asyncio async def test_rule_sync_rejects_current_pointer_drift(installed): from msg.bootstrap import sync_system_sources + app, _ = installed async with app.metadata.transaction(write=True) as tx: - rule=await tx.resource('r_rule_identity') - await tx.replace(replace(rule,revision=None,generation=rule.generation+1), - rule.generation) - with pytest.raises(Failure,match='system_source_pointer_drift'): + rule = await tx.resource('r_rule_identity') + await tx.replace( + replace(rule, revision=None, generation=rule.generation + 1), rule.generation + ) + with pytest.raises(Failure, match='system_source_pointer_drift'): async with app.metadata.transaction(write=True) as tx: - await sync_system_sources(tx,app.contents,NOW) + await sync_system_sources(tx, app.contents, NOW) diff --git a/tests/test_tags_index.py b/tests/test_tags_index.py index bec529ba..b6e3e9dd 100644 --- a/tests/test_tags_index.py +++ b/tests/test_tags_index.py @@ -1,4 +1,5 @@ """Tags are normalized resource metadata; index/search never bypass ACL.""" + from dataclasses import replace from datetime import timedelta @@ -17,32 +18,55 @@ @pytest.mark.asyncio async def test_taggable_types_normalize_tags_and_preserve_revision(installed): app, _ = installed - assert all(app.registry.resource_type(kind).taggable for kind in ('post','topic','repo')) + assert all(app.registry.resource_type(kind).taggable for kind in ('post', 'topic', 'repo')) key, uid, cert = await register(app, 'tag-normalization') - created = await call(app, 'content.post_create', {'parent': '/main', 'body': 'tagged'}, - key=key, subject=uid, certs=(cert,)) + created = await call( + app, + 'content.post_create', + {'parent': '/main', 'body': 'tagged'}, + key=key, + subject=uid, + certs=(cert,), + ) rid, revision = created.resources[0].id, created.resources[0].revision - tagged = await call(app, 'content.tags_set', - {'id': rid, 'tags': [' Café ', 'café', 'ＡＩ', 'ai']}, - key=key, subject=uid, certs=(cert,), - expected=((rid, created.data['generation']),)) + tagged = await call( + app, + 'content.tags_set', + {'id': rid, 'tags': [' Café ', 'café', 'ＡＩ', 'ai']}, + key=key, + subject=uid, + certs=(cert,), + expected=((rid, created.data['generation']),), + ) assert tagged.status == 'ok', tagged.error - projection = await call(app, 'discovery.get', {'id': rid}, - key=key, subject=uid, certs=(cert,)) + projection = await call(app, 'discovery.get', {'id': rid}, key=key, subject=uid, certs=(cert,)) assert projection.data['tags'] == ('ai', 'café') async with app.metadata.transaction(write=False) as tx: resource = await tx.resource(rid) assert resource.tags == ('ai', 'café') assert resource.revision == revision - assert tx.rows('SELECT tag FROM resource_tags WHERE resource_id=? ORDER BY tag', - (rid,)) == [('ai',), ('café',)] - duplicate = await call(app, 'content.tags_set', {'id': rid, 'tags': ['new']}, - key=key, subject=uid, certs=(cert,), - expected=((rid, created.data['generation']),)) + assert tx.rows( + 'SELECT tag FROM resource_tags WHERE resource_id=? ORDER BY tag', (rid,) + ) == [('ai',), ('café',)] + duplicate = await call( + app, + 'content.tags_set', + {'id': rid, 'tags': ['new']}, + key=key, + subject=uid, + certs=(cert,), + expected=((rid, created.data['generation']),), + ) assert duplicate.error.code == 'generation_conflict' - invalid = await call(app, 'content.tags_set', {'id': rid, 'tags': ['bad/tag']}, - key=key, subject=uid, certs=(cert,), - expected=((rid, tagged.data['generation']),)) + invalid = await call( + app, + 'content.tags_set', + {'id': rid, 'tags': ['bad/tag']}, + key=key, + subject=uid, + certs=(cert,), + expected=((rid, tagged.data['generation']),), + ) assert invalid.error.code == 'invalid_tag' @@ -50,21 +74,40 @@ async def test_taggable_types_normalize_tags_and_preserve_revision(installed): async def test_topic_and_repo_use_the_same_tag_index(installed): app, _ = installed key, uid, cert = await register(app, 'tag-topic-repo') - topic = await call(app, 'content.topic_create', {'parent': '/main', 'name': 'tag-topic'}, - key=key, subject=uid, certs=(cert,)) - repo = await call(app, 'git.create', {'parent': uid, 'name': 'tag-repo.git'}, - key=key, subject=uid, certs=(cert,)) + topic = await call( + app, + 'content.topic_create', + {'parent': '/main', 'name': 'tag-topic'}, + key=key, + subject=uid, + certs=(cert,), + ) + repo = await call( + app, + 'git.create', + {'parent': uid, 'name': 'tag-repo.git'}, + key=key, + subject=uid, + certs=(cert,), + ) assert topic.status == repo.status == 'ok' ids = [] for result in (topic, repo): rid = result.resources[0].id - tagged = await call(app, 'content.tags_set', {'id': rid, 'tags': ['Shared']}, - key=key, subject=uid, certs=(cert,), - expected=((rid, result.data['generation']),)) + tagged = await call( + app, + 'content.tags_set', + {'id': rid, 'tags': ['Shared']}, + key=key, + subject=uid, + certs=(cert,), + expected=((rid, result.data['generation']),), + ) assert tagged.status == 'ok', tagged.error ids.append(rid) - async with httpx.AsyncClient(transport=httpx.ASGITransport(app=create_app(app)), - base_url='http://testserver') as http: + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=create_app(app)), base_url='http://testserver' + ) as http: response = await http.get('/_i/by-tag/shared') assert response.status_code == 200, response.text assert {row['id'] for row in response.json()['items']} == set(ids) @@ -76,19 +119,31 @@ async def test_by_tag_index_aliases_paginate_and_hide_revoked_resources(installe key, uid, cert = await register(app, 'tag-index') posts = [] for number in range(3): - created = await call(app, 'content.post_create', - {'parent': '/main', 'body': f'tag post {number}'}, - key=key, subject=uid, certs=(cert,)) + created = await call( + app, + 'content.post_create', + {'parent': '/main', 'body': f'tag post {number}'}, + key=key, + subject=uid, + certs=(cert,), + ) rid = created.resources[0].id - tagged = await call(app, 'content.tags_set', {'id': rid, 'tags': ['Café']}, - key=key, subject=uid, certs=(cert,), - expected=((rid, created.data['generation']),)) + tagged = await call( + app, + 'content.tags_set', + {'id': rid, 'tags': ['Café']}, + key=key, + subject=uid, + certs=(cert,), + expected=((rid, created.data['generation']),), + ) assert tagged.status == 'ok' posts.append((rid, tagged.data['generation'])) async with app.metadata.transaction(write=False) as tx: events_before = tx.one('SELECT COUNT(*) FROM events')[0] - async with httpx.AsyncClient(transport=httpx.ASGITransport(app=create_app(app)), - base_url='http://testserver') as http: + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=create_app(app)), base_url='http://testserver' + ) as http: long = await http.get('/_index/by-tag/caf%C3%A9?limit=1') short = await http.get('/_i/by-tag/caf%C3%A9?limit=1') assert long.status_code == short.status_code == 200, (long.text, short.text) @@ -121,40 +176,86 @@ async def test_by_tag_index_aliases_paginate_and_hide_revoked_resources(installe async def test_by_tag_visibility_changes_immediately_and_signed_search_can_read_private(installed): app, _ = installed key, uid, cert = await register(app, 'tag-private') - created = await call(app, 'content.post_create', {'parent': '/main', 'body': 'secret tag'}, - key=key, subject=uid, certs=(cert,)) + created = await call( + app, + 'content.post_create', + {'parent': '/main', 'body': 'secret tag'}, + key=key, + subject=uid, + certs=(cert,), + ) rid = created.resources[0].id - tagged = await call(app, 'content.tags_set', {'id': rid, 'tags': ['Private']}, - key=key, subject=uid, certs=(cert,), - expected=((rid, created.data['generation']),)) - async with httpx.AsyncClient(transport=httpx.ASGITransport(app=create_app(app)), - base_url='http://testserver') as http: + tagged = await call( + app, + 'content.tags_set', + {'id': rid, 'tags': ['Private']}, + key=key, + subject=uid, + certs=(cert,), + expected=((rid, created.data['generation']),), + ) + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=create_app(app)), base_url='http://testserver' + ) as http: before = await http.get('/_index/by-tag/private') assert [item['id'] for item in before.json()['items']] == [rid] - await call(app, 'content.chmod', {'id': rid, 'mode': '0600'}, key=key, subject=uid, - certs=(cert,), expected=((rid, tagged.data['generation']),)) - async with httpx.AsyncClient(transport=httpx.ASGITransport(app=create_app(app)), - base_url='http://testserver') as http: + await call( + app, + 'content.chmod', + {'id': rid, 'mode': '0600'}, + key=key, + subject=uid, + certs=(cert,), + expected=((rid, tagged.data['generation']),), + ) + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=create_app(app)), base_url='http://testserver' + ) as http: path = '/_index/by-tag/private' anonymous = await http.get(path) assert anonymous.status_code == 200 assert anonymous.json()['items'] == [] assert 'etag' not in anonymous.headers - packet = request_for('discovery.list', {'tag': 'private', 'limit': 50}, - app.settings.service_url, signer=key, subject=uid, - expires_at=NOW + timedelta(seconds=120)) + packet = request_for( + 'discovery.list', + {'tag': 'private', 'limit': 50}, + app.settings.service_url, + signer=key, + subject=uid, + expires_at=NOW + timedelta(seconds=120), + ) permitted = await http.get(path, headers={'X-Msg-Request': b64(canonical(packet))}) assert permitted.status_code == 200, permitted.text assert [item['id'] for item in permitted.json()['items']] == [rid] - wrong = await http.get(path, headers={'X-Msg-Request': b64(canonical(request_for( - 'discovery.list', {'tag': 'other', 'limit': 50}, app.settings.service_url, - signer=key, subject=uid, expires_at=NOW + timedelta(seconds=120))))}) + wrong = await http.get( + path, + headers={ + 'X-Msg-Request': b64( + canonical( + request_for( + 'discovery.list', + {'tag': 'other', 'limit': 50}, + app.settings.service_url, + signer=key, + subject=uid, + expires_at=NOW + timedelta(seconds=120), + ) + ) + ) + }, + ) assert wrong.status_code == 400 assert wrong.json()['error']['code'] == 'representation_mismatch' - changed = await call(app, 'content.tags_set', {'id': rid, 'tags': []}, - key=key, subject=uid, certs=(cert,), - expected=((rid, tagged.data['generation'] + 1),)) + changed = await call( + app, + 'content.tags_set', + {'id': rid, 'tags': []}, + key=key, + subject=uid, + certs=(cert,), + expected=((rid, tagged.data['generation'] + 1),), + ) assert changed.status == 'ok', changed.error removed = await http.get(path, headers={'X-Msg-Request': b64(canonical(packet))}) assert removed.status_code == 200 and removed.json()['items'] == [] @@ -171,10 +272,14 @@ async def test_tags_index_has_same_transactional_contract_on_both_stores(backend async with store.transaction(write=True) as tx: old = await tx.resource('post') assert old.tags == ('ai',) - assert tx.rows('SELECT resource_id FROM resource_tags WHERE tag=?', ('ai',)) == [('post',)] + assert tx.rows('SELECT resource_id FROM resource_tags WHERE tag=?', ('ai',)) == [ + ('post',) + ] await tx.replace(replace(old, tags=('café',), generation=1), 0) async with store.transaction(write=False) as tx: assert (await tx.resource('post')).tags == ('café',) - assert tx.rows('SELECT tag FROM resource_tags WHERE resource_id=?', ('post',)) == [('café',)] + assert tx.rows('SELECT tag FROM resource_tags WHERE resource_id=?', ('post',)) == [ + ('café',) + ] finally: await store.close() diff --git a/tests/test_task_a_custodial_binding.py b/tests/test_task_a_custodial_binding.py index 264f2523..b7343f3d 100644 --- a/tests/test_task_a_custodial_binding.py +++ b/tests/test_task_a_custodial_binding.py @@ -3,6 +3,7 @@ These cases use real PostgreSQL and real Ed25519/age; a corrupted row must not become trusted merely because there is no ciphertext ACK to verify. """ + from copy import deepcopy import pytest @@ -13,7 +14,7 @@ from msg.core.models import Credential from msg.security.age_keys import encryption_key_id, generate_age_key, public_from_recipient from msg.security.crypto import Ed25519Signer -from msg.security.custodial_migration import snapshot +from msg.security.custodial_migration import snapshot as snapshot from msg.storage.postgres import PostgresMetadataStore @@ -22,25 +23,55 @@ async def bound_history(pg_dsn): store = PostgresMetadataStore(pg_dsn) old, new = Ed25519Signer.generate(), Ed25519Signer.generate() old_recipient, recipient = generate_age_key()[1], generate_age_key()[1] - challenge = {'subject_id': 'u_owner', 'challenge_id': 'cupg_bound', - 'public_key': b64(new.public_key), 'encryption_recipient': recipient} - details = {'age_inventory': [], 'new_identity_key_id': new.key_id, - 'old_identity_key_id': old.key_id, - 'new_encryption_key_id': encryption_key_id(public_from_recipient(recipient)), - 'old_encryption_key_id': encryption_key_id(public_from_recipient(old_recipient)), - 'server_signature': wire(old.sign(canonical(challenge), - purpose='custodial-upgrade-challenge'))} - credential = Credential(id=old.key_id, subject_id='u_owner', kind='signing_key', - verifier=old.public_key, ceiling=(), not_before=NOW, - expires_at=None, revoked_at=None) + challenge = { + 'subject_id': 'u_owner', + 'challenge_id': 'cupg_bound', + 'public_key': b64(new.public_key), + 'encryption_recipient': recipient, + } + details = { + 'age_inventory': [], + 'new_identity_key_id': new.key_id, + 'old_identity_key_id': old.key_id, + 'new_encryption_key_id': encryption_key_id(public_from_recipient(recipient)), + 'old_encryption_key_id': encryption_key_id(public_from_recipient(old_recipient)), + 'server_signature': wire( + old.sign(canonical(challenge), purpose='custodial-upgrade-challenge') + ), + } + credential = Credential( + id=old.key_id, + subject_id='u_owner', + kind='signing_key', + verifier=old.public_key, + ceiling=(), + not_before=NOW, + expires_at=None, + revoked_at=None, + ) async with store.transaction(write=True) as tx: - tx.execute('INSERT INTO credentials VALUES (?,?,?)', - (old.key_id, 'u_owner', canonical(credential).decode()), write=True) - for key, address, primary in ((details['old_encryption_key_id'], old_recipient, 0), - (details['new_encryption_key_id'], recipient, 1)): - tx.execute('INSERT INTO encryption_subkeys VALUES (?,?,?,?,?,?,?)', - (key, 'u_owner', address, b64(public_from_recipient(address)), - wire(NOW), wire(NOW) if not primary else None, primary), write=True) + tx.execute( + 'INSERT INTO credentials VALUES (?,?,?)', + (old.key_id, 'u_owner', canonical(credential).decode()), + write=True, + ) + for key, address, primary in ( + (details['old_encryption_key_id'], old_recipient, 0), + (details['new_encryption_key_id'], recipient, 1), + ): + tx.execute( + 'INSERT INTO encryption_subkeys VALUES (?,?,?,?,?,?,?)', + ( + key, + 'u_owner', + address, + b64(public_from_recipient(address)), + wire(NOW), + wire(NOW) if not primary else None, + primary, + ), + write=True, + ) yield store, challenge, details await store.close() @@ -57,21 +88,33 @@ async def test_valid_empty_inventory_is_read_only_and_not_backup_retirement(boun @pytest.mark.asyncio -@pytest.mark.parametrize('field', ['subject_id', 'challenge_id', 'public_key', 'encryption_recipient']) +@pytest.mark.parametrize( + 'field', ['subject_id', 'challenge_id', 'public_key', 'encryption_recipient'] +) async def test_persisted_challenge_tampering_is_rejected_without_any_acks(bound_history, field): store, challenge, details = bound_history changed = dict(challenge) - changed[field] = {'subject_id': 'u_other', 'challenge_id': 'cupg_other', - 'public_key': b64(Ed25519Signer.generate().public_key), - 'encryption_recipient': generate_age_key()[1]}[field] + changed[field] = { + 'subject_id': 'u_other', + 'challenge_id': 'cupg_other', + 'public_key': b64(Ed25519Signer.generate().public_key), + 'encryption_recipient': generate_age_key()[1], + }[field] async with store.transaction(write=False) as tx: with pytest.raises(Failure, match='^custodial_challenge_invalid$'): await snapshot(tx, 'u_owner', 'pending_rewrap', changed, details) @pytest.mark.asyncio -@pytest.mark.parametrize('field', ['new_identity_key_id', 'new_encryption_key_id', - 'old_identity_key_id', 'old_encryption_key_id']) +@pytest.mark.parametrize( + 'field', + [ + 'new_identity_key_id', + 'new_encryption_key_id', + 'old_identity_key_id', + 'old_encryption_key_id', + ], +) async def test_stored_key_id_cannot_override_the_signed_challenge(bound_history, field): store, challenge, details = bound_history changed = dict(details, **{field: 'foreign-key'}) @@ -84,8 +127,11 @@ async def test_stored_key_id_cannot_override_the_signed_challenge(bound_history, async def test_retired_new_target_blocks_in_progress_migration(bound_history): store, challenge, details = bound_history async with store.transaction(write=True) as tx: - tx.execute('UPDATE encryption_subkeys SET retired_at=?,is_primary=0 WHERE key_id=?', - (wire(NOW), details['new_encryption_key_id']), write=True) + tx.execute( + 'UPDATE encryption_subkeys SET retired_at=?,is_primary=0 WHERE key_id=?', + (wire(NOW), details['new_encryption_key_id']), + write=True, + ) async with store.transaction(write=False) as tx: with pytest.raises(Failure, match='^custodial_encryption_target_changed$'): await snapshot(tx, 'u_owner', 'pending_rewrap', challenge, details) diff --git a/tests/test_task_a_preflight.py b/tests/test_task_a_preflight.py index 8dea6af8..214fef61 100644 --- a/tests/test_task_a_preflight.py +++ b/tests/test_task_a_preflight.py @@ -10,17 +10,23 @@ def tree(path): - return {str(file.relative_to(path)): file.read_bytes() for file in path.rglob('*') if file.is_file()} + return { + str(file.relative_to(path)): file.read_bytes() for file in path.rglob('*') if file.is_file() + } def test_preflight_does_not_initialize_an_empty_database(pg_dsn, tmp_path): - settings = write_example(tmp_path / 'etc', tmp_path / 'data', 'http://testserver', postgres_dsn=pg_dsn) + settings = write_example( + tmp_path / 'etc', tmp_path / 'data', 'http://testserver', postgres_dsn=pg_dsn + ) before = tree(tmp_path) result = preflight(settings.config_dir) assert result['decision'] == 'blocked' and result['mutation_performed'] is False assert result['database']['state'] == 'empty' with psycopg.connect(pg_dsn) as conn: - assert conn.execute("SELECT count(*) FROM pg_tables WHERE schemaname='public'").fetchone() == (0,) + assert conn.execute( + "SELECT count(*) FROM pg_tables WHERE schemaname='public'" + ).fetchone() == (0,) assert tree(tmp_path) == before assert pg_dsn not in str(result) assert all(item['status'] == 'missing' for item in result['field_evidence']) @@ -28,7 +34,9 @@ def test_preflight_does_not_initialize_an_empty_database(pg_dsn, tmp_path): def test_existing_schema_is_checked_without_touching_sequences_or_files(pg_dsn, tmp_path): PostgresMetadataStore(pg_dsn) - settings = write_example(tmp_path / 'etc', tmp_path / 'data', 'http://testserver', postgres_dsn=pg_dsn) + settings = write_example( + tmp_path / 'etc', tmp_path / 'data', 'http://testserver', postgres_dsn=pg_dsn + ) before = tree(tmp_path) with psycopg.connect(pg_dsn) as conn: seq = conn.execute('SELECT last_value,is_called FROM money_ledger_seq_seq').fetchone() @@ -38,12 +46,17 @@ def test_existing_schema_is_checked_without_touching_sequences_or_files(pg_dsn, assert result['decision'] == 'blocked' # A consistent test DB is not a field attestation. assert result['database']['ca_inventory'] == [] with psycopg.connect(pg_dsn) as conn: - assert conn.execute('SELECT last_value,is_called FROM money_ledger_seq_seq').fetchone() == seq + assert ( + conn.execute('SELECT last_value,is_called FROM money_ledger_seq_seq').fetchone() == seq + ) assert conn.execute('SELECT COUNT(*) FROM audit').fetchone() == (0,) assert conn.execute('SELECT COUNT(*) FROM jobs').fetchone() == (0,) assert tree(tmp_path) == before def test_inspection_refuses_a_writable_transaction(pg_dsn): - with psycopg.connect(pg_dsn) as conn, pytest.raises(Failure, match='^preflight_read_only_required$'): + with ( + psycopg.connect(pg_dsn) as conn, + pytest.raises(Failure, match='^preflight_read_only_required$'), + ): inspect_database(conn) diff --git a/tests/test_task_a_recovery_checkpoint.py b/tests/test_task_a_recovery_checkpoint.py index 77e3fa32..1202f46b 100644 --- a/tests/test_task_a_recovery_checkpoint.py +++ b/tests/test_task_a_recovery_checkpoint.py @@ -1,4 +1,5 @@ """Independent checkpoint pins, not database flags, authorize partial authority replay.""" + import asyncio from copy import deepcopy from dataclasses import replace @@ -16,36 +17,69 @@ async def replay(store, packet, *, pin): - return await _replay(store, packet, pin=pin, operator="isolated-test-usecase") + return await _replay(store, packet, pin=pin, operator='isolated-test-usecase') @pytest.fixture def checkpoint(): signer = Ed25519Signer.generate() - body = {'format': 'msg-revocation-checkpoint-v1', 'service': 'http://testserver', - 'source_backup_sha256': 'a' * 64, 'sequence': 1, - 'entries': [{'sequence': 1, 'kind': 'credential.revoke', 'subject': 'u_owner', - 'target': 'token_one', 'at': wire(NOW)}]} - pin = TrustedCheckpointPin(service=body['service'], public_key=signer.public_key, - digest=digest(body), sequence=1) + body = { + 'format': 'msg-revocation-checkpoint-v1', + 'service': 'http://testserver', + 'source_backup_sha256': 'a' * 64, + 'sequence': 1, + 'entries': [ + { + 'sequence': 1, + 'kind': 'credential.revoke', + 'subject': 'u_owner', + 'target': 'token_one', + 'at': wire(NOW), + } + ], + } + pin = TrustedCheckpointPin( + service=body['service'], public_key=signer.public_key, digest=digest(body), sequence=1 + ) + def signed(value): - return {'checkpoint': value, - 'signature': wire(signer.sign(canonical(value), purpose='recovery-checkpoint-v1'))} + return { + 'checkpoint': value, + 'signature': wire(signer.sign(canonical(value), purpose='recovery-checkpoint-v1')), + } + return body, pin, signed @pytest.fixture async def restored(pg_dsn): store = PostgresMetadataStore(pg_dsn) - token = Credential(id='token_one', subject_id='u_owner', kind='token', verifier=b'x' * 32, - ceiling=(), not_before=NOW, expires_at=None, revoked_at=None) + token = Credential( + id='token_one', + subject_id='u_owner', + kind='token', + verifier=b'x' * 32, + ceiling=(), + not_before=NOW, + expires_at=None, + revoked_at=None, + ) async with store.transaction(write=True) as tx: - tx.set_setting(SETTING, {'format': 'msg-recovery-quarantine-v1', - 'source_backup_sha256': 'a' * 64, - 'outbound_enabled': False, 'authority': 'health_only', - 'revocation_replay': 'required'}) - tx.execute('INSERT INTO credentials VALUES (?,?,?)', - (token.id, token.subject_id, canonical(token).decode()), write=True) + tx.set_setting( + SETTING, + { + 'format': 'msg-recovery-quarantine-v1', + 'source_backup_sha256': 'a' * 64, + 'outbound_enabled': False, + 'authority': 'health_only', + 'revocation_replay': 'required', + }, + ) + tx.execute( + 'INSERT INTO credentials VALUES (?,?,?)', + (token.id, token.subject_id, canonical(token).decode()), + write=True, + ) yield store await store.close() @@ -89,10 +123,17 @@ async def test_replay_is_atomic_monotonic_idempotent_and_never_promotes(restored body, pin, signed = checkpoint results = await asyncio.gather(*(replay(restored, signed(body), pin=pin) for _ in range(3))) assert sum(result['changed'] for result in results) == 1 - assert all(result['promotion'] == 'blocked' and result['backup_retired'] is False and - result['promotion_blocked_reasons'] == [ - 'quarantine_remains', 'supported_fact_inventory_incomplete', - 'backup_retirement_not_attested'] for result in results) + assert all( + result['promotion'] == 'blocked' + and result['backup_retired'] is False + and result['promotion_blocked_reasons'] + == [ + 'quarantine_remains', + 'supported_fact_inventory_incomplete', + 'backup_retirement_not_attested', + ] + for result in results + ) async with restored.transaction(write=False) as tx: assert active(tx) credential = await tx.credential('token_one') @@ -113,8 +154,13 @@ async def test_replay_is_atomic_monotonic_idempotent_and_never_promotes(restored async def test_missing_or_wrong_subject_fact_rolls_back_earlier_revocation(restored, checkpoint): body, pin, signed = checkpoint body = deepcopy(body) - body['entries'].append({'sequence': 2, 'kind': 'credential.revoke', - 'subject': 'u_owner', 'target': 'missing', 'at': wire(NOW)}) + body['entries'].append({ + 'sequence': 2, + 'kind': 'credential.revoke', + 'subject': 'u_owner', + 'target': 'missing', + 'at': wire(NOW), + }) body['sequence'] = 2 pin = replace(pin, digest=digest(body), sequence=2) with pytest.raises(Failure, match='^recovery_fact_missing$'): @@ -145,7 +191,9 @@ async def test_wrong_backup_and_live_database_are_not_replay_targets(restored, c @pytest.mark.asyncio -async def test_cached_receipt_cannot_authorize_regression_or_skip_restored_authority(restored, checkpoint): +async def test_cached_receipt_cannot_authorize_regression_or_skip_restored_authority( + restored, checkpoint +): body, pin, signed = checkpoint await replay(restored, signed(body), pin=pin) # A coherent current DB receipt is only a guard, never an external freshness source. @@ -158,8 +206,11 @@ async def test_cached_receipt_cannot_authorize_regression_or_skip_restored_autho await replay(restored, signed(corrupt), pin=replace(pin, digest=digest(corrupt))) async with restored.transaction(write=True) as tx: token = await tx.credential('token_one') - tx.execute('UPDATE credentials SET body=? WHERE id=?', - (canonical(replace(token, revoked_at=None)).decode(), token.id), write=True) + tx.execute( + 'UPDATE credentials SET body=? WHERE id=?', + (canonical(replace(token, revoked_at=None)).decode(), token.id), + write=True, + ) # Re-applying the pinned full log re-revokes the row, even with an identical cached receipt. assert (await replay(restored, signed(body), pin=pin))['changed'] == 1 async with restored.transaction(write=False) as tx: @@ -170,6 +221,7 @@ async def test_cached_receipt_cannot_authorize_regression_or_skip_restored_autho @pytest.mark.asyncio async def test_actual_backend_termination_rolls_back_and_replay_can_resume(restored, checkpoint): import psycopg + body, pin, signed = checkpoint # Kill the actual PostgreSQL session after its revoke but before its COMMIT. # No patched driver or mock transaction supplies the rollback evidence. @@ -178,7 +230,9 @@ async def test_actual_backend_termination_rolls_back_and_replay_can_resume(resto assert (await replay(restored, signed(body), pin=pin))['changed'] == 1 pid = tx.one('SELECT pg_backend_pid()')[0] with psycopg.connect(restored.dsn) as killer: - assert killer.execute('SELECT pg_terminate_backend(%s)', (pid,)).fetchone() == (True,) + assert killer.execute('SELECT pg_terminate_backend(%s)', (pid,)).fetchone() == ( + True, + ) tx.one('SELECT 1') async with restored.transaction(write=False) as tx: assert (await tx.credential('token_one')).revoked_at is None @@ -188,14 +242,17 @@ async def test_actual_backend_termination_rolls_back_and_replay_can_resume(resto @pytest.mark.asyncio -async def test_public_replay_entry_rejects_nonconsole_before_database_access(restored, checkpoint, tmp_path): +async def test_public_replay_entry_rejects_nonconsole_before_database_access( + restored, checkpoint, tmp_path +): import os import subprocess import sys from pathlib import Path from msg.core.codec import loads - script = ''' + + script = """ import asyncio from pathlib import Path from msg.admin.recovery_replay import replay @@ -205,15 +262,24 @@ async def test_public_replay_entry_rejects_nonconsole_before_database_access(res asyncio.run(replay(None, {}, pin=None, config_dir=Path('.'))) except Failure as exc: print(canonical({'code': exc.code}).decode()) -''' +""" # Capture pipes are not a physical console. On the non-root CI runner the # real OS-admin check must reject even earlier; do not mock uid or accept an # arbitrary error to disguise the order of these independent safety gates. - environment = {key: value for key, value in os.environ.items() - if key not in {'SSH_CONNECTION', 'SSH_CLIENT', 'SSH_TTY'}} + environment = { + key: value + for key, value in os.environ.items() + if key not in {'SSH_CONNECTION', 'SSH_CLIENT', 'SSH_TTY'} + } environment['PYTHONPATH'] = str(Path(__file__).resolve().parents[1] / 'src') - result = await asyncio.to_thread(subprocess.run, [sys.executable, '-c', script], - capture_output=True, text=True, check=True, env=environment) + result = await asyncio.to_thread( + subprocess.run, + [sys.executable, '-c', script], + capture_output=True, + text=True, + check=True, + env=environment, + ) expected = 'local_console_required' if os.geteuid() == 0 else 'local_os_administrator_required' assert loads(result.stdout) == {'code': expected} async with restored.transaction(write=False) as tx: @@ -223,14 +289,22 @@ async def test_public_replay_entry_rejects_nonconsole_before_database_access(res def test_offline_selftest_is_not_field_or_database_acceptance(): from msg.admin.recovery_replay import selftest - assert selftest() == {'format': 'msg-revocation-checkpoint-v1', - 'scope': 'isolated_signature_contract_only', 'checks': 3, - 'status': 'passed', 'database_used': False, - 'production_evidence': False, 'promotion': 'blocked'} + + assert selftest() == { + 'format': 'msg-revocation-checkpoint-v1', + 'scope': 'isolated_signature_contract_only', + 'checks': 3, + 'status': 'passed', + 'database_used': False, + 'production_evidence': False, + 'promotion': 'blocked', + } @pytest.mark.asyncio -async def test_all_supported_owned_revocations_preserve_signed_bytes_and_never_import_keys(restored, checkpoint): +async def test_all_supported_owned_revocations_preserve_signed_bytes_and_never_import_keys( + restored, checkpoint +): from datetime import timedelta from msg.admin.recovery_replay import SUPPORTED_FACTS @@ -239,93 +313,232 @@ async def test_all_supported_owned_revocations_preserve_signed_bytes_and_never_i from msg.security.age_keys import encryption_key_id, generate_age_key, public_from_recipient from msg.security.certificates import sign_certificate from msg.security.crypto import key_id + body, pin, signed = checkpoint owner = 'u_owner' signer = Ed25519Signer.generate() recipient = generate_age_key()[1] age_id = encryption_key_id(public_from_recipient(recipient)) - credential = Credential(id=signer.key_id, subject_id=owner, kind='signing_key', - verifier=signer.public_key, ceiling=(), not_before=NOW, expires_at=None, revoked_at=None) - certificate = sign_certificate(Certificate(resource_id='cert_one', serial='serial-one', - subject_id=owner, key_id=key_id(signer.public_key), issuer_id=owner, - parent_certificate_id=None, authority_sources=(), kind='capability', grants=(), - not_before=NOW, expires_at=NOW+timedelta(days=1), target_service=pin.service, - delegation_depth=0, issuance=None, - signature=Signature(key_id=signer.key_id, algorithm='ed25519', value=b'')), signer) + credential = Credential( + id=signer.key_id, + subject_id=owner, + kind='signing_key', + verifier=signer.public_key, + ceiling=(), + not_before=NOW, + expires_at=None, + revoked_at=None, + ) + certificate = sign_certificate( + Certificate( + resource_id='cert_one', + serial='serial-one', + subject_id=owner, + key_id=key_id(signer.public_key), + issuer_id=owner, + parent_certificate_id=None, + authority_sources=(), + kind='capability', + grants=(), + not_before=NOW, + expires_at=NOW + timedelta(days=1), + target_service=pin.service, + delegation_depth=0, + issuance=None, + signature=Signature(key_id=signer.key_id, algorithm='ed25519', value=b''), + ), + signer, + ) original = canonical(certificate).decode() member = Membership(organization_id='g_test', subject_id=owner, role='member', version=1) - resource = Resource(id='t_test', type='topic', type_version=1, name='test', parent=None, - owner=owner, group='g_public', mode=0o700, generation=1, revision=None, state='active', - created_at=NOW, created_by=owner, modified_at=NOW, modified_by=owner) + resource = Resource( + id='t_test', + type='topic', + type_version=1, + name='test', + parent=None, + owner=owner, + group='g_public', + mode=0o700, + generation=1, + revision=None, + state='active', + created_at=NOW, + created_by=owner, + modified_at=NOW, + modified_by=owner, + ) async with restored.transaction(write=True) as tx: await tx.insert(resource) - tx.execute('INSERT INTO credentials VALUES (?,?,?)', - (credential.id, owner, canonical(credential).decode()), write=True) - tx.execute('INSERT INTO certificates VALUES (?,?,NULL,0,?)', - (certificate.resource_id, owner, original), write=True) - tx.execute('INSERT INTO memberships VALUES (?,?,?,?)', - (member.organization_id, owner, member.version, canonical(member).decode()), write=True) - tx.execute('INSERT INTO topic_memberships VALUES (?,?,?,?,?,NULL)', - (resource.id, owner, 'member', 'active', wire(NOW)), write=True) - tx.execute('INSERT INTO identity_keys VALUES (?,?,?,?,NULL,1)', - (signer.key_id, owner, b64(signer.public_key), wire(NOW)), write=True) - tx.execute('INSERT INTO encryption_subkeys VALUES (?,?,?,?,?,NULL,1)', - (age_id, owner, recipient, b64(public_from_recipient(recipient)), wire(NOW)), write=True) - tx.execute('INSERT INTO custodial_vault VALUES (?,?,?,?,?,?,?,?,?,NULL)', - (owner, signer.key_id, age_id, 'signing-nonce', 'sealed-signing', - 'age-nonce', 'sealed-age', 'active', wire(NOW)), write=True) - tx.execute('INSERT INTO token_deliveries VALUES (?,?,?,?,?,?,NULL,NULL)', - ('token_one', owner, 'original-create', 'digest', 'verifier', - wire(NOW+timedelta(hours=1))), write=True) - tx.execute('INSERT INTO share_grants VALUES (?,?,?,?,?,?,NULL)', - ('sg_one', resource.id, owner, 'u_reader', wire(NOW), - wire(NOW+timedelta(days=1))), write=True) - tx.execute('INSERT INTO share_grants_v2 VALUES (?,?,?,?,?,NULL,?,?,0,?,?,NULL)', - ('sg_two', resource.id, owner, 'u_reader', 'user', '["read"]', '{}', - wire(NOW), wire(NOW+timedelta(days=1))), write=True) - tx.execute('INSERT INTO share_links VALUES (?,?,?,?,?,?,?,NULL)', - ('link_one', resource.id, owner, signer.key_id, 'link-verifier', - wire(NOW), wire(NOW+timedelta(days=1))), write=True) - tx.execute('''INSERT INTO topic_bans (topic,subject,actor,created_at,expires_at,reason,status) - VALUES (?,?,?,?,NULL,?,?)''', - (resource.id, owner, owner, wire(NOW), 'restored-ban', 'active'), write=True) + tx.execute( + 'INSERT INTO credentials VALUES (?,?,?)', + (credential.id, owner, canonical(credential).decode()), + write=True, + ) + tx.execute( + 'INSERT INTO certificates VALUES (?,?,NULL,0,?)', + (certificate.resource_id, owner, original), + write=True, + ) + tx.execute( + 'INSERT INTO memberships VALUES (?,?,?,?)', + (member.organization_id, owner, member.version, canonical(member).decode()), + write=True, + ) + tx.execute( + 'INSERT INTO topic_memberships VALUES (?,?,?,?,?,NULL)', + (resource.id, owner, 'member', 'active', wire(NOW)), + write=True, + ) + tx.execute( + 'INSERT INTO identity_keys VALUES (?,?,?,?,NULL,1)', + (signer.key_id, owner, b64(signer.public_key), wire(NOW)), + write=True, + ) + tx.execute( + 'INSERT INTO encryption_subkeys VALUES (?,?,?,?,?,NULL,1)', + (age_id, owner, recipient, b64(public_from_recipient(recipient)), wire(NOW)), + write=True, + ) + tx.execute( + 'INSERT INTO custodial_vault VALUES (?,?,?,?,?,?,?,?,?,NULL)', + ( + owner, + signer.key_id, + age_id, + 'signing-nonce', + 'sealed-signing', + 'age-nonce', + 'sealed-age', + 'active', + wire(NOW), + ), + write=True, + ) + tx.execute( + 'INSERT INTO token_deliveries VALUES (?,?,?,?,?,?,NULL,NULL)', + ( + 'token_one', + owner, + 'original-create', + 'digest', + 'verifier', + wire(NOW + timedelta(hours=1)), + ), + write=True, + ) + tx.execute( + 'INSERT INTO share_grants VALUES (?,?,?,?,?,?,NULL)', + ('sg_one', resource.id, owner, 'u_reader', wire(NOW), wire(NOW + timedelta(days=1))), + write=True, + ) + tx.execute( + 'INSERT INTO share_grants_v2 VALUES (?,?,?,?,?,NULL,?,?,0,?,?,NULL)', + ( + 'sg_two', + resource.id, + owner, + 'u_reader', + 'user', + '["read"]', + '{}', + wire(NOW), + wire(NOW + timedelta(days=1)), + ), + write=True, + ) + tx.execute( + 'INSERT INTO share_links VALUES (?,?,?,?,?,?,?,NULL)', + ( + 'link_one', + resource.id, + owner, + signer.key_id, + 'link-verifier', + wire(NOW), + wire(NOW + timedelta(days=1)), + ), + write=True, + ) + tx.execute( + """INSERT INTO topic_bans (topic,subject,actor,created_at,expires_at,reason,status) + VALUES (?,?,?,?,NULL,?,?)""", + (resource.id, owner, owner, wire(NOW), 'restored-ban', 'active'), + write=True, + ) banned = replace(resource, id='t_ban', name='ban', parent=resource.id) await tx.insert(banned) - targets = [('credential.revoke', 'token_one'), ('certificate.revoke', 'cert_one'), - ('share_grant.revoke', 'sg_one'), ('share_grant_v2.revoke', 'sg_two'), - ('share_link.revoke', 'link_one'), ('membership.remove', 'g_test'), - ('topic_membership.remove', 't_test'), ('topic_ban.apply', 't_ban'), - ('topic_ban.lift', resource.id), - ('identity_key.retire', signer.key_id), - ('encryption_key.retire', age_id), ('vault.destroy', age_id)] + targets = [ + ('credential.revoke', 'token_one'), + ('certificate.revoke', 'cert_one'), + ('share_grant.revoke', 'sg_one'), + ('share_grant_v2.revoke', 'sg_two'), + ('share_link.revoke', 'link_one'), + ('membership.remove', 'g_test'), + ('topic_membership.remove', 't_test'), + ('topic_ban.apply', 't_ban'), + ('topic_ban.lift', resource.id), + ('identity_key.retire', signer.key_id), + ('encryption_key.retire', age_id), + ('vault.destroy', age_id), + ] assert {kind for kind, _ in targets} == SUPPORTED_FACTS - facts = [{'sequence': i, 'kind': kind, 'subject': owner, 'target': target, 'at': wire(NOW)} - for i, (kind, target) in enumerate(targets, 1)] + facts = [ + {'sequence': i, 'kind': kind, 'subject': owner, 'target': target, 'at': wire(NOW)} + for i, (kind, target) in enumerate(targets, 1) + ] body = dict(body, entries=facts, sequence=len(facts)) pin = replace(pin, sequence=len(facts), digest=digest(body)) result = await replay(restored, signed(body), pin=pin) assert result['changed'] == len(facts) and result['backup_retired'] is False assert result['promotion_blocked_reasons'] == [ - 'quarantine_remains', 'supported_fact_inventory_incomplete', 'backup_retirement_not_attested'] + 'quarantine_remains', + 'supported_fact_inventory_incomplete', + 'backup_retirement_not_attested', + ] async with restored.transaction(write=False) as tx: - assert tx.one('SELECT body,revoked FROM certificates WHERE id=?', ('cert_one',)) == (original, 1) + assert tx.one('SELECT body,revoked FROM certificates WHERE id=?', ('cert_one',)) == ( + original, + 1, + ) assert (await tx.credential(signer.key_id)).revoked_at == NOW - assert tx.one('SELECT consumed_at FROM token_deliveries WHERE credential_id=?', ('token_one',)) == (wire(NOW),) + assert tx.one( + 'SELECT consumed_at FROM token_deliveries WHERE credential_id=?', ('token_one',) + ) == (wire(NOW),) for table in ('identity_keys', 'encryption_subkeys'): - assert tx.one(f'SELECT retired_at,is_primary FROM {table} WHERE subject=?', (owner,)) == (wire(NOW), 0) - for table, target in (('share_grants', 'sg_one'), ('share_grants_v2', 'sg_two'), ('share_links', 'link_one')): + assert tx.one( + f'SELECT retired_at,is_primary FROM {table} WHERE subject=?', (owner,) + ) == (wire(NOW), 0) + for table, target in ( + ('share_grants', 'sg_one'), + ('share_grants_v2', 'sg_two'), + ('share_links', 'link_one'), + ): assert tx.one(f'SELECT revoked_at FROM {table} WHERE id=?', (target,)) == (wire(NOW),) - current = decode(Membership, loads(tx.one('SELECT body FROM memberships WHERE org=? AND subject=?', - ('g_test', owner))[0])) + current = decode( + Membership, + loads( + tx.one('SELECT body FROM memberships WHERE org=? AND subject=?', ('g_test', owner))[ + 0 + ] + ), + ) assert current.version == 2 and current.status == 'rejected' - assert tx.one('SELECT status FROM topic_memberships WHERE topic=? AND subject=?', - ('t_test', owner)) == ('removed',) - assert tx.one('SELECT status FROM topic_bans WHERE topic=? AND subject=?', - ('t_test', owner)) == ('lifted',) - assert tx.one('SELECT status,actor,reason FROM topic_bans WHERE topic=? AND subject=?', - ('t_ban', owner)) == ('active', 'u_root', 'recovery-replay') - assert tx.one('SELECT status,signing_nonce,signing_ciphertext,age_nonce,age_ciphertext ' - 'FROM custodial_vault WHERE subject=?', (owner,)) == ('destroyed', None, None, None, None) + assert tx.one( + 'SELECT status FROM topic_memberships WHERE topic=? AND subject=?', ('t_test', owner) + ) == ('removed',) + assert tx.one( + 'SELECT status FROM topic_bans WHERE topic=? AND subject=?', ('t_test', owner) + ) == ('lifted',) + assert tx.one( + 'SELECT status,actor,reason FROM topic_bans WHERE topic=? AND subject=?', + ('t_ban', owner), + ) == ('active', 'u_root', 'recovery-replay') + assert tx.one( + 'SELECT status,signing_nonce,signing_ciphertext,age_nonce,age_ciphertext ' + 'FROM custodial_vault WHERE subject=?', + (owner,), + ) == ('destroyed', None, None, None, None) assert tx.one('SELECT count(*) FROM jobs')[0] == 0 and active(tx) assert canonical(await tx.resource(resource.id)).decode() == canonical(resource).decode() assert (await replay(restored, signed(body), pin=pin))['changed'] == 0 @@ -338,6 +551,7 @@ def test_packaged_schema_and_example_match_the_replay_contract(): from msg.admin.recovery_replay import SUPPORTED_FACTS from msg.core.codec import loads, unb64 + root = files('msg').joinpath('data') schema = loads(root.joinpath('recovery-checkpoint.schema.json').read_bytes()) example = loads(root.joinpath('recovery-checkpoint.example.json').read_bytes()) @@ -360,18 +574,37 @@ def test_packaged_schema_and_example_match_the_replay_contract(): @pytest.mark.parametrize('lift', [False, True]) async def test_replayed_ban_removes_restored_admin_even_after_lift(restored, checkpoint, lift): from msg.core.models import Resource + body, pin, signed = checkpoint - resource = Resource(id='t_old_admin', type='topic', type_version=1, name='old-admin', - parent=None, owner='u_owner', group='g_public', mode=0o700, generation=1, - revision=None, state='active', created_at=NOW, created_by='u_owner', - modified_at=NOW, modified_by='u_owner') + resource = Resource( + id='t_old_admin', + type='topic', + type_version=1, + name='old-admin', + parent=None, + owner='u_owner', + group='g_public', + mode=0o700, + generation=1, + revision=None, + state='active', + created_at=NOW, + created_by='u_owner', + modified_at=NOW, + modified_by='u_owner', + ) async with restored.transaction(write=True) as tx: await tx.insert(resource) - tx.execute('INSERT INTO topic_memberships VALUES (?,?,?,?,?,NULL)', - (resource.id, 'u_owner', 'admin', 'active', wire(NOW)), write=True) + tx.execute( + 'INSERT INTO topic_memberships VALUES (?,?,?,?,?,NULL)', + (resource.id, 'u_owner', 'admin', 'active', wire(NOW)), + write=True, + ) kinds = ['topic_ban.apply'] + (['topic_ban.lift'] if lift else []) - facts = [{'sequence': i, 'kind': kind, 'subject': 'u_owner', - 'target': resource.id, 'at': wire(NOW)} for i, kind in enumerate(kinds, 1)] + facts = [ + {'sequence': i, 'kind': kind, 'subject': 'u_owner', 'target': resource.id, 'at': wire(NOW)} + for i, kind in enumerate(kinds, 1) + ] body = dict(body, entries=facts, sequence=len(facts)) pin = replace(pin, digest=digest(body), sequence=len(facts)) for _ in range(2): @@ -379,8 +612,12 @@ async def test_replayed_ban_removes_restored_admin_even_after_lift(restored, che assert receipt['scope'] == 'supported_authority_reconciliation_facts' assert 'supported_fact_inventory_incomplete' in receipt['promotion_blocked_reasons'] async with restored.transaction(write=False) as tx: - assert tx.one('SELECT role,status FROM topic_memberships WHERE topic=? AND subject=?', - (resource.id, 'u_owner')) == ('member', 'removed') - assert tx.one('SELECT status FROM topic_bans WHERE topic=? AND subject=?', - (resource.id, 'u_owner')) == ('lifted' if lift else 'active',) + assert tx.one( + 'SELECT role,status FROM topic_memberships WHERE topic=? AND subject=?', + (resource.id, 'u_owner'), + ) == ('member', 'removed') + assert tx.one( + 'SELECT status FROM topic_bans WHERE topic=? AND subject=?', + (resource.id, 'u_owner'), + ) == ('lifted' if lift else 'active',) assert active(tx) diff --git a/tests/test_task_a_root_files.py b/tests/test_task_a_root_files.py index e653dabb..23777a60 100644 --- a/tests/test_task_a_root_files.py +++ b/tests/test_task_a_root_files.py @@ -1,4 +1,5 @@ """Real disposable Root storage; these tests do not claim physical-console evidence.""" + from concurrent.futures import ThreadPoolExecutor import pytest @@ -38,6 +39,7 @@ async def test_history_validation_precedes_database_root_transition(installed, t history.chmod(0o644) elif unsafe == 'hardlink': import os + os.link(history, tmp_path / 'duplicate-key') else: history.unlink() @@ -58,16 +60,25 @@ async def test_history_validation_precedes_database_root_transition(installed, t async def test_concurrent_prepare_does_not_replace_an_approved_journal(installed): app, root = installed old_envelope(app, root) + def attempt(_): try: - return prepare(app, Ed25519Signer.generate(), PIN, - old_signer=root, operator='isolated-concurrent-test') + return prepare( + app, + Ed25519Signer.generate(), + PIN, + old_signer=root, + operator='isolated-concurrent-test', + ) except Failure as exc: return exc.code + with ThreadPoolExecutor(max_workers=2) as pool: results = list(pool.map(attempt, range(2))) journals = [result for result in results if isinstance(result, dict)] assert len(journals) == 1 assert next(result for result in results if isinstance(result, str)) in { - 'root_rotation_busy', 'root_rotation_pending'} + 'root_rotation_busy', + 'root_rotation_pending', + } assert canonical(loads(journal_path(app).read_bytes())) == canonical(journals[0]) diff --git a/tests/test_temporary_dual_keys.py b/tests/test_temporary_dual_keys.py index 0b5023bb..9cfe8c41 100644 --- a/tests/test_temporary_dual_keys.py +++ b/tests/test_temporary_dual_keys.py @@ -1,129 +1,167 @@ """A new temporary subject binds independent client-held keys at creation.""" + import hashlib import os import pytest +from test_service import call, temporary_v3_args from msg.core.codec import b64, canonical, unb64, wire from msg.security.age_keys import generate_age_key, public_from_recipient from msg.security.crypto import Ed25519Signer -from test_service import call, temporary_v3_args def temporary_arguments(signer, recipient, *, nonce, request_id): - subject='u_tmp_'+hashlib.sha256(unb64(nonce)).hexdigest()[:32] - proof=signer.sign(canonical({'subject_id':subject,'nonce':nonce, - 'public_key':b64(signer.public_key), - 'encryption_recipient':recipient,'request_id':request_id}), - purpose='temporary-key-possession-v1') - return {'nonce':nonce,'recovery_secret':b64(os.urandom(32)), - 'public_key':b64(signer.public_key), - 'encryption_recipient':recipient,'possession_proof':wire(proof)} + subject = 'u_tmp_' + hashlib.sha256(unb64(nonce)).hexdigest()[:32] + proof = signer.sign( + canonical({ + 'subject_id': subject, + 'nonce': nonce, + 'public_key': b64(signer.public_key), + 'encryption_recipient': recipient, + 'request_id': request_id, + }), + purpose='temporary-key-possession-v1', + ) + return { + 'nonce': nonce, + 'recovery_secret': b64(os.urandom(32)), + 'public_key': b64(signer.public_key), + 'encryption_recipient': recipient, + 'possession_proof': wire(proof), + } @pytest.mark.asyncio async def test_temporary_v3_creates_two_client_keys_and_upgrade_reuses_them(installed): - app,_=installed - signer=Ed25519Signer.generate() - _,recipient=generate_age_key() - assert signer.public_key!=public_from_recipient(recipient) - request_id='temporary-dual-key' - args=temporary_arguments(signer,recipient,nonce=b64(os.urandom(32)), - request_id=request_id) - created=await call(app,'identity.temporary',args,rid=request_id, - contract_version=3) - assert created.status=='ok',wire(created) - subject=created.data['subject_id'] - assert created.data['encryption_recipient']==recipient + app, _ = installed + signer = Ed25519Signer.generate() + _, recipient = generate_age_key() + assert signer.public_key != public_from_recipient(recipient) + request_id = 'temporary-dual-key' + args = temporary_arguments(signer, recipient, nonce=b64(os.urandom(32)), request_id=request_id) + created = await call(app, 'identity.temporary', args, rid=request_id, contract_version=3) + assert created.status == 'ok', wire(created) + subject = created.data['subject_id'] + assert created.data['encryption_recipient'] == recipient assert created.data['server_signable'] is False assert created.data['server_decryptable'] is False async with app.metadata.transaction(write=False) as tx: - signing=tx.one('SELECT key_id,public_key FROM identity_keys WHERE subject=?', - (subject,)) - encryption=tx.one('SELECT key_id,recipient,public_key FROM encryption_subkeys WHERE subject=?', - (subject,)) - assert signing[0]==created.data['identity_key_id'] - assert signing[1]==b64(signer.public_key) - assert encryption[0]==created.data['encryption_key_id'] - assert encryption[1]==recipient - assert encryption[2]==b64(public_from_recipient(recipient)) - assert tx.one('SELECT COUNT(*) FROM custodial_vault WHERE subject=?',(subject,))[0]==0 - replay=await call(app,'identity.temporary',args,rid=request_id,contract_version=3) - assert replay.error.code=='token_delivery_unavailable' - signed={'subject_id':subject,'handle':'temporary-promoted', - 'public_key':args['public_key'],'encryption_recipient':recipient} - upgraded=await call(app,'identity.upgrade',{ - 'handle':'temporary-promoted','public_key':args['public_key'], - 'encryption_recipient':recipient, - 'possession_proof':wire(signer.sign(canonical(signed),purpose='upgrade'))}, + signing = tx.one('SELECT key_id,public_key FROM identity_keys WHERE subject=?', (subject,)) + encryption = tx.one( + 'SELECT key_id,recipient,public_key FROM encryption_subkeys WHERE subject=?', (subject,) + ) + assert signing[0] == created.data['identity_key_id'] + assert signing[1] == b64(signer.public_key) + assert encryption[0] == created.data['encryption_key_id'] + assert encryption[1] == recipient + assert encryption[2] == b64(public_from_recipient(recipient)) + assert tx.one('SELECT COUNT(*) FROM custodial_vault WHERE subject=?', (subject,))[0] == 0 + replay = await call(app, 'identity.temporary', args, rid=request_id, contract_version=3) + assert replay.error.code == 'token_delivery_unavailable' + signed = { + 'subject_id': subject, + 'handle': 'temporary-promoted', + 'public_key': args['public_key'], + 'encryption_recipient': recipient, + } + upgraded = await call( + app, + 'identity.upgrade', + { + 'handle': 'temporary-promoted', + 'public_key': args['public_key'], + 'encryption_recipient': recipient, + 'possession_proof': wire(signer.sign(canonical(signed), purpose='upgrade')), + }, + subject=subject, + token=(created.data['credential_id'], unb64(created.data['token'])), + contract_version=2, + ) + assert upgraded.status == 'ok', wire(upgraded) + assert upgraded.data['subject_id'] == subject + assert upgraded.data['key_id'] == created.data['identity_key_id'] + assert upgraded.data['encryption_key_id'] == created.data['encryption_key_id'] + old_token = (created.data['credential_id'], unb64(created.data['token'])) + denied = await call( + app, + 'content.post_create', + {'parent': '/main', 'body': 'old token'}, + subject=subject, + token=old_token, + ) + assert denied.error.code == 'credential_revoked' + signed = await call( + app, + 'content.post_create', + {'parent': '/main', 'body': 'new signing key'}, + key=signer, subject=subject, - token=(created.data['credential_id'],unb64(created.data['token'])), - contract_version=2) - assert upgraded.status=='ok',wire(upgraded) - assert upgraded.data['subject_id']==subject - assert upgraded.data['key_id']==created.data['identity_key_id'] - assert upgraded.data['encryption_key_id']==created.data['encryption_key_id'] - old_token=(created.data['credential_id'],unb64(created.data['token'])) - denied=await call(app,'content.post_create',{'parent':'/main','body':'old token'}, - subject=subject,token=old_token) - assert denied.error.code=='credential_revoked' - signed=await call(app,'content.post_create', - {'parent':'/main','body':'new signing key'}, - key=signer,subject=subject) - assert signed.status=='ok',wire(signed) + ) + assert signed.status == 'ok', wire(signed) async with app.metadata.transaction(write=False) as tx: - assert tx.one('SELECT COUNT(*) FROM identity_keys WHERE subject=?',(subject,))[0]==1 - assert tx.one('SELECT COUNT(*) FROM encryption_subkeys WHERE subject=?',(subject,))[0]==1 + assert tx.one('SELECT COUNT(*) FROM identity_keys WHERE subject=?', (subject,))[0] == 1 + assert tx.one('SELECT COUNT(*) FROM encryption_subkeys WHERE subject=?', (subject,))[0] == 1 @pytest.mark.asyncio async def test_legacy_temporary_versions_and_false_key_proof_create_no_subject(installed): - app,_=installed + app, _ = installed async with app.metadata.transaction(write=False) as tx: - before=tx.one("SELECT COUNT(*) FROM identities WHERE kind='temporary'")[0] - nonce=b64(os.urandom(32)) - for version,args in ((1,{'nonce':nonce}), - (2,{'nonce':nonce,'recovery_secret':b64(os.urandom(32))})): - denied=await call(app,'identity.temporary',args,contract_version=version) - assert denied.error.code=='temporary_dual_keys_required' - signer=Ed25519Signer.generate() - _,recipient=generate_age_key() - args=temporary_arguments(signer,recipient,nonce=b64(os.urandom(32)), - request_id='bad-temporary-proof') - args['public_key']=b64(Ed25519Signer.generate().public_key) - denied=await call(app,'identity.temporary',args,rid='bad-temporary-proof', - contract_version=3) - assert denied.status=='error' + before = tx.one("SELECT COUNT(*) FROM identities WHERE kind='temporary'")[0] + nonce = b64(os.urandom(32)) + for version, args in ( + (1, {'nonce': nonce}), + (2, {'nonce': nonce, 'recovery_secret': b64(os.urandom(32))}), + ): + denied = await call(app, 'identity.temporary', args, contract_version=version) + assert denied.error.code == 'temporary_dual_keys_required' + signer = Ed25519Signer.generate() + _, recipient = generate_age_key() + args = temporary_arguments( + signer, recipient, nonce=b64(os.urandom(32)), request_id='bad-temporary-proof' + ) + args['public_key'] = b64(Ed25519Signer.generate().public_key) + denied = await call( + app, 'identity.temporary', args, rid='bad-temporary-proof', contract_version=3 + ) + assert denied.status == 'error' async with app.metadata.transaction(write=False) as tx: - assert tx.one("SELECT COUNT(*) FROM identities WHERE kind='temporary'")[0]==before + assert tx.one("SELECT COUNT(*) FROM identities WHERE kind='temporary'")[0] == before @pytest.mark.asyncio async def test_temporary_upgrade_cannot_silently_replace_creation_keys(installed): - app,_=installed - original=Ed25519Signer.generate() - _,recipient=generate_age_key() - args,request_id,_,_=temporary_v3_args( - signer=original,recipient=recipient) - created=await call(app,'identity.temporary',args,rid=request_id, - contract_version=3) - assert created.status=='ok',wire(created) - subject=created.data['subject_id'] - replacement=Ed25519Signer.generate() - _,new_recipient=generate_age_key() - signed={'subject_id':subject,'handle':'changed-temporary', - 'public_key':b64(replacement.public_key), - 'encryption_recipient':new_recipient} - denied=await call(app,'identity.upgrade',{ - 'handle':signed['handle'],'public_key':signed['public_key'], - 'encryption_recipient':new_recipient, - 'possession_proof':wire(replacement.sign(canonical(signed),purpose='upgrade'))}, + app, _ = installed + original = Ed25519Signer.generate() + _, recipient = generate_age_key() + args, request_id, _, _ = temporary_v3_args(signer=original, recipient=recipient) + created = await call(app, 'identity.temporary', args, rid=request_id, contract_version=3) + assert created.status == 'ok', wire(created) + subject = created.data['subject_id'] + replacement = Ed25519Signer.generate() + _, new_recipient = generate_age_key() + signed = { + 'subject_id': subject, + 'handle': 'changed-temporary', + 'public_key': b64(replacement.public_key), + 'encryption_recipient': new_recipient, + } + denied = await call( + app, + 'identity.upgrade', + { + 'handle': signed['handle'], + 'public_key': signed['public_key'], + 'encryption_recipient': new_recipient, + 'possession_proof': wire(replacement.sign(canonical(signed), purpose='upgrade')), + }, subject=subject, - token=(created.data['credential_id'],unb64(created.data['token'])), - contract_version=2) - assert denied.error.code=='temporary_identity_key_mismatch' + token=(created.data['credential_id'], unb64(created.data['token'])), + contract_version=2, + ) + assert denied.error.code == 'temporary_identity_key_mismatch' async with app.metadata.transaction(write=False) as tx: - assert (await tx.subject(subject)).kind=='temporary' - assert tx.one('SELECT COUNT(*) FROM identity_keys WHERE subject=?',(subject,))[0]==1 - assert tx.one('SELECT COUNT(*) FROM encryption_subkeys WHERE subject=?', - (subject,))[0]==1 + assert (await tx.subject(subject)).kind == 'temporary' + assert tx.one('SELECT COUNT(*) FROM identity_keys WHERE subject=?', (subject,))[0] == 1 + assert tx.one('SELECT COUNT(*) FROM encryption_subkeys WHERE subject=?', (subject,))[0] == 1 diff --git a/tests/test_temporary_legacy_migration.py b/tests/test_temporary_legacy_migration.py index ecd9d6b6..12eb4c8a 100644 --- a/tests/test_temporary_legacy_migration.py +++ b/tests/test_temporary_legacy_migration.py @@ -1,4 +1,5 @@ """A stored token-only temporary subject upgrades without losing ownership.""" + import hashlib import os from datetime import timedelta @@ -6,6 +7,7 @@ from uuid import uuid4 import pytest +from test_service import NOW, call from msg.core.codec import b64, canonical, wire from msg.core.errors import Failure @@ -13,81 +15,115 @@ from msg.plugins import identity from msg.security.age_keys import generate_age_key from msg.security.crypto import Ed25519Signer -from test_service import NOW, call @pytest.mark.asyncio -async def test_legacy_temporary_upgrade_is_atomic_and_preserves_subject(installed, - monkeypatch): - app,_=installed - subject='u_tmp_legacy_'+uuid4().hex[:20] - token_bytes=os.urandom(32) - token_id='t_'+subject[2:] +async def test_legacy_temporary_upgrade_is_atomic_and_preserves_subject(installed, monkeypatch): + app, _ = installed + subject = 'u_tmp_legacy_' + uuid4().hex[:20] + token_bytes = os.urandom(32) + token_id = 't_' + subject[2:] async with app.metadata.transaction(write=True) as tx: - await identity.make_user(app,tx,SimpleNamespace(now=NOW),subject, - 'tmp-'+subject[-20:],'temporary') - await tx.save_credential(Credential( - id=token_id,subject_id=subject,kind='token', - verifier=hashlib.sha256(token_bytes).digest(), - ceiling=app.temporary_ceiling(),not_before=NOW, - expires_at=NOW+timedelta(hours=1),revoked_at=None),0) - token=(token_id,token_bytes) - post=await call(app,'content.post_create', - {'parent':'/tmp','body':'legacy temporary work'}, - subject=subject,token=token) - assert post.status=='ok',wire(post) - owner_ref=post.resources[0].id - signer=Ed25519Signer.generate() - _,recipient=generate_age_key() - signed={'subject_id':subject,'public_key':b64(signer.public_key), - 'handle':'legacy-promoted','encryption_recipient':recipient} - args={'handle':'legacy-promoted','public_key':signed['public_key'], - 'encryption_recipient':recipient, - 'possession_proof':wire(signer.sign(canonical(signed),purpose='upgrade'))} - original=identity.issue_online + await identity.make_user( + app, tx, SimpleNamespace(now=NOW), subject, 'tmp-' + subject[-20:], 'temporary' + ) + await tx.save_credential( + Credential( + id=token_id, + subject_id=subject, + kind='token', + verifier=hashlib.sha256(token_bytes).digest(), + ceiling=app.temporary_ceiling(), + not_before=NOW, + expires_at=NOW + timedelta(hours=1), + revoked_at=None, + ), + 0, + ) + token = (token_id, token_bytes) + post = await call( + app, + 'content.post_create', + {'parent': '/tmp', 'body': 'legacy temporary work'}, + subject=subject, + token=token, + ) + assert post.status == 'ok', wire(post) + owner_ref = post.resources[0].id + signer = Ed25519Signer.generate() + _, recipient = generate_age_key() + signed = { + 'subject_id': subject, + 'public_key': b64(signer.public_key), + 'handle': 'legacy-promoted', + 'encryption_recipient': recipient, + } + args = { + 'handle': 'legacy-promoted', + 'public_key': signed['public_key'], + 'encryption_recipient': recipient, + 'possession_proof': wire(signer.sign(canonical(signed), purpose='upgrade')), + } + original = identity.issue_online - async def fail_after_identity_writes(*_args,**_kwargs): + async def fail_after_identity_writes(*_args, **_kwargs): raise Failure('forced_issue_failure') - monkeypatch.setattr(identity,'issue_online',fail_after_identity_writes) - failed=await call(app,'identity.upgrade',args,subject=subject,token=token, - contract_version=2) - assert failed.error.code=='forced_issue_failure' + monkeypatch.setattr(identity, 'issue_online', fail_after_identity_writes) + failed = await call( + app, 'identity.upgrade', args, subject=subject, token=token, contract_version=2 + ) + assert failed.error.code == 'forced_issue_failure' async with app.metadata.transaction(write=False) as tx: - assert (await tx.subject(subject)).kind=='temporary' - assert (await tx.resource(owner_ref)).owner==subject - assert tx.one('SELECT COUNT(*) FROM identity_keys WHERE subject=?',(subject,))[0]==0 - assert tx.one('SELECT COUNT(*) FROM encryption_subkeys WHERE subject=?', - (subject,))[0]==0 + assert (await tx.subject(subject)).kind == 'temporary' + assert (await tx.resource(owner_ref)).owner == subject + assert tx.one('SELECT COUNT(*) FROM identity_keys WHERE subject=?', (subject,))[0] == 0 + assert tx.one('SELECT COUNT(*) FROM encryption_subkeys WHERE subject=?', (subject,))[0] == 0 assert (await tx.credential(token_id)).revoked_at is None - monkeypatch.setattr(identity,'issue_online',original) - upgraded=await call(app,'identity.upgrade',args,subject=subject,token=token, - contract_version=2) - assert upgraded.status=='ok',wire(upgraded) - assert upgraded.data['subject_id']==subject - assert upgraded.data['encryption_recipient']==recipient + monkeypatch.setattr(identity, 'issue_online', original) + upgraded = await call( + app, 'identity.upgrade', args, subject=subject, token=token, contract_version=2 + ) + assert upgraded.status == 'ok', wire(upgraded) + assert upgraded.data['subject_id'] == subject + assert upgraded.data['encryption_recipient'] == recipient async with app.metadata.transaction(write=False) as tx: - assert (await tx.subject(subject)).kind=='registered' - assert (await tx.resource(owner_ref)).owner==subject - assert tx.one('SELECT COUNT(*) FROM identity_keys WHERE subject=?',(subject,))[0]==1 - key=tx.one('SELECT key_id,public_key FROM identity_keys WHERE subject=? AND is_primary=1', - (subject,)) - assert key==(upgraded.data['key_id'],b64(signer.public_key)) - assert tx.one('SELECT COUNT(*) FROM encryption_subkeys WHERE subject=?', - (subject,))[0]==1 - encryption=tx.one('SELECT key_id,recipient,public_key FROM encryption_subkeys WHERE subject=? AND is_primary=1', - (subject,)) - assert encryption==(upgraded.data['encryption_key_id'],recipient, - b64(identity.public_from_recipient(recipient))) + assert (await tx.subject(subject)).kind == 'registered' + assert (await tx.resource(owner_ref)).owner == subject + assert tx.one('SELECT COUNT(*) FROM identity_keys WHERE subject=?', (subject,))[0] == 1 + key = tx.one( + 'SELECT key_id,public_key FROM identity_keys WHERE subject=? AND is_primary=1', + (subject,), + ) + assert key == (upgraded.data['key_id'], b64(signer.public_key)) + assert tx.one('SELECT COUNT(*) FROM encryption_subkeys WHERE subject=?', (subject,))[0] == 1 + encryption = tx.one( + 'SELECT key_id,recipient,public_key FROM encryption_subkeys WHERE subject=? AND is_primary=1', + (subject,), + ) + assert encryption == ( + upgraded.data['encryption_key_id'], + recipient, + b64(identity.public_from_recipient(recipient)), + ) assert (await tx.credential(token_id)).revoked_at is not None - denied=await call(app,'content.post_create', - {'parent':'/tmp','body':'legacy token after upgrade'}, - subject=subject,token=token) - assert denied.error.code=='credential_revoked' - signed_post=await call(app,'content.post_create', - {'parent':'/main','body':'migrated signing key'}, - key=signer,subject=subject) - assert signed_post.status=='ok',wire(signed_post) - repeated=await call(app,'identity.upgrade',args,subject=subject,token=token, - contract_version=2) - assert repeated.error.code=='credential_revoked' + denied = await call( + app, + 'content.post_create', + {'parent': '/tmp', 'body': 'legacy token after upgrade'}, + subject=subject, + token=token, + ) + assert denied.error.code == 'credential_revoked' + signed_post = await call( + app, + 'content.post_create', + {'parent': '/main', 'body': 'migrated signing key'}, + key=signer, + subject=subject, + ) + assert signed_post.status == 'ok', wire(signed_post) + repeated = await call( + app, 'identity.upgrade', args, subject=subject, token=token, contract_version=2 + ) + assert repeated.error.code == 'credential_revoked' diff --git a/tests/test_temporary_signed_business.py b/tests/test_temporary_signed_business.py index 6c64c658..04dec054 100644 --- a/tests/test_temporary_signed_business.py +++ b/tests/test_temporary_signed_business.py @@ -1,22 +1,26 @@ """Signed temporary subjects and their bootstrap token have distinct limits.""" + import pytest +from test_service import call, temporary_v3_args from msg.core.codec import unb64, wire -from test_service import call, temporary_v3_args @pytest.mark.asyncio async def test_temporary_signing_key_can_create_low_risk_content(installed): app, _ = installed args, request_id, signer, _ = temporary_v3_args() - created = await call(app, 'identity.temporary', args, rid=request_id, - contract_version=3) + created = await call(app, 'identity.temporary', args, rid=request_id, contract_version=3) assert created.status == 'ok', wire(created) subject = created.data['subject_id'] - post = await call(app, 'content.post_create', - {'parent': '/tmp', 'body': 'signed temporary post'}, - key=signer, subject=subject) + post = await call( + app, + 'content.post_create', + {'parent': '/tmp', 'body': 'signed temporary post'}, + key=signer, + subject=subject, + ) assert post.status == 'ok', f'{post.error.code}: {wire(post)}' @@ -24,20 +28,27 @@ async def test_temporary_signing_key_can_create_low_risk_content(installed): async def test_temporary_token_allows_low_risk_write_but_not_permission_changes(installed): app, _ = installed args, request_id, _, _ = temporary_v3_args() - created = await call(app, 'identity.temporary', args, rid=request_id, - contract_version=3) + created = await call(app, 'identity.temporary', args, rid=request_id, contract_version=3) assert created.status == 'ok', wire(created) subject = created.data['subject_id'] token = (created.data['credential_id'], unb64(created.data['token'])) - post = await call(app, 'content.post_create', - {'parent': '/tmp', 'body': 'token temporary post'}, - subject=subject, token=token) + post = await call( + app, + 'content.post_create', + {'parent': '/tmp', 'body': 'token temporary post'}, + subject=subject, + token=token, + ) assert post.status == 'ok', wire(post) - chmod = await call(app, 'content.chmod', - {'id': post.resources[0].id, 'mode': '0777'}, - subject=subject, token=token, - expected=((post.resources[0].id, post.data['generation']),)) + chmod = await call( + app, + 'content.chmod', + {'id': post.resources[0].id, 'mode': '0777'}, + subject=subject, + token=token, + expected=((post.resources[0].id, post.data['generation']),), + ) assert chmod.status == 'error' assert chmod.error.code == 'credential_ceiling', wire(chmod) diff --git a/tests/test_text_patch.py b/tests/test_text_patch.py index ae9b3b82..5a2c3e20 100644 --- a/tests/test_text_patch.py +++ b/tests/test_text_patch.py @@ -1,101 +1,208 @@ """Exact text patches keep a single authorized Revision boundary.""" + import pytest +from test_service import call, register from msg.core.codec import b64, wire from msg.core.models import ResourceRef -from test_service import call, register -async def state(app,rid): +async def state(app, rid): async with app.metadata.transaction(write=False) as tx: - resource=await tx.resource(rid) - return (resource.generation,resource.revision, - tx.one('SELECT COUNT(*) FROM revisions')[0], - tx.one('SELECT COUNT(*) FROM events')[0]) + resource = await tx.resource(rid) + return ( + resource.generation, + resource.revision, + tx.one('SELECT COUNT(*) FROM revisions')[0], + tx.one('SELECT COUNT(*) FROM events')[0], + ) -async def content(app,rid): +async def content(app, rid): async with app.metadata.transaction(write=False) as tx: - revision=await tx.revision(ResourceRef(id=rid)) - return revision,(await app.contents.read_bytes(revision.content)).decode('utf-8') + revision = await tx.revision(ResourceRef(id=rid)) + return revision, (await app.contents.read_bytes(revision.content)).decode('utf-8') @pytest.mark.asyncio async def test_context_patch_post_and_file_preserve_revision_history(installed): - app,_=installed - key,owner,_=await register(app,'patch-owner') - post=await call(app,'content.post_create',{'parent':'/main','body':'甲 fish\n乙 fish\n'}, - key=key,subject=owner) - assert post.status=='ok',wire(post) - rid=post.resources[0].id - old=post.resources[0].revision - changed=await call(app,'content.text_patch',{'id':rid,'base_revision':old, - 'exact':'fish','replacement':'鱼','before':'乙 ','after':'\n'}, - key=key,subject=owner,expected=((rid,post.data['generation']),)) - assert changed.status=='ok',wire(changed) - revision,body=await content(app,rid) - assert body=='甲 fish\n乙 鱼\n' - assert revision.parents==(old,) - assert revision.content.media_type=='text/markdown' - assert revision.actor==owner and revision.subject==owner and revision.manifest_digest + app, _ = installed + key, owner, _ = await register(app, 'patch-owner') + post = await call( + app, + 'content.post_create', + {'parent': '/main', 'body': '甲 fish\n乙 fish\n'}, + key=key, + subject=owner, + ) + assert post.status == 'ok', wire(post) + rid = post.resources[0].id + old = post.resources[0].revision + changed = await call( + app, + 'content.text_patch', + { + 'id': rid, + 'base_revision': old, + 'exact': 'fish', + 'replacement': '鱼', + 'before': '乙 ', + 'after': '\n', + }, + key=key, + subject=owner, + expected=((rid, post.data['generation']),), + ) + assert changed.status == 'ok', wire(changed) + revision, body = await content(app, rid) + assert body == '甲 fish\n乙 鱼\n' + assert revision.parents == (old,) + assert revision.content.media_type == 'text/markdown' + assert revision.actor == owner and revision.subject == owner and revision.manifest_digest - file=await call(app,'content.file_put',{'parent':'/@patch-owner/files','name':'sample.txt', - 'data':b64('汉字 A'.encode()),'media_type':'text/plain'},key=key,subject=owner) - assert file.status=='ok',wire(file) - fid=file.resources[0].id - result=await call(app,'content.text_patch',{'id':fid,'base_revision':file.resources[0].revision, - 'exact':'汉字','replacement':'文字'},key=key,subject=owner, - expected=((fid,file.data['generation']),)) - assert result.status=='ok',wire(result) - revision,body=await content(app,fid) - assert body=='文字 A' and revision.content.media_type=='text/plain' + file = await call( + app, + 'content.file_put', + { + 'parent': '/@patch-owner/files', + 'name': 'sample.txt', + 'data': b64('汉字 A'.encode()), + 'media_type': 'text/plain', + }, + key=key, + subject=owner, + ) + assert file.status == 'ok', wire(file) + fid = file.resources[0].id + result = await call( + app, + 'content.text_patch', + { + 'id': fid, + 'base_revision': file.resources[0].revision, + 'exact': '汉字', + 'replacement': '文字', + }, + key=key, + subject=owner, + expected=((fid, file.data['generation']),), + ) + assert result.status == 'ok', wire(result) + revision, body = await content(app, fid) + assert body == '文字 A' and revision.content.media_type == 'text/plain' @pytest.mark.asyncio -async def test_patch_rejects_ambiguous_stale_missing_and_unauthorized_without_publication(installed): - app,_=installed - key,owner,_=await register(app,'patch-conflicts') - other,outsider,_=await register(app,'patch-outsider') - post=await call(app,'content.post_create',{'parent':'/main','body':'one one'}, - key=key,subject=owner) - rid=post.resources[0].id - base=post.resources[0].revision - generation=post.data['generation'] - original=await state(app,rid) - cases=[ - (key,owner,{'id':rid,'base_revision':base,'exact':'one','replacement':'two'},generation,'patch_ambiguous'), - (key,owner,{'id':rid,'base_revision':base,'exact':'absent','replacement':'two'},generation,'patch_no_match'), - (key,owner,{'id':rid,'base_revision':'v_stale','exact':'one','replacement':'two'},generation,'revision_conflict'), - (key,owner,{'id':rid,'base_revision':base,'exact':'one','replacement':'two'},generation+1,'generation_conflict'), - (other,outsider,{'id':rid,'base_revision':base,'exact':'one','replacement':'two'},generation,'permission_denied'), +async def test_patch_rejects_ambiguous_stale_missing_and_unauthorized_without_publication( + installed, +): + app, _ = installed + key, owner, _ = await register(app, 'patch-conflicts') + other, outsider, _ = await register(app, 'patch-outsider') + post = await call( + app, 'content.post_create', {'parent': '/main', 'body': 'one one'}, key=key, subject=owner + ) + rid = post.resources[0].id + base = post.resources[0].revision + generation = post.data['generation'] + original = await state(app, rid) + cases = [ + ( + key, + owner, + {'id': rid, 'base_revision': base, 'exact': 'one', 'replacement': 'two'}, + generation, + 'patch_ambiguous', + ), + ( + key, + owner, + {'id': rid, 'base_revision': base, 'exact': 'absent', 'replacement': 'two'}, + generation, + 'patch_no_match', + ), + ( + key, + owner, + {'id': rid, 'base_revision': 'v_stale', 'exact': 'one', 'replacement': 'two'}, + generation, + 'revision_conflict', + ), + ( + key, + owner, + {'id': rid, 'base_revision': base, 'exact': 'one', 'replacement': 'two'}, + generation + 1, + 'generation_conflict', + ), + ( + other, + outsider, + {'id': rid, 'base_revision': base, 'exact': 'one', 'replacement': 'two'}, + generation, + 'permission_denied', + ), ] - for signer,subject,args,expected_generation,code in cases: - failed=await call(app,'content.text_patch',args,key=signer,subject=subject, - expected=((rid,expected_generation),)) - assert failed.status=='error' and failed.error.code==code,wire(failed) - assert await state(app,rid)==original + for signer, subject, args, expected_generation, code in cases: + failed = await call( + app, + 'content.text_patch', + args, + key=signer, + subject=subject, + expected=((rid, expected_generation),), + ) + assert failed.status == 'error' and failed.error.code == code, wire(failed) + assert await state(app, rid) == original @pytest.mark.asyncio async def test_patch_rejects_binary_and_bounded_complexity(installed): - app,_=installed - key,owner,_=await register(app,'patch-bounds') - file=await call(app,'content.file_put',{'parent':'/@patch-bounds/files','name':'binary.bin', - 'data':b64(b'\xff\x00'),'media_type':'application/octet-stream'},key=key,subject=owner) - fid=file.resources[0].id - before=await state(app,fid) - failed=await call(app,'content.text_patch',{'id':fid,'base_revision':file.resources[0].revision, - 'exact':'a','replacement':'b'},key=key,subject=owner, - expected=((fid,file.data['generation']),)) - assert failed.status=='error' and failed.error.code=='text_patch_required',wire(failed) - assert await state(app,fid)==before + app, _ = installed + key, owner, _ = await register(app, 'patch-bounds') + file = await call( + app, + 'content.file_put', + { + 'parent': '/@patch-bounds/files', + 'name': 'binary.bin', + 'data': b64(b'\xff\x00'), + 'media_type': 'application/octet-stream', + }, + key=key, + subject=owner, + ) + fid = file.resources[0].id + before = await state(app, fid) + failed = await call( + app, + 'content.text_patch', + {'id': fid, 'base_revision': file.resources[0].revision, 'exact': 'a', 'replacement': 'b'}, + key=key, + subject=owner, + expected=((fid, file.data['generation']),), + ) + assert failed.status == 'error' and failed.error.code == 'text_patch_required', wire(failed) + assert await state(app, fid) == before - post=await call(app,'content.post_create',{'parent':'/main','body':'a'*5000}, - key=key,subject=owner) - rid=post.resources[0].id - before=await state(app,rid) - failed=await call(app,'content.text_patch',{'id':rid,'base_revision':post.resources[0].revision, - 'exact':'a','replacement':'b','before':'z'},key=key,subject=owner, - expected=((rid,post.data['generation']),)) - assert failed.status=='error' and failed.error.code=='patch_too_complex',wire(failed) - assert await state(app,rid)==before + post = await call( + app, 'content.post_create', {'parent': '/main', 'body': 'a' * 5000}, key=key, subject=owner + ) + rid = post.resources[0].id + before = await state(app, rid) + failed = await call( + app, + 'content.text_patch', + { + 'id': rid, + 'base_revision': post.resources[0].revision, + 'exact': 'a', + 'replacement': 'b', + 'before': 'z', + }, + key=key, + subject=owner, + expected=((rid, post.data['generation']),), + ) + assert failed.status == 'error' and failed.error.code == 'patch_too_complex', wire(failed) + assert await state(app, rid) == before diff --git a/tests/test_text_patch_line_boundaries.py b/tests/test_text_patch_line_boundaries.py index 9998be02..dd1d1ee6 100644 --- a/tests/test_text_patch_line_boundaries.py +++ b/tests/test_text_patch_line_boundaries.py @@ -1,4 +1,5 @@ """Git diff lines are LF-delimited; Unicode separators are signed content bytes.""" + import subprocess import pytest @@ -14,15 +15,20 @@ def test_unified_patch_preserves_non_lf_bytes_and_agrees_with_git(tmp_path, separator): before = f'left{separator}old\nunchanged\n' after = f'left{separator}new\nunchanged\n' - diff = ('--- a/document.txt\n+++ b/document.txt\n@@ -1,2 +1,2 @@\n' - f'-left{separator}old\n+left{separator}new\n unchanged\n') + diff = ( + '--- a/document.txt\n+++ b/document.txt\n@@ -1,2 +1,2 @@\n' + f'-left{separator}old\n+left{separator}new\n unchanged\n' + ) target = tmp_path / 'document.txt' target.write_bytes(before.encode()) - subprocess.run(['git', 'apply', '-'], cwd=tmp_path, input=diff.encode(), - check=True, capture_output=True) + subprocess.run( + ['git', 'apply', '-'], cwd=tmp_path, input=diff.encode(), check=True, capture_output=True + ) assert target.read_bytes() == after.encode() assert apply_patch(before, {'kind': 'unified', 'diff': diff}).encode() == target.read_bytes() - rebased = apply_patch('prefix\n' + before, {'kind': 'unified', 'diff': diff}, base_source=before) + rebased = apply_patch( + 'prefix\n' + before, {'kind': 'unified', 'diff': diff}, base_source=before + ) assert rebased.encode() == ('prefix\n' + after).encode() @@ -31,24 +37,45 @@ def test_inline_unicode_separator_cannot_forge_a_markdown_heading(separator): source = f'inline{separator}# Not a heading\nparagraph\n' fake_section = '# Not a heading\nparagraph\n' with pytest.raises(Failure) as caught: - apply_patch(source, {'kind': 'heading', 'heading': 'Not a heading', - 'digest': digest(fake_section.encode()), 'replacement': 'tampered\n'}) + apply_patch( + source, + { + 'kind': 'heading', + 'heading': 'Not a heading', + 'digest': digest(fake_section.encode()), + 'replacement': 'tampered\n', + }, + ) assert caught.value.code == 'patch_no_match' - assert apply_patch(source, {'kind': 'block', 'digest': digest(source.encode()), - 'replacement': 'replaced\n'}) == 'replaced\n' + assert ( + apply_patch( + source, + {'kind': 'block', 'digest': digest(source.encode()), 'replacement': 'replaced\n'}, + ) + == 'replaced\n' + ) @pytest.mark.parametrize('ending', ['\n', '\r\n', '\r']) def test_markdown_keeps_real_line_endings_and_signed_section_digest(ending): section = '# Title' + ending + 'text' + ending source = section + '# Other' + ending + 'untouched' + ending - patch = {'kind': 'heading', 'heading': 'Title', 'digest': digest(section.encode()), - 'replacement': '# New' + ending} - assert apply_patch(source, patch) == '# New' + ending + '# Other' + ending + 'untouched' + ending + patch = { + 'kind': 'heading', + 'heading': 'Title', + 'digest': digest(section.encode()), + 'replacement': '# New' + ending, + } + assert ( + apply_patch(source, patch) == '# New' + ending + '# Other' + ending + 'untouched' + ending + ) def test_unified_no_newline_markers_preserve_inline_unicode(): before, after = 'same\u2028old', 'same\u2028new' - patch = {'kind': 'unified', 'diff': '@@ -1 +1 @@\n-same\u2028old\n' - '\\ No newline at end of file\n+same\u2028new\n\\ No newline at end of file\n'} + patch = { + 'kind': 'unified', + 'diff': '@@ -1 +1 @@\n-same\u2028old\n' + '\\ No newline at end of file\n+same\u2028new\n\\ No newline at end of file\n', + } assert apply_patch(before, patch) == after diff --git a/tests/test_text_patch_rebase_batch.py b/tests/test_text_patch_rebase_batch.py index 49d6f78a..7f47fe5e 100644 --- a/tests/test_text_patch_rebase_batch.py +++ b/tests/test_text_patch_rebase_batch.py @@ -1,104 +1,202 @@ """Context rebase and multi-file patches use one PostgreSQL commit boundary.""" + import asyncio import pytest +from test_service import call, register from msg.core.codec import wire from msg.core.models import ResourceRef -from test_service import call, register async def snapshot(app, ids): async with app.metadata.transaction(write=False) as tx: - resources=[await tx.resource(rid) for rid in ids] - return (tuple((resource.generation,resource.revision) for resource in resources), - tx.one('SELECT COUNT(*) FROM revisions')[0], - tx.one('SELECT COUNT(*) FROM events')[0]) + resources = [await tx.resource(rid) for rid in ids] + return ( + tuple((resource.generation, resource.revision) for resource in resources), + tx.one('SELECT COUNT(*) FROM revisions')[0], + tx.one('SELECT COUNT(*) FROM events')[0], + ) async def body(app, rid): async with app.metadata.transaction(write=False) as tx: - revision=await tx.revision(ResourceRef(id=rid)) + revision = await tx.revision(ResourceRef(id=rid)) return (await app.contents.read_bytes(revision.content)).decode('utf-8') @pytest.mark.asyncio async def test_rebase_unique_context_and_reject_changed_or_ambiguous(installed): - app,_=installed - key,subject,_=await register(app,'rebase-owner') - created=await call(app,'content.post_create',{'parent':'/main','body':'alpha one\nbeta two\n'},key=key,subject=subject) - rid=created.resources[0].id - base=created.resources[0].revision - first=await call(app,'content.text_patch',{'id':rid,'base_revision':base, - 'exact':'alpha','replacement':'ALPHA'},key=key,subject=subject, - expected=((rid,created.data['generation']),)) - assert first.status=='ok',wire(first) - stale={'id':rid,'base_revision':base,'base_generation':created.data['generation'], - 'rebase':True,'exact':'beta','replacement':'BETA','after':' two'} - rebased=await call(app,'content.text_patch',stale,key=key,subject=subject, - expected=((rid,first.data['generation']),),contract_version=2) - assert rebased.status=='ok',wire(rebased) - assert await body(app,rid)=='ALPHA one\nBETA two\n' - before=await snapshot(app,(rid,)) - changed=await call(app,'content.text_patch',{**stale,'exact':'alpha'},key=key,subject=subject, - expected=((rid,rebased.data['generation']),),contract_version=2) - assert changed.error.code=='patch_no_match',wire(changed) - bad_generation=await call(app,'content.text_patch',{**stale,'base_generation':rebased.data['generation']}, - key=key,subject=subject,expected=((rid,rebased.data['generation']),),contract_version=2) - assert bad_generation.error.code=='base_generation_conflict',wire(bad_generation) - assert await snapshot(app,(rid,))==before + app, _ = installed + key, subject, _ = await register(app, 'rebase-owner') + created = await call( + app, + 'content.post_create', + {'parent': '/main', 'body': 'alpha one\nbeta two\n'}, + key=key, + subject=subject, + ) + rid = created.resources[0].id + base = created.resources[0].revision + first = await call( + app, + 'content.text_patch', + {'id': rid, 'base_revision': base, 'exact': 'alpha', 'replacement': 'ALPHA'}, + key=key, + subject=subject, + expected=((rid, created.data['generation']),), + ) + assert first.status == 'ok', wire(first) + stale = { + 'id': rid, + 'base_revision': base, + 'base_generation': created.data['generation'], + 'rebase': True, + 'exact': 'beta', + 'replacement': 'BETA', + 'after': ' two', + } + rebased = await call( + app, + 'content.text_patch', + stale, + key=key, + subject=subject, + expected=((rid, first.data['generation']),), + contract_version=2, + ) + assert rebased.status == 'ok', wire(rebased) + assert await body(app, rid) == 'ALPHA one\nBETA two\n' + before = await snapshot(app, (rid,)) + changed = await call( + app, + 'content.text_patch', + {**stale, 'exact': 'alpha'}, + key=key, + subject=subject, + expected=((rid, rebased.data['generation']),), + contract_version=2, + ) + assert changed.error.code == 'patch_no_match', wire(changed) + bad_generation = await call( + app, + 'content.text_patch', + {**stale, 'base_generation': rebased.data['generation']}, + key=key, + subject=subject, + expected=((rid, rebased.data['generation']),), + contract_version=2, + ) + assert bad_generation.error.code == 'base_generation_conflict', wire(bad_generation) + assert await snapshot(app, (rid,)) == before @pytest.mark.asyncio async def test_batch_preflight_failure_and_concurrent_writers_publish_nothing_partial(installed): - app,_=installed - key,subject,_=await register(app,'batch-patcher') - a=await call(app,'content.post_create',{'parent':'/main','body':'first A'},key=key,subject=subject) - b=await call(app,'content.post_create',{'parent':'/main','body':'second B'},key=key,subject=subject) - ids=(a.resources[0].id,b.resources[0].id) - expected=((ids[0],a.data['generation']),(ids[1],b.data['generation'])) - patches=[{'id':ids[0],'base_revision':a.resources[0].revision,'base_generation':a.data['generation'], - 'exact':'A','replacement':'AA'}, - {'id':ids[1],'base_revision':b.resources[0].revision,'base_generation':b.data['generation'], - 'exact':'absent','replacement':'BB'}] - before=await snapshot(app,ids) - rejected=await call(app,'content.text_patch_batch',{'patches':patches},key=key,subject=subject,expected=expected) - assert rejected.error.code=='patch_no_match',wire(rejected) - assert await snapshot(app,ids)==before - patches[1]['exact']='B' - first,second=await asyncio.gather(*( - call(app,'content.text_patch_batch',{'patches':patches},key=key,subject=subject, - expected=expected) for _ in range(2))) - assert sorted((first.status,second.status))==['error','ok'],(wire(first),wire(second)) - assert (first.error or second.error).code=='generation_conflict' - assert await body(app,ids[0])=='first AA' - assert await body(app,ids[1])=='second BB' + app, _ = installed + key, subject, _ = await register(app, 'batch-patcher') + a = await call( + app, 'content.post_create', {'parent': '/main', 'body': 'first A'}, key=key, subject=subject + ) + b = await call( + app, + 'content.post_create', + {'parent': '/main', 'body': 'second B'}, + key=key, + subject=subject, + ) + ids = (a.resources[0].id, b.resources[0].id) + expected = ((ids[0], a.data['generation']), (ids[1], b.data['generation'])) + patches = [ + { + 'id': ids[0], + 'base_revision': a.resources[0].revision, + 'base_generation': a.data['generation'], + 'exact': 'A', + 'replacement': 'AA', + }, + { + 'id': ids[1], + 'base_revision': b.resources[0].revision, + 'base_generation': b.data['generation'], + 'exact': 'absent', + 'replacement': 'BB', + }, + ] + before = await snapshot(app, ids) + rejected = await call( + app, + 'content.text_patch_batch', + {'patches': patches}, + key=key, + subject=subject, + expected=expected, + ) + assert rejected.error.code == 'patch_no_match', wire(rejected) + assert await snapshot(app, ids) == before + patches[1]['exact'] = 'B' + first, second = await asyncio.gather( + *( + call( + app, + 'content.text_patch_batch', + {'patches': patches}, + key=key, + subject=subject, + expected=expected, + ) + for _ in range(2) + ) + ) + assert sorted((first.status, second.status)) == ['error', 'ok'], (wire(first), wire(second)) + assert (first.error or second.error).code == 'generation_conflict' + assert await body(app, ids[0]) == 'first AA' + assert await body(app, ids[1]) == 'second BB' async with app.metadata.transaction(write=False) as tx: - assert tx.one('SELECT COUNT(*) FROM revisions WHERE resource_id IN (?,?)',ids)[0]==4 + assert tx.one('SELECT COUNT(*) FROM revisions WHERE resource_id IN (?,?)', ids)[0] == 4 @pytest.mark.asyncio -async def test_batch_second_revision_failure_rolls_back_pg(monkeypatch,installed): - app,_=installed - key,subject,_=await register(app,'batch-rollback') - created=[await call(app,'content.post_create',{'parent':'/main','body':body}, - key=key,subject=subject) for body in ('one X','two Y')] - ids=tuple(item.resources[0].id for item in created) - before=await snapshot(app,ids) - patches=[{'id':item.resources[0].id,'base_revision':item.resources[0].revision, - 'base_generation':item.data['generation'],'exact':needle,'replacement':'changed'} - for item,needle in zip(created,('X','Y'))] - original=app.contents.commit_revision - calls=0 - async def fail_second(topic,revision): +async def test_batch_second_revision_failure_rolls_back_pg(monkeypatch, installed): + app, _ = installed + key, subject, _ = await register(app, 'batch-rollback') + created = [ + await call( + app, 'content.post_create', {'parent': '/main', 'body': body}, key=key, subject=subject + ) + for body in ('one X', 'two Y') + ] + ids = tuple(item.resources[0].id for item in created) + before = await snapshot(app, ids) + patches = [ + { + 'id': item.resources[0].id, + 'base_revision': item.resources[0].revision, + 'base_generation': item.data['generation'], + 'exact': needle, + 'replacement': 'changed', + } + for item, needle in zip(created, ('X', 'Y'), strict=True) + ] + original = app.contents.commit_revision + calls = 0 + + async def fail_second(topic, revision): nonlocal calls - calls+=1 - if calls==2: + calls += 1 + if calls == 2: raise RuntimeError('injected second revision failure') - return await original(topic,revision) - monkeypatch.setattr(app.contents,'commit_revision',fail_second) - result=await call(app,'content.text_patch_batch',{'patches':patches},key=key,subject=subject, - expected=tuple((item.resources[0].id,item.data['generation']) for item in created)) - assert result.error.code=='internal_error',wire(result) - assert await snapshot(app,ids)==before - assert await body(app,ids[0])=='one X' and await body(app,ids[1])=='two Y' + return await original(topic, revision) + + monkeypatch.setattr(app.contents, 'commit_revision', fail_second) + result = await call( + app, + 'content.text_patch_batch', + {'patches': patches}, + key=key, + subject=subject, + expected=tuple((item.resources[0].id, item.data['generation']) for item in created), + ) + assert result.error.code == 'internal_error', wire(result) + assert await snapshot(app, ids) == before + assert await body(app, ids[0]) == 'one X' and await body(app, ids[1]) == 'two Y' diff --git a/tests/test_todo_due_inbox.py b/tests/test_todo_due_inbox.py index 784426f1..8df87e36 100644 --- a/tests/test_todo_due_inbox.py +++ b/tests/test_todo_due_inbox.py @@ -1,11 +1,12 @@ """Due reminders are explicit worker writes, private and idempotent.""" + from datetime import timedelta import pytest +from test_service import NOW, call, register from msg.core.codec import wire from msg.workers.maintenance import run_maintenance -from test_service import NOW, call, register @pytest.mark.asyncio @@ -13,37 +14,55 @@ async def test_due_todo_is_only_in_owner_inbox_and_repeated_maintenance_is_noop( app, _ = installed owner_key, owner, _ = await register(app, 'due-owner') other_key, other, _ = await register(app, 'due-other') - todo = await call(app, 'identity.todo_put', + todo = await call( + app, + 'identity.todo_put', {'name': 'due', 'title': 'Private due title', 'due_at': wire(NOW - timedelta(hours=1))}, - key=owner_key, subject=owner) + key=owner_key, + subject=owner, + ) assert todo.status == 'ok', wire(todo) rid = todo.resources[0].id async with app.metadata.transaction(write=False) as tx: before = tx.one('SELECT COUNT(*) FROM audit')[0] - assert tx.one("SELECT COUNT(*) FROM messages WHERE resource=?", (rid,))[0] == 0 + assert tx.one('SELECT COUNT(*) FROM messages WHERE resource=?', (rid,))[0] == 0 # Reads neither materialize the due reminder nor add an audit record. assert (await call(app, 'communication.inbox', {}, key=owner_key, subject=owner)).status == 'ok' - assert (await call(app, 'identity.todo_get', {'name': 'due'}, key=owner_key, subject=owner)).status == 'ok' + assert ( + await call(app, 'identity.todo_get', {'name': 'due'}, key=owner_key, subject=owner) + ).status == 'ok' async with app.metadata.transaction(write=False) as tx: assert tx.one('SELECT COUNT(*) FROM audit')[0] == before - assert tx.one("SELECT COUNT(*) FROM messages WHERE resource=?", (rid,))[0] == 0 + assert tx.one('SELECT COUNT(*) FROM messages WHERE resource=?', (rid,))[0] == 0 - assert (await run_maintenance(app, 'deliver_due_todos', scheduled=True)) == {'delivered_todo_reminders': 1} + assert (await run_maintenance(app, 'deliver_due_todos', scheduled=True)) == { + 'delivered_todo_reminders': 1 + } async with app.metadata.transaction(write=False) as tx: assert tx.one('SELECT COUNT(*) FROM audit')[0] == before + 1 - assert (await run_maintenance(app, 'deliver_due_todos', scheduled=True)) == {'delivered_todo_reminders': 0} + assert (await run_maintenance(app, 'deliver_due_todos', scheduled=True)) == { + 'delivered_todo_reminders': 0 + } async with app.metadata.transaction(write=False) as tx: assert tx.one('SELECT COUNT(*) FROM audit')[0] == before + 1 - assert tx.one("SELECT COUNT(*) FROM messages WHERE resource=?", (rid,))[0] == 1 + assert tx.one('SELECT COUNT(*) FROM messages WHERE resource=?', (rid,))[0] == 1 own = await call(app, 'communication.inbox', {}, key=owner_key, subject=owner) notices = [item for item in own.data['items'] if item['resource']['id'] == rid] assert len(notices) == 1 and notices[0]['source'] == 'todo_due' assert notices[0]['recipient'] == owner assert notices[0]['todo_revision_at_delivery'] == todo.resources[0].revision - assert not [item for item in (await call(app, 'communication.inbox', {}, key=other_key, - subject=other)).data['items'] if item['resource']['id'] == rid] - assert not (await call(app, 'discovery.search', {'query': 'Private due title'}, - key=other_key, subject=other)).data.get('results', []) + assert not [ + item + for item in (await call(app, 'communication.inbox', {}, key=other_key, subject=other)).data[ + 'items' + ] + if item['resource']['id'] == rid + ] + assert not ( + await call( + app, 'discovery.search', {'query': 'Private due title'}, key=other_key, subject=other + ) + ).data.get('results', []) @pytest.mark.asyncio @@ -52,33 +71,72 @@ async def test_only_current_pending_due_revision_can_be_delivered(installed): key, owner, _ = await register(app, 'due-revisions') past = wire(NOW - timedelta(hours=1)) future = wire(NOW + timedelta(days=2)) - original = await call(app, 'identity.todo_put', - {'name': 'rescheduled', 'title': 'Rescheduled', 'due_at': past}, key=key, subject=owner) + original = await call( + app, + 'identity.todo_put', + {'name': 'rescheduled', 'title': 'Rescheduled', 'due_at': past}, + key=key, + subject=owner, + ) rid = original.resources[0].id - revised = await call(app, 'identity.todo_put', - {'name': 'rescheduled', 'title': 'Rescheduled', 'due_at': future, - 'expected_revision': original.resources[0].revision}, key=key, subject=owner, - expected=((rid, original.data['generation']),)) + revised = await call( + app, + 'identity.todo_put', + { + 'name': 'rescheduled', + 'title': 'Rescheduled', + 'due_at': future, + 'expected_revision': original.resources[0].revision, + }, + key=key, + subject=owner, + expected=((rid, original.data['generation']),), + ) assert revised.status == 'ok', wire(revised) - completed = await call(app, 'identity.todo_put', + completed = await call( + app, + 'identity.todo_put', {'name': 'done', 'title': 'Completed', 'due_at': past, 'status': 'done'}, - key=key, subject=owner) - archived = await call(app, 'identity.todo_put', - {'name': 'archived', 'title': 'Archived', 'due_at': past}, key=key, subject=owner) + key=key, + subject=owner, + ) + archived = await call( + app, + 'identity.todo_put', + {'name': 'archived', 'title': 'Archived', 'due_at': past}, + key=key, + subject=owner, + ) archived_id = archived.resources[0].id - result = await call(app, 'identity.todo_archive', + result = await call( + app, + 'identity.todo_archive', {'name': 'archived', 'expected_revision': archived.resources[0].revision}, - key=key, subject=owner, expected=((archived_id, archived.data['generation']),)) + key=key, + subject=owner, + expected=((archived_id, archived.data['generation']),), + ) assert result.status == 'ok', wire(result) - assert (await run_maintenance(app, 'deliver_due_todos', scheduled=True)) == {'delivered_todo_reminders': 0} + assert (await run_maintenance(app, 'deliver_due_todos', scheduled=True)) == { + 'delivered_todo_reminders': 0 + } async with app.metadata.transaction(write=False) as tx: - assert tx.one("SELECT COUNT(*) FROM messages WHERE resource IN (?,?,?)", - (rid, completed.resources[0].id, archived_id))[0] == 0 + assert ( + tx.one( + 'SELECT COUNT(*) FROM messages WHERE resource IN (?,?,?)', + (rid, completed.resources[0].id, archived_id), + )[0] + == 0 + ) # A later due instant is a distinct reminder, delivered only once then. app.clock = lambda: NOW + timedelta(days=3) - assert (await run_maintenance(app, 'deliver_due_todos', scheduled=True)) == {'delivered_todo_reminders': 1} - assert (await run_maintenance(app, 'deliver_due_todos', scheduled=True)) == {'delivered_todo_reminders': 0} + assert (await run_maintenance(app, 'deliver_due_todos', scheduled=True)) == { + 'delivered_todo_reminders': 1 + } + assert (await run_maintenance(app, 'deliver_due_todos', scheduled=True)) == { + 'delivered_todo_reminders': 0 + } async with app.metadata.transaction(write=False) as tx: row = tx.one('SELECT body FROM messages WHERE resource=?', (rid,)) assert row is not None and future in row[0] and past not in row[0] diff --git a/tests/test_token_delivery.py b/tests/test_token_delivery.py index e78e9edf..5637d300 100644 --- a/tests/test_token_delivery.py +++ b/tests/test_token_delivery.py @@ -1,15 +1,16 @@ """Strict token delivery is opt-in at issuance version 2.""" + import asyncio import os from datetime import timedelta import pytest +from test_service import NOW, call, register, temporary_v3_args from msg.application import Application -from msg.core.codec import b64,unb64,wire -from msg.core.requests import receipt_bytes,request_for +from msg.core.codec import b64, unb64, wire +from msg.core.requests import receipt_bytes, request_for from msg.security.crypto import verify -from test_service import NOW,call,register,temporary_v3_args def secret(): @@ -18,149 +19,263 @@ def secret(): @pytest.mark.asyncio async def test_v2_bootstrap_claim_once_and_recover_lost_response(installed): - app,_=installed - original,_,_,_=temporary_v3_args(request_id='strict-bootstrap') - created=await call(app,'identity.temporary',original,rid='strict-bootstrap',contract_version=3) - assert created.status=='ok' and created.data['token'] - uid=created.data['subject_id'] - old_id=created.data['credential_id'] + app, _ = installed + original, _, _, _ = temporary_v3_args(request_id='strict-bootstrap') + created = await call( + app, 'identity.temporary', original, rid='strict-bootstrap', contract_version=3 + ) + assert created.status == 'ok' and created.data['token'] + uid = created.data['subject_id'] + old_id = created.data['credential_id'] async with app.metadata.transaction(write=False) as tx: - row=tx.one('SELECT recovery_verifier,claimed_at,consumed_at FROM token_deliveries WHERE credential_id=?',(old_id,)) + row = tx.one( + 'SELECT recovery_verifier,claimed_at,consumed_at FROM token_deliveries WHERE credential_id=?', + (old_id,), + ) assert row[1] and row[2] is None assert original['recovery_secret'] not in str(row) assert created.data['token'] not in str(row) - replay=await call(app,'identity.temporary',original,rid='strict-bootstrap',contract_version=3) - assert replay.error.code=='token_delivery_unavailable' - args={'credential_id':old_id,'original_request_id':'strict-bootstrap', - 'recovery_secret':original['recovery_secret'], - 'nonce':secret(),'new_recovery_secret':secret()} - wrong=await call(app,'identity.token_recover',{**args,'recovery_secret':secret()}, - subject=uid,rid='bad-recovery') - assert wrong.error.code=='recovery_unavailable' - recovered=await call(app,'identity.token_recover',args,subject=uid,rid='recover-once') - assert recovered.status=='ok' and recovered.data['token'],recovered.error.code - verify(app.receipt_signer.public_key,receipt_bytes(recovered),recovered.receipt,purpose='receipt') - new=(recovered.data['credential_id'],unb64(recovered.data['token'])) - old=(old_id,unb64(created.data['token'])) - assert new[0]!=old[0] - repeated=await call(app,'identity.token_recover',args,subject=uid,rid='recover-once') - assert repeated.error.code=='token_delivery_unavailable' - reuse=await call(app,'identity.token_recover',{**args,'nonce':secret()}, - subject=uid,rid='another-recovery') - assert reuse.error.code=='recovery_unavailable' - blocked=await call(app,'content.post_create',{'parent':'/main','body':'old'},subject=uid,token=old) - assert blocked.error.code=='credential_revoked' - allowed=await call(app,'content.post_create',{'parent':'/main','body':'new'},subject=uid,token=new) - assert allowed.status=='ok',wire(allowed) + replay = await call( + app, 'identity.temporary', original, rid='strict-bootstrap', contract_version=3 + ) + assert replay.error.code == 'token_delivery_unavailable' + args = { + 'credential_id': old_id, + 'original_request_id': 'strict-bootstrap', + 'recovery_secret': original['recovery_secret'], + 'nonce': secret(), + 'new_recovery_secret': secret(), + } + wrong = await call( + app, + 'identity.token_recover', + {**args, 'recovery_secret': secret()}, + subject=uid, + rid='bad-recovery', + ) + assert wrong.error.code == 'recovery_unavailable' + recovered = await call(app, 'identity.token_recover', args, subject=uid, rid='recover-once') + assert recovered.status == 'ok' and recovered.data['token'], recovered.error.code + verify( + app.receipt_signer.public_key, + receipt_bytes(recovered), + recovered.receipt, + purpose='receipt', + ) + new = (recovered.data['credential_id'], unb64(recovered.data['token'])) + old = (old_id, unb64(created.data['token'])) + assert new[0] != old[0] + repeated = await call(app, 'identity.token_recover', args, subject=uid, rid='recover-once') + assert repeated.error.code == 'token_delivery_unavailable' + reuse = await call( + app, + 'identity.token_recover', + {**args, 'nonce': secret()}, + subject=uid, + rid='another-recovery', + ) + assert reuse.error.code == 'recovery_unavailable' + blocked = await call( + app, 'content.post_create', {'parent': '/main', 'body': 'old'}, subject=uid, token=old + ) + assert blocked.error.code == 'credential_revoked' + allowed = await call( + app, 'content.post_create', {'parent': '/main', 'body': 'new'}, subject=uid, token=new + ) + assert allowed.status == 'ok', wire(allowed) @pytest.mark.asyncio async def test_v2_concurrent_claim_and_cross_subject_denial(installed): - app,_=installed - args={'handle':'delivery-race','nonce':secret(),'recovery_secret':secret()} - first,second=await asyncio.gather(*( - call(app,'identity.custodial_create',args,rid='race',contract_version=2) - for _ in range(2))) - assert sorted((first.status,second.status))==['error','ok'] - assert next(r for r in (first,second) if r.status=='error').error.code=='token_delivery_unavailable' - winner=next(r for r in (first,second) if r.status=='ok') - other_args,other_rid,_,_=temporary_v3_args() - other=await call(app,'identity.temporary',other_args, - rid=other_rid,contract_version=3) - denied=await call(app,'identity.token_recover',{ - 'credential_id':winner.data['credential_id'],'original_request_id':'race', - 'recovery_secret':args['recovery_secret'], - 'nonce':secret(),'new_recovery_secret':secret()},subject=other.data['subject_id']) - assert denied.error.code=='recovery_unavailable' + app, _ = installed + args = {'handle': 'delivery-race', 'nonce': secret(), 'recovery_secret': secret()} + first, second = await asyncio.gather( + *( + call(app, 'identity.custodial_create', args, rid='race', contract_version=2) + for _ in range(2) + ) + ) + assert sorted((first.status, second.status)) == ['error', 'ok'] + assert ( + next(r for r in (first, second) if r.status == 'error').error.code + == 'token_delivery_unavailable' + ) + winner = next(r for r in (first, second) if r.status == 'ok') + other_args, other_rid, _, _ = temporary_v3_args() + other = await call(app, 'identity.temporary', other_args, rid=other_rid, contract_version=3) + denied = await call( + app, + 'identity.token_recover', + { + 'credential_id': winner.data['credential_id'], + 'original_request_id': 'race', + 'recovery_secret': args['recovery_secret'], + 'nonce': secret(), + 'new_recovery_secret': secret(), + }, + subject=other.data['subject_id'], + ) + assert denied.error.code == 'recovery_unavailable' @pytest.mark.asyncio async def test_v2_unclaimed_commit_survives_restart_and_expiry(installed): - app,_=installed - args,_,_,_=temporary_v3_args(request_id='restart-before-claim') - original_hook=app.executor.response_hook - app.executor.response_hook=None # simulates commit followed by process loss before claim - committed=await call(app,'identity.temporary',args,rid='restart-before-claim',contract_version=3) - assert committed.status=='ok' and 'token' not in committed.data - second=Application(app.settings,clock=lambda:NOW) + app, _ = installed + args, _, _, _ = temporary_v3_args(request_id='restart-before-claim') + original_hook = app.executor.response_hook + app.executor.response_hook = None # simulates commit followed by process loss before claim + committed = await call( + app, 'identity.temporary', args, rid='restart-before-claim', contract_version=3 + ) + assert committed.status == 'ok' and 'token' not in committed.data + second = Application(app.settings, clock=lambda: NOW) await second.load() try: - claimed=await call(second,'identity.temporary',args,rid='restart-before-claim',contract_version=3) - assert claimed.status=='ok' and claimed.replayed and claimed.data['token'] - exhausted=await call(second,'identity.temporary',args,rid='restart-before-claim',contract_version=3) - assert exhausted.error.code=='token_delivery_unavailable' - uid=claimed.data['subject_id'] - recovery={'credential_id':claimed.data['credential_id'], - 'original_request_id':'restart-before-claim', - 'recovery_secret':args['recovery_secret'],'nonce':secret(), - 'new_recovery_secret':secret()} - second.clock=lambda:NOW+timedelta(minutes=16) - second.authenticator.clock=second.clock - packet=request_for('identity.token_recover',recovery,second.settings.service_url, - subject=uid,expires_at=NOW+timedelta(minutes=18)) - expired=await second.executor.execute(packet) - assert expired.error.code=='recovery_unavailable' + claimed = await call( + second, 'identity.temporary', args, rid='restart-before-claim', contract_version=3 + ) + assert claimed.status == 'ok' and claimed.replayed and claimed.data['token'] + exhausted = await call( + second, 'identity.temporary', args, rid='restart-before-claim', contract_version=3 + ) + assert exhausted.error.code == 'token_delivery_unavailable' + uid = claimed.data['subject_id'] + recovery = { + 'credential_id': claimed.data['credential_id'], + 'original_request_id': 'restart-before-claim', + 'recovery_secret': args['recovery_secret'], + 'nonce': secret(), + 'new_recovery_secret': secret(), + } + second.clock = lambda: NOW + timedelta(minutes=16) + second.authenticator.clock = second.clock + packet = request_for( + 'identity.token_recover', + recovery, + second.settings.service_url, + subject=uid, + expires_at=NOW + timedelta(minutes=18), + ) + expired = await second.executor.execute(packet) + assert expired.error.code == 'recovery_unavailable' finally: await second.close() - app.executor.response_hook=original_hook + app.executor.response_hook = original_hook @pytest.mark.asyncio async def test_v2_rotate_and_signed_create_bind_separate_recovery(installed): - app,_=installed - bootstrap_args,_,_,_=temporary_v3_args(request_id='rotate-bootstrap') - bootstrap=await call(app,'identity.temporary',bootstrap_args, - rid='rotate-bootstrap',contract_version=3) - uid=bootstrap.data['subject_id'] - token=(bootstrap.data['credential_id'],unb64(bootstrap.data['token'])) - rotate_args={'nonce':secret(),'recovery_secret':secret()} - rotated=await call(app,'identity.token_rotate',rotate_args,subject=uid,token=token, - rid='rotate-v2',contract_version=2) - assert rotated.status=='ok' and rotated.data['token'],rotated.error - replay=await call(app,'identity.token_rotate',rotate_args,subject=uid,token=token, - rid='rotate-v2',contract_version=2) - assert replay.error.code=='token_delivery_unavailable' - recovered=await call(app,'identity.token_recover',{ - 'credential_id':rotated.data['credential_id'], - 'original_request_id':'rotate-v2', - 'recovery_secret':rotate_args['recovery_secret'], - 'nonce':secret(),'new_recovery_secret':secret()},subject=uid) - assert recovered.status=='ok' and recovered.data['token'],recovered.error - - signer,signed_uid,_=await register(app,'signed-delivery') - created_args={'nonce':secret(),'recovery_secret':secret(),'ceiling':[],'ttl':900} - issued=await call(app,'identity.token_create',created_args,key=signer, - subject=signed_uid,contract_version=2) - assert issued.status=='ok' and issued.data['token'],issued.error - again=await call(app,'identity.token_create',created_args,key=signer, - subject=signed_uid,rid=issued.request_id,contract_version=2) - assert again.error.code=='token_delivery_unavailable' - restored=await call(app,'identity.token_recover',{ - 'credential_id':issued.data['credential_id'], - 'original_request_id':issued.request_id, - 'recovery_secret':created_args['recovery_secret'], - 'nonce':secret(),'new_recovery_secret':secret()},subject=signed_uid) - assert restored.status=='ok' and restored.data['token'],restored.error + app, _ = installed + bootstrap_args, _, _, _ = temporary_v3_args(request_id='rotate-bootstrap') + bootstrap = await call( + app, 'identity.temporary', bootstrap_args, rid='rotate-bootstrap', contract_version=3 + ) + uid = bootstrap.data['subject_id'] + token = (bootstrap.data['credential_id'], unb64(bootstrap.data['token'])) + rotate_args = {'nonce': secret(), 'recovery_secret': secret()} + rotated = await call( + app, + 'identity.token_rotate', + rotate_args, + subject=uid, + token=token, + rid='rotate-v2', + contract_version=2, + ) + assert rotated.status == 'ok' and rotated.data['token'], rotated.error + replay = await call( + app, + 'identity.token_rotate', + rotate_args, + subject=uid, + token=token, + rid='rotate-v2', + contract_version=2, + ) + assert replay.error.code == 'token_delivery_unavailable' + recovered = await call( + app, + 'identity.token_recover', + { + 'credential_id': rotated.data['credential_id'], + 'original_request_id': 'rotate-v2', + 'recovery_secret': rotate_args['recovery_secret'], + 'nonce': secret(), + 'new_recovery_secret': secret(), + }, + subject=uid, + ) + assert recovered.status == 'ok' and recovered.data['token'], recovered.error + + signer, signed_uid, _ = await register(app, 'signed-delivery') + created_args = {'nonce': secret(), 'recovery_secret': secret(), 'ceiling': [], 'ttl': 900} + issued = await call( + app, + 'identity.token_create', + created_args, + key=signer, + subject=signed_uid, + contract_version=2, + ) + assert issued.status == 'ok' and issued.data['token'], issued.error + again = await call( + app, + 'identity.token_create', + created_args, + key=signer, + subject=signed_uid, + rid=issued.request_id, + contract_version=2, + ) + assert again.error.code == 'token_delivery_unavailable' + restored = await call( + app, + 'identity.token_recover', + { + 'credential_id': issued.data['credential_id'], + 'original_request_id': issued.request_id, + 'recovery_secret': created_args['recovery_secret'], + 'nonce': secret(), + 'new_recovery_secret': secret(), + }, + subject=signed_uid, + ) + assert restored.status == 'ok' and restored.data['token'], restored.error @pytest.mark.asyncio async def test_v2_recovery_secret_is_consumed_once_under_race(installed): - app,_=installed - args,request_id,_,_=temporary_v3_args() - created=await call(app,'identity.temporary',args,rid=request_id,contract_version=3) - uid=created.data['subject_id'] - base={'credential_id':created.data['credential_id'], - 'original_request_id':created.request_id, - 'recovery_secret':args['recovery_secret']} - first,second=await asyncio.gather( - call(app,'identity.token_recover',{ - **base,'nonce':secret(),'new_recovery_secret':secret()}, - subject=uid,rid='recover-race-a'), - call(app,'identity.token_recover',{ - **base,'nonce':secret(),'new_recovery_secret':secret()}, - subject=uid,rid='recover-race-b')) - assert sorted((first.status,second.status))==['error','ok'] - assert next(r for r in (first,second) if r.status=='error').error.code=='recovery_unavailable' + app, _ = installed + args, request_id, _, _ = temporary_v3_args() + created = await call(app, 'identity.temporary', args, rid=request_id, contract_version=3) + uid = created.data['subject_id'] + base = { + 'credential_id': created.data['credential_id'], + 'original_request_id': created.request_id, + 'recovery_secret': args['recovery_secret'], + } + first, second = await asyncio.gather( + call( + app, + 'identity.token_recover', + {**base, 'nonce': secret(), 'new_recovery_secret': secret()}, + subject=uid, + rid='recover-race-a', + ), + call( + app, + 'identity.token_recover', + {**base, 'nonce': secret(), 'new_recovery_secret': secret()}, + subject=uid, + rid='recover-race-b', + ), + ) + assert sorted((first.status, second.status)) == ['error', 'ok'] + assert ( + next(r for r in (first, second) if r.status == 'error').error.code == 'recovery_unavailable' + ) async with app.metadata.transaction(write=False) as tx: - assert tx.one('SELECT consumed_at FROM token_deliveries WHERE credential_id=?', - (created.data['credential_id'],))[0] + assert tx.one( + 'SELECT consumed_at FROM token_deliveries WHERE credential_id=?', + (created.data['credential_id'],), + )[0] diff --git a/tests/test_token_delivery_boundary.py b/tests/test_token_delivery_boundary.py index e8e4743f..df333300 100644 --- a/tests/test_token_delivery_boundary.py +++ b/tests/test_token_delivery_boundary.py @@ -1,4 +1,5 @@ """Credential delivery has one owner, separate from service composition.""" + import ast import asyncio import hashlib @@ -6,17 +7,17 @@ import importlib import importlib.util import inspect -from pathlib import Path import subprocess import sys +from pathlib import Path import pytest +from test_service import NOW, temporary_v3_args from msg.core.codec import b64, canonical, loads, unb64, wire from msg.core.errors import Failure -from msg.core.requests import request_for, receipt_bytes +from msg.core.requests import receipt_bytes, request_for from msg.security.crypto import verify -from test_service import NOW, temporary_v3_args def delivery_module(): @@ -26,10 +27,14 @@ def delivery_module(): def test_composition_only_delegates_the_existing_delivery_api(): from msg.application import Application + module = delivery_module() tree = ast.parse(inspect.getsource(Application)) - methods = {node.name: node for node in tree.body[0].body - if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef))} + methods = { + node.name: node + for node in tree.body[0].body + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) + } for name in ('issued_token', 'record_token_delivery', '_secrets_for_caller'): method = methods[name] assert len(method.body) == 1 and isinstance(method.body[0], ast.Return) @@ -41,7 +46,11 @@ def test_composition_only_delegates_the_existing_delivery_api(): def test_domain_import_and_constructor_need_no_application_or_plugins(): module = delivery_module() assert set(inspect.signature(module.TokenDelivery).parameters) == { - 'metadata', 'token_secret', 'clock', 'recovery_window'} + 'metadata', + 'token_secret', + 'clock', + 'recovery_window', + } code = """ import importlib.abc import sys @@ -61,22 +70,39 @@ def find_spec(self, fullname, path=None, target=None): def test_derivation_and_recovery_verifier_keep_the_published_bytes(): module = delivery_module() - instance = module.TokenDelivery(metadata=None, token_secret=b'k'*32, - clock=lambda: NOW, recovery_window=lambda: 900) - request = request_for('identity.token_create', {'nonce': b64(b'n'*32)}, - 'https://example.invalid', request_id='stable-delivery', contract_version=2) - expected = hmac.digest(b'k'*32, b'issued-token-v1\0' + canonical({ - 'subject': 'u_subject', 'request_id': 'stable-delivery', 'nonce': b64(b'n'*32)}), 'sha256') + instance = module.TokenDelivery( + metadata=None, token_secret=b'k' * 32, clock=lambda: NOW, recovery_window=lambda: 900 + ) + request = request_for( + 'identity.token_create', + {'nonce': b64(b'n' * 32)}, + 'https://example.invalid', + request_id='stable-delivery', + contract_version=2, + ) + expected = hmac.digest( + b'k' * 32, + b'issued-token-v1\0' + + canonical({ + 'subject': 'u_subject', + 'request_id': 'stable-delivery', + 'nonce': b64(b'n' * 32), + }), + 'sha256', + ) assert instance.issued_token(request, 'u_subject') == expected assert instance.issued_token(request, 'u_other') != expected - assert module.recovery_verifier(b64(b'r'*32)) == hashlib.sha256( - b'token-recovery-v1\0' + b'r'*32).hexdigest() + assert ( + module.recovery_verifier(b64(b'r' * 32)) + == hashlib.sha256(b'token-recovery-v1\0' + b'r' * 32).hexdigest() + ) with pytest.raises(Failure, match='invalid_recovery_secret'): - module.recovery_verifier(b64(b'r'*31)) + module.recovery_verifier(b64(b'r' * 31)) def test_authentication_uses_the_same_verifier_owner(): from msg.security import authentication + module = delivery_module() assert authentication.recovery_verifier is module.recovery_verifier source = Path(authentication.__file__).read_text() @@ -95,16 +121,28 @@ async def test_standalone_delivery_claim_commits_once_without_persisting_secrets app, _ = installed module = delivery_module() args, rid, _, _ = temporary_v3_args(request_id='domain-claim-once') - request = request_for('identity.temporary', args, app.settings.service_url, - request_id=rid, contract_version=3, - expires_at=NOW.replace(minute=1)) + request = request_for( + 'identity.temporary', + args, + app.settings.service_url, + request_id=rid, + contract_version=3, + expires_at=NOW.replace(minute=1), + ) app.executor.response_hook = None committed = await app.executor.execute(request) assert committed.status == 'ok' and 'token' not in committed.data, wire(committed) - delivery = module.TokenDelivery(metadata=app.metadata, token_secret=app._token_secret, - clock=lambda: NOW, recovery_window=lambda: app.settings.credential_delivery_recovery_window) - answers = await asyncio.gather(delivery.release(request, committed), - delivery.release(request, committed), return_exceptions=True) + delivery = module.TokenDelivery( + metadata=app.metadata, + token_secret=app._token_secret, + clock=lambda: NOW, + recovery_window=lambda: app.settings.credential_delivery_recovery_window, + ) + answers = await asyncio.gather( + delivery.release(request, committed), + delivery.release(request, committed), + return_exceptions=True, + ) successes = [value for value in answers if not isinstance(value, BaseException)] failures = [value for value in answers if isinstance(value, Failure)] assert len(successes) == len(failures) == 1 @@ -112,13 +150,19 @@ async def test_standalone_delivery_claim_commits_once_without_persisting_secrets released = successes[0] assert unb64(released.data['token']) == app.issued_token(request, committed.subject) assert released.receipt == committed.receipt - verify(app.receipt_signer.public_key, receipt_bytes(released), released.receipt, purpose='receipt') + verify( + app.receipt_signer.public_key, receipt_bytes(released), released.receipt, purpose='receipt' + ) async with app.metadata.transaction(write=False) as tx: saved = await tx.request_result(committed.subject, rid, request.payload_digest) - assert saved.receipt == committed.receipt and receipt_bytes(saved) == receipt_bytes(committed) + assert saved.receipt == committed.receipt and receipt_bytes(saved) == receipt_bytes( + committed + ) assert saved.data == committed.data and 'token' not in saved.data - row = tx.one('SELECT claimed_at,recovery_verifier FROM token_deliveries WHERE credential_id=?', - (committed.data['credential_id'],)) + row = tx.one( + 'SELECT claimed_at,recovery_verifier FROM token_deliveries WHERE credential_id=?', + (committed.data['credential_id'],), + ) assert row[0] is not None and row[1] == module.recovery_verifier(args['recovery_secret']) assert released.data['token'] not in str(row) and args['recovery_secret'] not in str(row) # Event fields are stored in canonical JSON, not separate SQL columns. @@ -134,20 +178,36 @@ async def test_delivery_failure_returns_no_new_secret_or_claim(installed): app, _ = installed module = delivery_module() args, rid, _, _ = temporary_v3_args(request_id='domain-expired-claim') - request = request_for('identity.temporary', args, app.settings.service_url, - request_id=rid, contract_version=3, expires_at=NOW.replace(minute=1)) + request = request_for( + 'identity.temporary', + args, + app.settings.service_url, + request_id=rid, + contract_version=3, + expires_at=NOW.replace(minute=1), + ) app.executor.response_hook = None committed = await app.executor.execute(request) assert committed.status == 'ok', wire(committed) async with app.metadata.transaction(write=False) as tx: credential = await tx.credential(committed.data['credential_id']) - delivery = module.TokenDelivery(metadata=app.metadata, token_secret=app._token_secret, - clock=lambda: credential.expires_at, recovery_window=lambda: 900) + delivery = module.TokenDelivery( + metadata=app.metadata, + token_secret=app._token_secret, + clock=lambda: credential.expires_at, + recovery_window=lambda: 900, + ) with pytest.raises(Failure, match='credential_expired'): await delivery.release(request, committed) async with app.metadata.transaction(write=False) as tx: - assert tx.one('SELECT claimed_at FROM token_deliveries WHERE credential_id=?', - (credential.id,))[0] is None + assert ( + tx.one( + 'SELECT claimed_at FROM token_deliveries WHERE credential_id=?', (credential.id,) + )[0] + is None + ) saved = await tx.request_result(committed.subject, rid, request.payload_digest) - assert saved.receipt == committed.receipt and receipt_bytes(saved) == receipt_bytes(committed) + assert saved.receipt == committed.receipt and receipt_bytes(saved) == receipt_bytes( + committed + ) assert saved.data == committed.data and 'token' not in saved.data diff --git a/tests/test_tool_feature_selftest.py b/tests/test_tool_feature_selftest.py index 0bfdf162..3cec958e 100644 --- a/tests/test_tool_feature_selftest.py +++ b/tests/test_tool_feature_selftest.py @@ -10,14 +10,17 @@ @pytest.mark.asyncio async def test_doctor_exercises_real_bubblewrap_deny_path(tmp_path): - app = Application(write_example(tmp_path/'etc', tmp_path/'data')) + app = Application(write_example(tmp_path / 'etc', tmp_path / 'data')) assert await inspect_tool_sandbox(app) == { - 'runner': 'bubblewrap', 'policy_denial': True, 'external_requests': 0} + 'runner': 'bubblewrap', + 'policy_denial': True, + 'external_requests': 0, + } @pytest.mark.asyncio async def test_missing_sandbox_is_not_reported_ready(tmp_path, monkeypatch): - app = Application(write_example(tmp_path/'etc', tmp_path/'data')) + app = Application(write_example(tmp_path / 'etc', tmp_path / 'data')) monkeypatch.setattr('msg.workers.sandbox.shutil.which', lambda _: None) with pytest.raises(Failure, match='tool_isolation_unavailable'): await inspect_tool_sandbox(app) @@ -26,33 +29,46 @@ async def test_missing_sandbox_is_not_reported_ready(tmp_path, monkeypatch): @pytest.mark.asyncio async def test_tool_selftest_rejects_live_application(): from msg.admin.tool_check import check_tool_sandbox + with pytest.raises(Failure, match='selftest_namespace_required'): await check_tool_sandbox(SimpleNamespace(selftest_run_id=None), None, None, None, None) @pytest.mark.asyncio -async def test_tool_selftest_distinguishes_runner_output_from_operation_denial(installed, monkeypatch): +async def test_tool_selftest_distinguishes_runner_output_from_operation_denial( + installed, monkeypatch +): from test_authorization import approve from test_service import call, register from msg.admin.tool_check import check_tool_sandbox + app, root = installed monkeypatch.setattr(app, 'selftest_run_id', 'isolated-tool-fixture') key, subject, _ = await register(app, 'tool-selftest') + async def invoke(name, arguments, signer, uid, certificates): return await call(app, name, arguments, key=signer, subject=uid, certs=certificates) + async def certify(signer, uid, grants): return await approve(app, root, uid, signer, grants) + result = await check_tool_sandbox(app, invoke, certify, key, subject) - assert result == {'operation': 'private_target_denied', + assert result == { + 'operation': 'private_target_denied', 'runner': 'loopback_output_and_policy_denial', - 'normal_operation_pipeline': 'not_exercised', 'external_requests': 0} + 'normal_operation_pipeline': 'not_exercised', + 'external_requests': 0, + } def test_on_demand_feature_samples_are_explicitly_empty(): from msg.bootstrap import feature_manifest + rows = {row['feature_id']: row for row in feature_manifest()} for name in ('content_editing', 'private_dm', 'transfer', 'following', 'collaboration'): assert rows[name]['sample_resource'] is None assert rows[name]['default_config'] == { - 'sample_state': 'empty', 'sample_creation': 'isolated_selftest'} + 'sample_state': 'empty', + 'sample_creation': 'isolated_selftest', + } diff --git a/tests/test_tool_runner_contract.py b/tests/test_tool_runner_contract.py index c1d915a0..6f2f1196 100644 --- a/tests/test_tool_runner_contract.py +++ b/tests/test_tool_runner_contract.py @@ -1,20 +1,21 @@ """The runner owns local output; only the worker may publish a ResourceRef.""" + from __future__ import annotations import ast -from dataclasses import FrozenInstanceError, fields import inspect +from dataclasses import FrozenInstanceError, fields from pathlib import Path from typing import get_type_hints import pytest +from test_authorization import approve, scoped +from test_service import call, register from msg.core.errors import Failure from msg.core.models import JsonMap, NetworkPolicy, ResourceRef, ToolSpec from msg.workers.effects import EffectWorker from msg.workers.sandbox import BubblewrapRunner -from test_service import call, register -from test_authorization import approve, scoped def test_tool_result_has_one_neutral_owner_and_compatible_export(tmp_path): @@ -41,8 +42,13 @@ def test_actual_worker_dependency_is_the_declared_callable_port(): def test_default_runner_and_port_have_the_same_input_and_local_output_types(): from msg.core.tool_execution import ToolResult, ToolRunner - expected = {'tool': ToolSpec, 'arguments': JsonMap, - 'policies': tuple[NetworkPolicy, ...], 'directory': Path, 'return': ToolResult} + expected = { + 'tool': ToolSpec, + 'arguments': JsonMap, + 'policies': tuple[NetworkPolicy, ...], + 'directory': Path, + 'return': ToolResult, + } assert get_type_hints(ToolRunner.__call__) == expected assert get_type_hints(BubblewrapRunner.__call__) == expected assert inspect.iscoroutinefunction(BubblewrapRunner.__call__) @@ -69,8 +75,12 @@ def test_sandbox_does_not_import_its_worker_or_application_composition(): elif isinstance(node, ast.Import): dependencies.extend(alias.name for alias in node.names) assert 'msg.core.tool_execution' in dependencies - assert not any(name == 'msg.workers.effects' or name.startswith('msg.workers.effects.') - or name == 'msg.application' for name in dependencies) + assert not any( + name == 'msg.workers.effects' + or name.startswith('msg.workers.effects.') + or name == 'msg.application' + for name in dependencies + ) async def test_default_runner_still_refuses_missing_mandatory_sandbox(monkeypatch, tmp_path): @@ -91,14 +101,20 @@ async def test_injected_port_cannot_publish_a_resource_itself(installed, invalid cap = scoped(app, 'tool.use', 'tool_dns', app.registry.capability('tool.use').operations) certificate = await approve(app, root, subject, key, (cap,)) certs = (certificate.resource_id,) - request = await call(app, 'tool.run', + request = await call( + app, + 'tool.run', {'id': '/tools/dns', 'arguments': {'name': 'example.org', 'type': 'A'}}, - key=key, subject=subject, certs=certs) + key=key, + subject=subject, + certs=certs, + ) assert request.status == 'accepted' invocations = [] - async def runner(tool: ToolSpec, arguments: JsonMap, - policies: tuple[NetworkPolicy, ...], directory: Path) -> ToolResult: + async def runner( + tool: ToolSpec, arguments: JsonMap, policies: tuple[NetworkPolicy, ...], directory: Path + ) -> ToolResult: assert tool.executor_key == 'dns' assert arguments['name'] == 'example.org' assert isinstance(policies, tuple) and policies diff --git a/tests/test_tool_sandbox_acceptance.py b/tests/test_tool_sandbox_acceptance.py index adf4e622..27cf10de 100644 --- a/tests/test_tool_sandbox_acceptance.py +++ b/tests/test_tool_sandbox_acceptance.py @@ -1,4 +1,5 @@ """Real bubblewrap child enforces network policy without contacting a peer.""" + from types import SimpleNamespace import pytest @@ -9,16 +10,35 @@ @pytest.mark.asyncio -@pytest.mark.parametrize('url', ['http://127.0.0.1:9/', 'http://169.254.169.254/', - 'file:///etc/passwd', 'http://[::1]:9/']) +@pytest.mark.parametrize( + 'url', + ['http://127.0.0.1:9/', 'http://169.254.169.254/', 'file:///etc/passwd', 'http://[::1]:9/'], +) async def test_real_sandbox_denies_private_or_non_http_target(tmp_path, url): - app = SimpleNamespace(settings=SimpleNamespace(server=SimpleNamespace( - limits=SimpleNamespace(max_request_bytes=1024)))) + app = SimpleNamespace( + settings=SimpleNamespace( + server=SimpleNamespace(limits=SimpleNamespace(max_request_bytes=1024)) + ) + ) tool = SimpleNamespace(executor_key='curl') with pytest.raises(Failure) as caught: - await BubblewrapRunner(app)(tool, {'url': url}, (NetworkPolicy(schemes=frozenset({'http', 'https'}), hosts=(), - ports=frozenset({9, 80, 443}), methods=frozenset({'GET'}), allow_private=False, - timeout_ms=1000, max_response_bytes=1024, max_redirects=0),), tmp_path) + await BubblewrapRunner(app)( + tool, + {'url': url}, + ( + NetworkPolicy( + schemes=frozenset({'http', 'https'}), + hosts=(), + ports=frozenset({9, 80, 443}), + methods=frozenset({'GET'}), + allow_private=False, + timeout_ms=1000, + max_response_bytes=1024, + max_redirects=0, + ), + ), + tmp_path, + ) assert caught.value.code == 'network_policy_denied' assert not (tmp_path / 'output.bin').exists() @@ -32,8 +52,11 @@ async def test_real_sandbox_explicit_local_grant_returns_bounded_bytes(tmp_path, async def receive(reader, writer): requests.append(await reader.readuntil(b'\r\n\r\n')) - writer.write(f'HTTP/1.1 200 OK\r\nContent-Length: {len(body)}\r\n'.encode() + - b'Content-Type: text/plain\r\nConnection: close\r\n\r\n' + body) + writer.write( + f'HTTP/1.1 200 OK\r\nContent-Length: {len(body)}\r\n'.encode() + + b'Content-Type: text/plain\r\nConnection: close\r\n\r\n' + + body + ) await writer.drain() writer.close() await writer.wait_closed() @@ -41,13 +64,27 @@ async def receive(reader, writer): server = await asyncio.start_server(receive, '127.0.0.1', 0) async with server: port = server.sockets[0].getsockname()[1] - app = SimpleNamespace(settings=SimpleNamespace(server=SimpleNamespace( - limits=SimpleNamespace(max_request_bytes=1024)))) - policy = NetworkPolicy(schemes=frozenset({'http'}), hosts=('127.0.0.1',), - ports=frozenset({port}), methods=frozenset({'GET'}), allow_private=True, - timeout_ms=3000, max_response_bytes=4, max_redirects=0) - invocation = BubblewrapRunner(app)(SimpleNamespace(executor_key='curl'), - {'url': f'http://127.0.0.1:{port}/fixture'}, (policy,), tmp_path) + app = SimpleNamespace( + settings=SimpleNamespace( + server=SimpleNamespace(limits=SimpleNamespace(max_request_bytes=1024)) + ) + ) + policy = NetworkPolicy( + schemes=frozenset({'http'}), + hosts=('127.0.0.1',), + ports=frozenset({port}), + methods=frozenset({'GET'}), + allow_private=True, + timeout_ms=3000, + max_response_bytes=4, + max_redirects=0, + ) + invocation = BubblewrapRunner(app)( + SimpleNamespace(executor_key='curl'), + {'url': f'http://127.0.0.1:{port}/fixture'}, + (policy,), + tmp_path, + ) if len(body) > policy.max_response_bytes: with pytest.raises(Failure, match='tool_response_too_large'): await invocation diff --git a/tests/test_tools.py b/tests/test_tools.py index 25ea8ec7..fd47470c 100644 --- a/tests/test_tools.py +++ b/tests/test_tools.py @@ -1,272 +1,449 @@ from dataclasses import replace from datetime import timedelta -from pathlib import Path -import pytest + import httpx +import pytest +from test_authorization import approve, scoped +from test_service import NOW, call, register from msg.application import Application -from msg.core.codec import wire,canonical,digest,loads +from msg.core.codec import canonical, digest, loads, wire from msg.core.errors import Failure -from msg.core.models import Scope,NetworkPolicy,ResourceRef,Revision -from msg.plugins.common import new_id -from msg.security.network import validate_url,validate_addresses,intersect_policy -from msg.workers.effects import EffectWorker,ToolResult +from msg.core.models import NetworkPolicy, ResourceRef, Revision from msg.extensions.tools import descriptor, read_tool +from msg.plugins.common import new_id +from msg.security.network import intersect_policy, validate_addresses, validate_url from msg.transports.dictionary import build_dictionary from msg.transports.http import create_app -from test_service import register,call,NOW -from test_authorization import approve,scoped +from msg.workers.effects import EffectWorker, ToolResult def policy(**kw): - return NetworkPolicy(schemes=frozenset({'http','https'}),hosts=(),ports=frozenset({80,443}), - methods=frozenset({'GET','HEAD'}),allow_private=kw.get('allow_private',False),timeout_ms=1000, - max_response_bytes=10000,max_redirects=3) - - -@pytest.mark.parametrize('url',['file:///etc/passwd','http://user:pass@example.org','http://example.org\\@127.0.0.1', - 'http://example.org:22/','unix:///var/run/socket','http://example.org/\nX:test']) + return NetworkPolicy( + schemes=frozenset({'http', 'https'}), + hosts=(), + ports=frozenset({80, 443}), + methods=frozenset({'GET', 'HEAD'}), + allow_private=kw.get('allow_private', False), + timeout_ms=1000, + max_response_bytes=10000, + max_redirects=3, + ) + + +@pytest.mark.parametrize( + 'url', + [ + 'file:///etc/passwd', + 'http://user:pass@example.org', + 'http://example.org\\@127.0.0.1', + 'http://example.org:22/', + 'unix:///var/run/socket', + 'http://example.org/\nX:test', + ], +) def test_network_tool_rejects_non_http_or_ambiguous_targets(url): - with pytest.raises(Failure):validate_url(url,'GET',policy()) - - -@pytest.mark.parametrize('address',['127.0.0.1','10.2.3.4','169.254.169.254','::1','::ffff:127.0.0.1','fe80::1','0.0.0.0','100.64.0.1']) + with pytest.raises(Failure): + validate_url(url, 'GET', policy()) + + +@pytest.mark.parametrize( + 'address', + [ + '127.0.0.1', + '10.2.3.4', + '169.254.169.254', + '::1', + '::ffff:127.0.0.1', + 'fe80::1', + '0.0.0.0', + '100.64.0.1', + ], +) def test_public_tool_rejects_nonpublic_addresses(address): - with pytest.raises(Failure,match='private_target_forbidden'):validate_addresses([address],policy()) - if address=='0.0.0.0': - with pytest.raises(Failure,match='invalid_network_target'):validate_addresses([address],policy(allow_private=True)) + with pytest.raises(Failure, match='private_target_forbidden'): + validate_addresses([address], policy()) + if address == '0.0.0.0': + with pytest.raises(Failure, match='invalid_network_target'): + validate_addresses([address], policy(allow_private=True)) else: - assert validate_addresses([address],policy(allow_private=True))==[address] + assert validate_addresses([address], policy(allow_private=True)) == [address] def test_tool_policy_intersection_does_not_widen_allowlists_or_limits(): - restricted=intersect_policy(policy(),{'hosts':['example.org'],'methods':['GET'],'timeout_ms':500}) - assert validate_url('https://example.org/path','GET',restricted).hostname=='example.org' - assert restricted.timeout_ms==500 - with pytest.raises(Failure):validate_url('https://other.example','GET',restricted) - with pytest.raises(Failure):validate_url('https://example.org','HEAD',restricted) - with pytest.raises(Failure):intersect_policy(policy(),{'ports':[22]}) + restricted = intersect_policy( + policy(), {'hosts': ['example.org'], 'methods': ['GET'], 'timeout_ms': 500} + ) + assert validate_url('https://example.org/path', 'GET', restricted).hostname == 'example.org' + assert restricted.timeout_ms == 500 + with pytest.raises(Failure): + validate_url('https://other.example', 'GET', restricted) + with pytest.raises(Failure): + validate_url('https://example.org', 'HEAD', restricted) + with pytest.raises(Failure): + intersect_policy(policy(), {'ports': [22]}) @pytest.mark.asyncio -async def test_only_scoped_tool_is_visible_jobs_are_deduped_and_output_transfers(installed,tmp_path): - app,root=installed - key,uid,base=await register(app,'tool-agent') - denied=await call(app,'tool.run',{'id':'/tools/dns','arguments':{'name':'example.org','type':'A'}},key=key,subject=uid) - assert denied.error.code=='tool_certificate_required',wire(denied) - cap=scoped(app,'tool.use','tool_dns',app.registry.capability('tool.use').operations) - cert=await approve(app,root,uid,key,(cap,)) - tools=await call(app,'discovery.get',{'id':'/tools'},key=key,subject=uid,certs=(cert.resource_id,)) - assert [t['name'] for t in tools.data['items']]==['dns'] - wrong=await call(app,'tool.run',{'id':'/tools/curl','arguments':{'url':'https://example.org'}},key=key,subject=uid,certs=(cert.resource_id,)) - assert wrong.error.code=='tool_certificate_required' - args={'id':'/tools/dns','arguments':{'name':'example.org','type':'A'}} - first=await call(app,'tool.run',args,key=key,subject=uid,certs=(cert.resource_id,),rid='tool-once') - second=await call(app,'tool.run',args,key=key,subject=uid,certs=(cert.resource_id,),rid='tool-once') - assert first.status=='accepted' and second.replayed,wire(first) - assert first.data['job_id']==second.data['job_id'] - executions=[] - async def runner(tool,args,policy,outdir): +async def test_only_scoped_tool_is_visible_jobs_are_deduped_and_output_transfers( + installed, tmp_path +): + app, root = installed + key, uid, base = await register(app, 'tool-agent') + denied = await call( + app, + 'tool.run', + {'id': '/tools/dns', 'arguments': {'name': 'example.org', 'type': 'A'}}, + key=key, + subject=uid, + ) + assert denied.error.code == 'tool_certificate_required', wire(denied) + cap = scoped(app, 'tool.use', 'tool_dns', app.registry.capability('tool.use').operations) + cert = await approve(app, root, uid, key, (cap,)) + tools = await call( + app, 'discovery.get', {'id': '/tools'}, key=key, subject=uid, certs=(cert.resource_id,) + ) + assert [t['name'] for t in tools.data['items']] == ['dns'] + wrong = await call( + app, + 'tool.run', + {'id': '/tools/curl', 'arguments': {'url': 'https://example.org'}}, + key=key, + subject=uid, + certs=(cert.resource_id,), + ) + assert wrong.error.code == 'tool_certificate_required' + args = {'id': '/tools/dns', 'arguments': {'name': 'example.org', 'type': 'A'}} + first = await call( + app, 'tool.run', args, key=key, subject=uid, certs=(cert.resource_id,), rid='tool-once' + ) + second = await call( + app, 'tool.run', args, key=key, subject=uid, certs=(cert.resource_id,), rid='tool-once' + ) + assert first.status == 'accepted' and second.replayed, wire(first) + assert first.data['job_id'] == second.data['job_id'] + executions = [] + + async def runner(tool, args, policy, outdir): executions.append(tool.executor_key) - path=outdir/'output.bin';path.write_bytes(canonical({'answers':['93.184.216.34']})) - return ToolResult(path=path,media_type='application/json',metadata={'kind':'dns'}) - worker=EffectWorker(app,tool_runner=runner) + path = outdir / 'output.bin' + path.write_bytes(canonical({'answers': ['93.184.216.34']})) + return ToolResult(path=path, media_type='application/json', metadata={'kind': 'dns'}) + + worker = EffectWorker(app, tool_runner=runner) assert await worker.run_once() assert not await worker.run_once() - assert executions==['dns'] - state=await call(app,'job.get',{'id':first.data['job_id']},key=key,subject=uid,certs=(cert.resource_id,)) - assert state.data['state']=='done',wire(state) - ref=state.data['output'] - download=await call(app,'transfer.open',{'direction':'download','target':ref},key=key,subject=uid,certs=(cert.resource_id,)) - assert download.status=='ok',wire(download) + assert executions == ['dns'] + state = await call( + app, + 'job.get', + {'id': first.data['job_id']}, + key=key, + subject=uid, + certs=(cert.resource_id,), + ) + assert state.data['state'] == 'done', wire(state) + ref = state.data['output'] + download = await call( + app, + 'transfer.open', + {'direction': 'download', 'target': ref}, + key=key, + subject=uid, + certs=(cert.resource_id,), + ) + assert download.status == 'ok', wire(download) @pytest.mark.asyncio -async def test_pending_job_rechecks_revocation_and_expired_lease_is_uncertain(installed,tmp_path): - app,root=installed - key,uid,base=await register(app,'worker-agent') - cert=await approve(app,root,uid,key,(scoped(app,'tool.use','tool_dns',app.registry.capability('tool.use').operations),)) - result=await call(app,'tool.run',{'id':'/tools/dns','arguments':{'name':'example.org','type':'A'}},key=key,subject=uid,certs=(cert.resource_id,)) +async def test_pending_job_rechecks_revocation_and_expired_lease_is_uncertain(installed, tmp_path): + app, root = installed + key, uid, base = await register(app, 'worker-agent') + cert = await approve( + app, + root, + uid, + key, + (scoped(app, 'tool.use', 'tool_dns', app.registry.capability('tool.use').operations),), + ) + result = await call( + app, + 'tool.run', + {'id': '/tools/dns', 'arguments': {'name': 'example.org', 'type': 'A'}}, + key=key, + subject=uid, + certs=(cert.resource_id,), + ) async with app.metadata.transaction(write=True) as tx: - tx.execute('UPDATE certificates SET revoked=1 WHERE id=?',(cert.resource_id,),write=True) - ran=[] - async def runner(*args):ran.append(True) - worker=EffectWorker(app,tool_runner=runner) + tx.execute('UPDATE certificates SET revoked=1 WHERE id=?', (cert.resource_id,), write=True) + ran = [] + + async def runner(*args): + ran.append(True) + + worker = EffectWorker(app, tool_runner=runner) await worker.run_once() assert not ran async with app.metadata.transaction(write=False) as tx: - job=await tx.job(result.data['job_id']) - assert job.state=='failed' - cert2=await approve(app,root,uid,key,(scoped(app,'tool.use','tool_dns',app.registry.capability('tool.use').operations),)) - job_result=await call(app,'tool.run',{'id':'/tools/dns','arguments':{'name':'example.org','type':'A'}},key=key,subject=uid,certs=(cert2.resource_id,)) + job = await tx.job(result.data['job_id']) + assert job.state == 'failed' + cert2 = await approve( + app, + root, + uid, + key, + (scoped(app, 'tool.use', 'tool_dns', app.registry.capability('tool.use').operations),), + ) + job_result = await call( + app, + 'tool.run', + {'id': '/tools/dns', 'arguments': {'name': 'example.org', 'type': 'A'}}, + key=key, + subject=uid, + certs=(cert2.resource_id,), + ) async with app.metadata.transaction(write=True) as tx: - job=await tx.job(job_result.data['job_id']) - await tx.save_job(replace(job,state='running',lease_until=NOW-timedelta(seconds=1))) + job = await tx.job(job_result.data['job_id']) + await tx.save_job(replace(job, state='running', lease_until=NOW - timedelta(seconds=1))) await worker.run_once() async with app.metadata.transaction(write=False) as tx: - assert (await tx.job(job.id)).state=='uncertain' + assert (await tx.job(job.id)).state == 'uncertain' assert not ran + @pytest.mark.asyncio async def test_tool_external_success_then_revocation_records_uncertain(installed): - app,root=installed - key,uid,_=await register(app,'late-revocation') - cert=await approve(app,root,uid,key,(scoped(app,'tool.use','tool_dns',app.registry.capability('tool.use').operations),)) - result=await call(app,'tool.run',{'id':'/tools/dns','arguments':{'name':'example.org','type':'A'}},key=key,subject=uid,certs=(cert.resource_id,)) - async def runner(tool,args,policies,outdir): + app, root = installed + key, uid, _ = await register(app, 'late-revocation') + cert = await approve( + app, + root, + uid, + key, + (scoped(app, 'tool.use', 'tool_dns', app.registry.capability('tool.use').operations),), + ) + result = await call( + app, + 'tool.run', + {'id': '/tools/dns', 'arguments': {'name': 'example.org', 'type': 'A'}}, + key=key, + subject=uid, + certs=(cert.resource_id,), + ) + + async def runner(tool, args, policies, outdir): async with app.metadata.transaction(write=True) as tx: - tx.execute('UPDATE certificates SET revoked=1 WHERE id=?',(cert.resource_id,),write=True) - path=outdir/'result.bin';path.write_bytes(b'[]') - return ToolResult(path=path,media_type='application/json',metadata={}) - await EffectWorker(app,tool_runner=runner).run_once() + tx.execute( + 'UPDATE certificates SET revoked=1 WHERE id=?', (cert.resource_id,), write=True + ) + path = outdir / 'result.bin' + path.write_bytes(b'[]') + return ToolResult(path=path, media_type='application/json', metadata={}) + + await EffectWorker(app, tool_runner=runner).run_once() async with app.metadata.transaction(write=False) as tx: - assert (await tx.job(result.data['job_id'])).state=='uncertain' - - -async def _claimed_dns_job(app,root,handle): - key,uid,_=await register(app,handle) - cert=await approve(app,root,uid,key,(scoped(app,'tool.use','tool_dns',app.registry.capability('tool.use').operations),)) - result=await call(app,'tool.run',{'id':'/tools/dns','arguments':{'name':'example.org','type':'A'}}, - key=key,subject=uid,certs=(cert.resource_id,)) - assert result.status=='accepted',wire(result) + assert (await tx.job(result.data['job_id'])).state == 'uncertain' + + +async def _claimed_dns_job(app, root, handle): + key, uid, _ = await register(app, handle) + cert = await approve( + app, + root, + uid, + key, + (scoped(app, 'tool.use', 'tool_dns', app.registry.capability('tool.use').operations),), + ) + result = await call( + app, + 'tool.run', + {'id': '/tools/dns', 'arguments': {'name': 'example.org', 'type': 'A'}}, + key=key, + subject=uid, + certs=(cert.resource_id,), + ) + assert result.status == 'accepted', wire(result) async with app.metadata.transaction(write=False) as tx: return await tx.job(result.data['job_id']) -async def _enqueue_like(app,template,seconds_ago,**changes): - job_id=new_id('job') - job=replace(template,id=job_id,dedupe_key='test:'+job_id,state='pending',attempts=0,lease_until=None, - next_attempt_at=NOW-timedelta(seconds=seconds_ago),**changes) +async def _enqueue_like(app, template, seconds_ago, **changes): + job_id = new_id('job') + job = replace( + template, + id=job_id, + dedupe_key='test:' + job_id, + state='pending', + attempts=0, + lease_until=None, + next_attempt_at=NOW - timedelta(seconds=seconds_ago), + **changes, + ) async with app.metadata.transaction(write=True) as tx: await tx.enqueue(job) return job -async def _tool_ref(app,rid): +async def _tool_ref(app, rid): async with app.metadata.transaction(write=False) as tx: - return wire(ResourceRef(id=rid,revision=(await tx.resource(rid)).revision)) + return wire(ResourceRef(id=rid, revision=(await tx.resource(rid)).revision)) -async def _dns_revision_declaring(app,concurrency): +async def _dns_revision_declaring(app, concurrency): async with app.metadata.transaction(write=True) as tx: - resource=await tx.resource('tool_dns') - blob=await app.contents.put_bytes(canonical({**descriptor('dns'),'concurrency':concurrency}),'application/json') - revision=Revision(format_version=1,id=new_id('v'),resource_id=resource.id,parents=(resource.revision,), - content=blob,relations=(),actor=resource.owner,subject=resource.owner,author=resource.owner, - created_at=app.clock(),manifest_digest='') - revision=replace(revision,manifest_digest=digest({k:v for k,v in wire(revision).items() - if k not in {'manifest_digest','signature'}})) - await app.contents.pin(blob,revision.id) - await app.contents.commit_revision(resource.parent,revision) + resource = await tx.resource('tool_dns') + blob = await app.contents.put_bytes( + canonical({**descriptor('dns'), 'concurrency': concurrency}), 'application/json' + ) + revision = Revision( + format_version=1, + id=new_id('v'), + resource_id=resource.id, + parents=(resource.revision,), + content=blob, + relations=(), + actor=resource.owner, + subject=resource.owner, + author=resource.owner, + created_at=app.clock(), + manifest_digest='', + ) + revision = replace( + revision, + manifest_digest=digest({ + k: v for k, v in wire(revision).items() if k not in {'manifest_digest', 'signature'} + }), + ) + await app.contents.pin(blob, revision.id) + await app.contents.commit_revision(resource.parent, revision) await tx.append_revision(revision) - return wire(ResourceRef(id=resource.id,revision=revision.id)) + return wire(ResourceRef(id=resource.id, revision=revision.id)) -async def _state(app,job_id): +async def _state(app, job_id): async with app.metadata.transaction(write=False) as tx: return (await tx.job(job_id)).state @pytest.mark.asyncio async def test_historical_tool_revision_bytes_have_no_concurrency_and_mean_one(installed): - app,_=installed + app, _ = installed assert 'concurrency' not in descriptor('dns') async with app.metadata.transaction(write=False) as tx: - rev=await tx.revision(ResourceRef(id='tool_dns')) - stored=await app.contents.read_bytes(rev.content,limit=65536) - assert stored==canonical(descriptor('dns')) + rev = await tx.revision(ResourceRef(id='tool_dns')) + stored = await app.contents.read_bytes(rev.content, limit=65536) + assert stored == canonical(descriptor('dns')) assert 'concurrency' not in loads(stored) @pytest.mark.asyncio async def test_same_tool_is_not_claimed_by_a_second_worker_while_other_jobs_proceed(installed): - app,root=installed - held=await _claimed_dns_job(app,root,'concurrency-holder') - first=EffectWorker(app,tool_runner=lambda *a:None) - claimed,execute=await first._claim() - assert execute and claimed.id==held.id - same_tool=await _enqueue_like(app,held,3) - other_tool=await _enqueue_like(app,held,2,arguments={**held.arguments,'tool':await _tool_ref(app,'tool_curl')}) - non_tool=await _enqueue_like(app,held,1,kind='mail',operation='communication.send', - arguments={'recipient_subject':held.principal.subject}) - peer=await Application(app.settings,clock=lambda:NOW).load() + app, root = installed + held = await _claimed_dns_job(app, root, 'concurrency-holder') + first = EffectWorker(app, tool_runner=lambda *a: None) + claimed, execute = await first._claim() + assert execute and claimed.id == held.id + same_tool = await _enqueue_like(app, held, 3) + other_tool = await _enqueue_like( + app, held, 2, arguments={**held.arguments, 'tool': await _tool_ref(app, 'tool_curl')} + ) + non_tool = await _enqueue_like( + app, + held, + 1, + kind='mail', + operation='communication.send', + arguments={'recipient_subject': held.principal.subject}, + ) + peer = await Application(app.settings, clock=lambda: NOW).load() try: - second=EffectWorker(peer,tool_runner=lambda *a:None) - order=[] + second = EffectWorker(peer, tool_runner=lambda *a: None) + order = [] for _ in range(2): - job,execute=await second._claim() + job, execute = await second._claim() assert execute order.append(job.id) - assert order==[other_tool.id,non_tool.id] - assert await second._claim()==(None,False) - assert await _state(app,same_tool.id)=='pending' - await first._finish(claimed,'done','ok') - job,execute=await second._claim() - assert execute and job.id==same_tool.id + assert order == [other_tool.id, non_tool.id] + assert await second._claim() == (None, False) + assert await _state(app, same_tool.id) == 'pending' + await first._finish(claimed, 'done', 'ok') + job, execute = await second._claim() + assert execute and job.id == same_tool.id finally: await peer.close() @pytest.mark.asyncio async def test_explicit_tool_concurrency_admits_up_to_the_declared_limit(installed): - app,root=installed - template=await _claimed_dns_job(app,root,'concurrency-two') - ref=await _dns_revision_declaring(app,2) + app, root = installed + template = await _claimed_dns_job(app, root, 'concurrency-two') + ref = await _dns_revision_declaring(app, 2) async with app.metadata.transaction(write=False) as tx: - assert (await read_tool(app,tx,'tool_dns',ref['revision'])).resource.revision==ref['revision'] + assert (await read_tool(app, tx, 'tool_dns', ref['revision'])).resource.revision == ref[ + 'revision' + ] async with app.metadata.transaction(write=True) as tx: - await tx.save_job(replace(template,state='done')) - jobs=[await _enqueue_like(app,template,3-i,arguments={**template.arguments,'tool':ref}) for i in range(3)] - peer=await Application(app.settings,clock=lambda:NOW).load() + await tx.save_job(replace(template, state='done')) + jobs = [ + await _enqueue_like(app, template, 3 - i, arguments={**template.arguments, 'tool': ref}) + for i in range(3) + ] + peer = await Application(app.settings, clock=lambda: NOW).load() try: - workers=(EffectWorker(app,tool_runner=lambda *a:None),EffectWorker(peer,tool_runner=lambda *a:None)) - for worker,job in zip(workers,jobs): - claimed,execute=await worker._claim() - assert execute and claimed.id==job.id + workers = ( + EffectWorker(app, tool_runner=lambda *a: None), + EffectWorker(peer, tool_runner=lambda *a: None), + ) + for worker, job in zip(workers, jobs, strict=False): + claimed, execute = await worker._claim() + assert execute and claimed.id == job.id for worker in workers: - assert await worker._claim()==(None,False) - assert await _state(app,jobs[2].id)=='pending' + assert await worker._claim() == (None, False) + assert await _state(app, jobs[2].id) == 'pending' finally: await peer.close() @pytest.mark.asyncio async def test_historical_limit_counts_running_jobs_across_revisions_of_one_tool(installed): - app,root=installed - template=await _claimed_dns_job(app,root,'concurrency-mixed') - ref=await _dns_revision_declaring(app,2) - worker=EffectWorker(app,tool_runner=lambda *a:None) - claimed,_=await worker._claim() - assert claimed.id==template.id - historical=await _enqueue_like(app,template,2) - declared=await _enqueue_like(app,template,1,arguments={**template.arguments,'tool':ref}) - job,execute=await worker._claim() - assert execute and job.id==declared.id - assert await worker._claim()==(None,False) - assert await _state(app,historical.id)=='pending' + app, root = installed + template = await _claimed_dns_job(app, root, 'concurrency-mixed') + ref = await _dns_revision_declaring(app, 2) + worker = EffectWorker(app, tool_runner=lambda *a: None) + claimed, _ = await worker._claim() + assert claimed.id == template.id + historical = await _enqueue_like(app, template, 2) + declared = await _enqueue_like(app, template, 1, arguments={**template.arguments, 'tool': ref}) + job, execute = await worker._claim() + assert execute and job.id == declared.id + assert await worker._claim() == (None, False) + assert await _state(app, historical.id) == 'pending' @pytest.mark.asyncio -@pytest.mark.parametrize('declared',[0,33,-1,True,'2',1.5,None]) -async def test_invalid_declared_tool_concurrency_is_rejected_not_unbounded(installed,declared): - app,root=installed - template=await _claimed_dns_job(app,root,'concurrency-invalid') - ref=await _dns_revision_declaring(app,declared) +@pytest.mark.parametrize('declared', [0, 33, -1, True, '2', 1.5, None]) +async def test_invalid_declared_tool_concurrency_is_rejected_not_unbounded(installed, declared): + app, root = installed + template = await _claimed_dns_job(app, root, 'concurrency-invalid') + ref = await _dns_revision_declaring(app, declared) async with app.metadata.transaction(write=False) as tx: - with pytest.raises(Failure,match='invalid_tool_concurrency'): - await read_tool(app,tx,'tool_dns',ref['revision']) + with pytest.raises(Failure, match='invalid_tool_concurrency'): + await read_tool(app, tx, 'tool_dns', ref['revision']) async with app.metadata.transaction(write=True) as tx: - await tx.save_job(replace(template,state='done')) - bad=await _enqueue_like(app,template,1,arguments={**template.arguments,'tool':ref}) - ran=[] - async def runner(*args):ran.append(args) - worker=EffectWorker(app,tool_runner=runner) - job,execute=await worker._claim() - assert job.id==bad.id and not execute - assert await _state(app,bad.id)=='failed' and not ran + await tx.save_job(replace(template, state='done')) + bad = await _enqueue_like(app, template, 1, arguments={**template.arguments, 'tool': ref}) + ran = [] + + async def runner(*args): + ran.append(args) + + worker = EffectWorker(app, tool_runner=runner) + job, execute = await worker._claim() + assert job.id == bad.id and not execute + assert await _state(app, bad.id) == 'failed' and not ran async with app.metadata.transaction(write=False) as tx: - assert tx.setting('job_status:'+bad.id)=={'code':'invalid_tool_concurrency'} - assert await worker._claim()==(None,False) + assert tx.setting('job_status:' + bad.id) == {'code': 'invalid_tool_concurrency'} + assert await worker._claim() == (None, False) @pytest.mark.asyncio @@ -285,8 +462,9 @@ async def test_tool_run_is_the_only_public_operation_and_old_short_code_is_retir assert descriptor('dns')['operation'] == 'tool.run' async with app.metadata.transaction(write=False) as tx: assert (await read_tool(app, tx, 'tool_dns')).operation == 'tool.run' - async with httpx.AsyncClient(transport=httpx.ASGITransport(app=create_app(app)), - base_url='http://testserver') as http: + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=create_app(app)), base_url='http://testserver' + ) as http: current = await http.get('/-/d/tool.run') assert current.status_code == 200 assert current.json()['operations'][0]['code'] == new_code diff --git a/tests/test_tools_directory.py b/tests/test_tools_directory.py index 7a2f9d53..e9bd4c0f 100644 --- a/tests/test_tools_directory.py +++ b/tests/test_tools_directory.py @@ -1,4 +1,5 @@ """Canonical /tools path and the certificate boundary on a fresh install.""" + from dataclasses import replace import pytest @@ -29,11 +30,11 @@ async def test_new_install_tools_path_keeps_permissions_and_capability_gate(inst assert denied.error.code == 'tool_certificate_required', wire(denied) cap = scoped(app, 'tool.use', 'tool_dns', app.registry.capability('tool.use').operations) cert = await approve(app, root, uid, key, (cap,)) - listing = await call(app, 'discovery.get', {'id': '/tools'}, key=key, subject=uid, - certs=(cert.resource_id,)) + listing = await call( + app, 'discovery.get', {'id': '/tools'}, key=key, subject=uid, certs=(cert.resource_id,) + ) assert [tool['name'] for tool in listing.data['items']] == ['dns'] - accepted = await call(app, 'tool.run', args, key=key, subject=uid, - certs=(cert.resource_id,)) + accepted = await call(app, 'tool.run', args, key=key, subject=uid, certs=(cert.resource_id,)) assert accepted.status == 'accepted', wire(accepted) @@ -42,20 +43,22 @@ async def test_legacy_tools_path_is_read_only_and_new_alias_still_checks_certifi app, root = installed async with app.metadata.transaction(write=True) as tx: current = await tx.resource('t_tools') - await tx.replace(replace(current, name='_tools', generation=current.generation + 1), - current.generation) + await tx.replace( + replace(current, name='_tools', generation=current.generation + 1), current.generation + ) await bootstrap(app.metadata, app.contents, app.registry, NOW) key, uid, _ = await register(app, 'legacy-tools-directory') cap = scoped(app, 'tool.use', 'tool_dns', app.registry.capability('tool.use').operations) cert = await approve(app, root, uid, key, (cap,)) - legacy_read = await call(app, 'discovery.get', {'id': '/_tools'}, key=key, subject=uid, - certs=(cert.resource_id,)) + legacy_read = await call( + app, 'discovery.get', {'id': '/_tools'}, key=key, subject=uid, certs=(cert.resource_id,) + ) assert [tool['name'] for tool in legacy_read.data['items']] == ['dns'] args = {'id': '/_tools/dns', 'arguments': {'name': 'example.org', 'type': 'A'}} - rejected = await call(app, 'tool.run', args, key=key, subject=uid, - certs=(cert.resource_id,)) + rejected = await call(app, 'tool.run', args, key=key, subject=uid, certs=(cert.resource_id,)) assert rejected.error.code == 'legacy_tool_path_read_only', wire(rejected) canonical = {**args, 'id': '/tools/dns'} - allowed = await call(app, 'tool.run', canonical, key=key, subject=uid, - certs=(cert.resource_id,)) + allowed = await call( + app, 'tool.run', canonical, key=key, subject=uid, certs=(cert.resource_id,) + ) assert allowed.status == 'accepted', wire(allowed) diff --git a/tests/test_topic_governance.py b/tests/test_topic_governance.py index 2e6a7df4..9e6092fd 100644 --- a/tests/test_topic_governance.py +++ b/tests/test_topic_governance.py @@ -1,14 +1,14 @@ """Topic roles, bans and virtual event history share the write transaction.""" + from datetime import timedelta import httpx import pytest +from test_service import NOW, call, register -from msg.core.codec import wire -from msg.core.codec import b64, canonical +from msg.core.codec import b64, canonical, wire from msg.core.requests import request_for from msg.transports.http import create_app -from test_service import NOW, call, register @pytest.mark.asyncio @@ -16,43 +16,74 @@ async def test_topic_open_membership_ban_and_last_admin(installed): app, _ = installed owner_key, owner, _ = await register(app, 'topic-owner') member_key, member, _ = await register(app, 'topic-member') - created = await call(app, 'content.topic_create', {'parent': '/main', 'name': 'governed'}, - key=owner_key, subject=owner) + created = await call( + app, + 'content.topic_create', + {'parent': '/main', 'name': 'governed'}, + key=owner_key, + subject=owner, + ) assert created.status == 'ok', wire(created) topic = created.resources[0].id async with app.metadata.transaction(write=False) as tx: resource = await tx.resource(topic) assert resource.created_by == owner - assert tx.one('SELECT role,status FROM topic_memberships WHERE topic=? AND subject=?', - (topic, owner)) == ('admin', 'active') + assert tx.one( + 'SELECT role,status FROM topic_memberships WHERE topic=? AND subject=?', (topic, owner) + ) == ('admin', 'active') joined = await call(app, 'content.topic_join', {'id': topic}, key=member_key, subject=member) assert joined.status == 'ok' and joined.data['status'] == 'active', wire(joined) refused = await call(app, 'content.topic_leave', {'id': topic}, key=owner_key, subject=owner) assert refused.status == 'error' and refused.error.code == 'last_topic_admin' async with app.metadata.transaction(write=False) as tx: - after_failed_leave=tx.one('SELECT COUNT(*) FROM events')[0] - demote = await call(app, 'content.topic_demote', {'id': topic, 'subject_id': owner}, - key=owner_key, subject=owner) + after_failed_leave = tx.one('SELECT COUNT(*) FROM events')[0] + demote = await call( + app, + 'content.topic_demote', + {'id': topic, 'subject_id': owner}, + key=owner_key, + subject=owner, + ) assert demote.status == 'error' and demote.error.code == 'last_topic_admin' async with app.metadata.transaction(write=False) as tx: assert tx.one('SELECT COUNT(*) FROM events')[0] == after_failed_leave - ban = await call(app, 'content.topic_ban', {'id': topic, 'subject_id': member, - 'reason': 'moderation detail'}, key=owner_key, subject=owner) + ban = await call( + app, + 'content.topic_ban', + {'id': topic, 'subject_id': member, 'reason': 'moderation detail'}, + key=owner_key, + subject=owner, + ) assert ban.status == 'ok', wire(ban) - blocked = await call(app, 'content.post_create', {'parent': topic, 'body': 'should fail'}, - key=member_key, subject=member) + blocked = await call( + app, + 'content.post_create', + {'parent': topic, 'body': 'should fail'}, + key=member_key, + subject=member, + ) assert blocked.status == 'error' and blocked.error.code == 'topic_banned' - unban = await call(app, 'content.topic_unban', {'id': topic, 'subject_id': member}, - key=owner_key, subject=owner) + unban = await call( + app, + 'content.topic_unban', + {'id': topic, 'subject_id': member}, + key=owner_key, + subject=owner, + ) assert unban.status == 'ok' async with app.metadata.transaction(write=False) as tx: - assert tx.one('SELECT status FROM topic_memberships WHERE topic=? AND subject=?', - (topic, member))[0] != 'active' + assert ( + tx.one( + 'SELECT status FROM topic_memberships WHERE topic=? AND subject=?', (topic, member) + )[0] + != 'active' + ) assert (await tx.resource(topic)).created_by == owner rejoined = await call(app, 'content.topic_join', {'id': topic}, key=member_key, subject=member) assert rejoined.status == 'ok' - no_system_power = await call(app, 'discovery.get', {'id': '/private'}, - key=owner_key, subject=owner) + no_system_power = await call( + app, 'discovery.get', {'id': '/private'}, key=owner_key, subject=owner + ) assert no_system_power.status == 'error' @@ -62,43 +93,88 @@ async def test_topic_policy_approval_invite_closed_and_role_transfer(installed): owner_key, owner, _ = await register(app, 'policy-owner') alice_key, alice, _ = await register(app, 'policy-alice') bob_key, bob, _ = await register(app, 'policy-bob') - topic = (await call(app, 'content.topic_create', {'parent': '/main', 'name': 'policy-topic'}, - key=owner_key, subject=owner)).resources[0].id - approval = await call(app, 'content.topic_policy_set', {'id': topic, 'membership_policy': 'approval'}, - key=owner_key, subject=owner) + topic = ( + ( + await call( + app, + 'content.topic_create', + {'parent': '/main', 'name': 'policy-topic'}, + key=owner_key, + subject=owner, + ) + ) + .resources[0] + .id + ) + approval = await call( + app, + 'content.topic_policy_set', + {'id': topic, 'membership_policy': 'approval'}, + key=owner_key, + subject=owner, + ) assert approval.status == 'ok' pending = await call(app, 'content.topic_join', {'id': topic}, key=alice_key, subject=alice) assert pending.data['status'] == 'pending' - approved = await call(app, 'content.topic_approve', {'id': topic, 'subject_id': alice}, - key=owner_key, subject=owner) + approved = await call( + app, + 'content.topic_approve', + {'id': topic, 'subject_id': alice}, + key=owner_key, + subject=owner, + ) assert approved.data['status'] == 'active' member_sync = await call(app, 'communication.changes', {}, key=alice_key, subject=alice) - assert any(item['type'].startswith('topic.') and item['data'].get('topic_id') == topic - for item in member_sync.data['items']) - assert sum(item['request_id']==approved.request_id for item in member_sync.data['items'])==1 - promoted = await call(app, 'content.topic_promote', {'id': topic, 'subject_id': alice}, - key=owner_key, subject=owner) + assert any( + item['type'].startswith('topic.') and item['data'].get('topic_id') == topic + for item in member_sync.data['items'] + ) + assert sum(item['request_id'] == approved.request_id for item in member_sync.data['items']) == 1 + promoted = await call( + app, + 'content.topic_promote', + {'id': topic, 'subject_id': alice}, + key=owner_key, + subject=owner, + ) assert promoted.data['role'] == 'admin' left = await call(app, 'content.topic_leave', {'id': topic}, key=owner_key, subject=owner) assert left.status == 'ok' - changed = await call(app, 'content.topic_policy_set', {'id': topic, 'membership_policy': 'invite'}, - key=alice_key, subject=alice) + changed = await call( + app, + 'content.topic_policy_set', + {'id': topic, 'membership_policy': 'invite'}, + key=alice_key, + subject=alice, + ) assert changed.status == 'ok' denied = await call(app, 'content.topic_join', {'id': topic}, key=bob_key, subject=bob) assert denied.status == 'error' - invited = await call(app, 'content.topic_invite', {'id': topic, 'subject_id': bob}, - key=alice_key, subject=alice) + invited = await call( + app, 'content.topic_invite', {'id': topic, 'subject_id': bob}, key=alice_key, subject=alice + ) assert invited.status == 'ok' joined = await call(app, 'content.topic_join', {'id': topic}, key=bob_key, subject=bob) assert joined.status == 'ok' - closed = await call(app, 'content.topic_policy_set', {'id': topic, 'membership_policy': 'closed'}, - key=alice_key, subject=alice) + closed = await call( + app, + 'content.topic_policy_set', + {'id': topic, 'membership_policy': 'closed'}, + key=alice_key, + subject=alice, + ) assert closed.status == 'ok' - reopened = await call(app, 'content.topic_policy_set', {'id': topic, 'membership_policy': 'open'}, - key=alice_key, subject=alice) + reopened = await call( + app, + 'content.topic_policy_set', + {'id': topic, 'membership_policy': 'open'}, + key=alice_key, + subject=alice, + ) assert reopened.status == 'ok' - removed = await call(app, 'content.topic_remove', {'id': topic, 'subject_id': bob}, - key=alice_key, subject=alice) + removed = await call( + app, 'content.topic_remove', {'id': topic, 'subject_id': bob}, key=alice_key, subject=alice + ) assert removed.data['status'] == 'removed' rejoined = await call(app, 'content.topic_join', {'id': topic}, key=bob_key, subject=bob) assert rejoined.data['status'] == 'active' @@ -109,20 +185,45 @@ async def test_expired_ban_stops_blocking_without_restoring_membership(installed app, _ = installed owner_key, owner, _ = await register(app, 'expiry-owner') member_key, member, _ = await register(app, 'expiry-member') - topic = (await call(app, 'content.topic_create', {'parent': '/main', 'name': 'expiry-topic'}, - key=owner_key, subject=owner)).resources[0].id + topic = ( + ( + await call( + app, + 'content.topic_create', + {'parent': '/main', 'name': 'expiry-topic'}, + key=owner_key, + subject=owner, + ) + ) + .resources[0] + .id + ) await call(app, 'content.topic_join', {'id': topic}, key=member_key, subject=member) - banned = await call(app, 'content.topic_ban', {'id': topic, 'subject_id': member, - 'expires_at': wire(NOW + timedelta(seconds=60))}, key=owner_key, subject=owner) + banned = await call( + app, + 'content.topic_ban', + {'id': topic, 'subject_id': member, 'expires_at': wire(NOW + timedelta(seconds=60))}, + key=owner_key, + subject=owner, + ) assert banned.status == 'ok' app.executor.clock = lambda: NOW + timedelta(seconds=61) async with app.metadata.transaction(write=False) as tx: - assert tx.one('SELECT status FROM topic_memberships WHERE topic=? AND subject=?', - (topic, member))[0] != 'active' + assert ( + tx.one( + 'SELECT status FROM topic_memberships WHERE topic=? AND subject=?', (topic, member) + )[0] + != 'active' + ) joined = await call(app, 'content.topic_join', {'id': topic}, key=member_key, subject=member) assert joined.status == 'ok' - post = await call(app, 'content.post_create', {'parent': topic, 'body': 'after expiry'}, - key=member_key, subject=member) + post = await call( + app, + 'content.post_create', + {'parent': topic, 'body': 'after expiry'}, + key=member_key, + subject=member, + ) assert post.status == 'ok' @@ -131,68 +232,113 @@ async def test_virtual_events_compact_are_read_only_and_reason_is_admin_only(ins app, _ = installed owner_key, owner, _ = await register(app, 'events-owner') member_key, member, _ = await register(app, 'events-member') - topic = (await call(app, 'content.topic_create', {'parent': '/main', 'name': 'events-topic'}, - key=owner_key, subject=owner)).resources[0].id + topic = ( + ( + await call( + app, + 'content.topic_create', + {'parent': '/main', 'name': 'events-topic'}, + key=owner_key, + subject=owner, + ) + ) + .resources[0] + .id + ) await call(app, 'content.topic_join', {'id': topic}, key=member_key, subject=member) watched = await call(app, 'communication.watch', {'id': topic}, key=member_key, subject=member) assert watched.status == 'ok' before = None async with app.metadata.transaction(write=False) as tx: - before = (tx.one('SELECT COUNT(*) FROM resources WHERE parent=? AND type=?',(topic,'post'))[0], - tx.one('SELECT COUNT(*) FROM revisions')[0]) - await call(app, 'content.topic_ban', {'id': topic, 'subject_id': member, - 'reason': 'private moderation note'}, key=owner_key, subject=owner) + before = ( + tx.one('SELECT COUNT(*) FROM resources WHERE parent=? AND type=?', (topic, 'post'))[0], + tx.one('SELECT COUNT(*) FROM revisions')[0], + ) + await call( + app, + 'content.topic_ban', + {'id': topic, 'subject_id': member, 'reason': 'private moderation note'}, + key=owner_key, + subject=owner, + ) compact = await call(app, 'content.topic_events', {'id': topic}, key=owner_key, subject=owner) assert compact.status == 'ok' and compact.data['view'] == 'compact' assert len(compact.data['items']) <= 10 and compact.data['items'] - normal = await call(app, 'content.topic_events', {'id': topic, 'view': 'normal'}, - key=owner_key, subject=owner) + normal = await call( + app, 'content.topic_events', {'id': topic, 'view': 'normal'}, key=owner_key, subject=owner + ) assert any('private moderation note' in str(item) for item in normal.data['items']) - outsider = await call(app, 'content.topic_events', {'id': topic, 'view': 'normal'}, - key=member_key, subject=member) + outsider = await call( + app, 'content.topic_events', {'id': topic, 'view': 'normal'}, key=member_key, subject=member + ) assert outsider.status == 'error' or 'private moderation note' not in str(outsider.data) changes = await call(app, 'communication.changes', {}, key=member_key, subject=member) assert changes.status == 'ok' and 'private moderation note' not in str(changes.data) inbox = await call(app, 'communication.inbox', {}, key=member_key, subject=member) - assert any(item.get('source') == 'topic_governance' and item.get('action') == 'member.ban' - for item in inbox.data['items']) + assert any( + item.get('source') == 'topic_governance' and item.get('action') == 'member.ban' + for item in inbox.data['items'] + ) for index in range(12): policy = 'approval' if index % 2 else 'open' - changed = await call(app, 'content.topic_policy_set', {'id': topic, - 'membership_policy': policy}, key=owner_key, subject=owner) + changed = await call( + app, + 'content.topic_policy_set', + {'id': topic, 'membership_policy': policy}, + key=owner_key, + subject=owner, + ) assert changed.status == 'ok' recent = await call(app, 'content.topic_events', {'id': topic}, key=owner_key, subject=owner) assert len(recent.data['items']) == 10 and recent.data['cursor'] async with app.metadata.transaction(write=False) as tx: - topic_path=await tx.path(topic) - async with httpx.AsyncClient(transport=httpx.ASGITransport(app=create_app(app)), - base_url='http://testserver') as http: - compact_http=await http.get(topic_path+'/_events.md') - compact_path=await http.get(topic_path+'/_events.md/compact/10') - assert compact_http.status_code==compact_path.status_code==200 - assert compact_http.content==compact_path.content - assert compact_http.headers['etag']==compact_path.headers['etag'] - assert len(compact_http.json()['items'])==10 - normal_query=await http.get(topic_path+'/_events.md?view=normal&limit=10') - normal_path=await http.get(topic_path+'/_events.md/normal/10') - assert normal_query.status_code==normal_path.status_code==200 - assert normal_query.content==normal_path.content - assert (await http.head(topic_path+'/_events.md')).status_code==200 - assert (await http.post(topic_path+'/_events.md')).status_code==405 - assert (await http.get(topic_path+'/_events.md',headers={ - 'If-None-Match':compact_http.headers['etag']})).status_code==304 - older = await call(app, 'content.topic_events', {'id': topic, - 'cursor': recent.data['cursor']}, key=owner_key, subject=owner) + topic_path = await tx.path(topic) + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=create_app(app)), base_url='http://testserver' + ) as http: + compact_http = await http.get(topic_path + '/_events.md') + compact_path = await http.get(topic_path + '/_events.md/compact/10') + assert compact_http.status_code == compact_path.status_code == 200 + assert compact_http.content == compact_path.content + assert compact_http.headers['etag'] == compact_path.headers['etag'] + assert len(compact_http.json()['items']) == 10 + normal_query = await http.get(topic_path + '/_events.md?view=normal&limit=10') + normal_path = await http.get(topic_path + '/_events.md/normal/10') + assert normal_query.status_code == normal_path.status_code == 200 + assert normal_query.content == normal_path.content + assert (await http.head(topic_path + '/_events.md')).status_code == 200 + assert (await http.post(topic_path + '/_events.md')).status_code == 405 + assert ( + await http.get( + topic_path + '/_events.md', headers={'If-None-Match': compact_http.headers['etag']} + ) + ).status_code == 304 + older = await call( + app, + 'content.topic_events', + {'id': topic, 'cursor': recent.data['cursor']}, + key=owner_key, + subject=owner, + ) assert older.status == 'ok' and older.data['items'] assert {item['c'] for item in recent.data['items']} <= set(recent.data['event_codes'].values()) async with app.metadata.transaction(write=False) as tx: - after = (tx.one('SELECT COUNT(*) FROM resources WHERE parent=? AND type=?',(topic,'post'))[0], - tx.one('SELECT COUNT(*) FROM revisions')[0]) - assert tx.one('SELECT id FROM resources WHERE parent=? AND name=?', - (topic, '_events.md')) is None + after = ( + tx.one('SELECT COUNT(*) FROM resources WHERE parent=? AND type=?', (topic, 'post'))[0], + tx.one('SELECT COUNT(*) FROM revisions')[0], + ) + assert ( + tx.one('SELECT id FROM resources WHERE parent=? AND name=?', (topic, '_events.md')) + is None + ) assert after == before - reserved = await call(app, 'content.post_create', {'parent': topic, 'name': '_events.md', - 'body': 'fake'}, key=owner_key, subject=owner) + reserved = await call( + app, + 'content.post_create', + {'parent': topic, 'name': '_events.md', 'body': 'fake'}, + key=owner_key, + subject=owner, + ) assert reserved.status == 'error' @@ -200,23 +346,41 @@ async def test_virtual_events_compact_are_read_only_and_reason_is_admin_only(ins async def test_virtual_events_private_topic_authorizes_before_etag(installed): app, _ = installed owner_key, owner, _ = await register(app, 'events-private-owner') - created=await call(app,'content.topic_create',{'parent':'/main','name':'private-events'}, - key=owner_key,subject=owner) - topic=created.resources[0].id - private=await call(app,'content.chmod',{'id':topic,'mode':'0700'},key=owner_key, - subject=owner,expected=((topic,created.data['generation']),)) - assert private.status=='ok',wire(private) + created = await call( + app, + 'content.topic_create', + {'parent': '/main', 'name': 'private-events'}, + key=owner_key, + subject=owner, + ) + topic = created.resources[0].id + private = await call( + app, + 'content.chmod', + {'id': topic, 'mode': '0700'}, + key=owner_key, + subject=owner, + expected=((topic, created.data['generation']),), + ) + assert private.status == 'ok', wire(private) async with app.metadata.transaction(write=False) as tx: - path=await tx.path(topic) - packet=request_for('content.topic_events',{'id':path,'view':'compact','limit':10}, - app.settings.service_url,subject=owner,signer=owner_key, - expires_at=NOW+timedelta(seconds=120)) - async with httpx.AsyncClient(transport=httpx.ASGITransport(app=create_app(app)), - base_url='http://testserver') as http: - for method in (http.get,http.head): - denied=await method(path+'/_events.md',headers={'If-None-Match':'*'}) - assert denied.status_code==403 and 'etag' not in denied.headers - allowed=await http.get(path+'/_events.md',headers={ - 'X-Msg-Request':b64(canonical(packet))}) - assert allowed.status_code==200,allowed.text - assert allowed.json()['topic_id']==topic + path = await tx.path(topic) + packet = request_for( + 'content.topic_events', + {'id': path, 'view': 'compact', 'limit': 10}, + app.settings.service_url, + subject=owner, + signer=owner_key, + expires_at=NOW + timedelta(seconds=120), + ) + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=create_app(app)), base_url='http://testserver' + ) as http: + for method in (http.get, http.head): + denied = await method(path + '/_events.md', headers={'If-None-Match': '*'}) + assert denied.status_code == 403 and 'etag' not in denied.headers + allowed = await http.get( + path + '/_events.md', headers={'X-Msg-Request': b64(canonical(packet))} + ) + assert allowed.status_code == 200, allowed.text + assert allowed.json()['topic_id'] == topic diff --git a/tests/test_transfer.py b/tests/test_transfer.py index 7b472536..08792710 100644 --- a/tests/test_transfer.py +++ b/tests/test_transfer.py @@ -1,82 +1,124 @@ """One persistent transfer state machine, independently of the adapter.""" -from dataclasses import replace -from datetime import timedelta -import hashlib + import pytest +from test_service import call, register -from msg.core.codec import b64,unb64,digest,wire -from msg.core.models import ResourceRef -from test_service import call,register,NOW +from msg.core.codec import b64, digest, unb64, wire @pytest.mark.asyncio async def test_upload_out_of_order_conflicts_seal_and_download(installed): - app,_=installed - key,uid,cert=await register(app,'transfer-user') - async def invoke(op,a): - return await call(app,op,a,key=key,subject=uid,certs=(cert,)) - data=b'plain text\n'+bytes(range(256))*7 - opened=await invoke('transfer.open',{'direction':'upload','size':len(data),'digest':digest(data), - 'requested_part_bytes':512,'media_type':'application/octet-stream'}) - assert opened.status=='ok',wire(opened) - tid=opened.data['transfer_id'] - size=opened.data['part_bytes'] - assert 0