From e5238e77de9cfa7c135109dc02aa6bce868e2767 Mon Sep 17 00:00:00 2001 From: LIghtJUNction Date: Wed, 30 Sep 2026 03:38:01 +0800 Subject: [PATCH 1/4] Quality gates for Python 3.15 and exact release artifacts Preserve the reviewed local fixes against merged main, enforce strict Ruff and locked tooling, cover fixture cleanup and release integrity, and verify all tests across eight disjoint partitions. Refs #83, #84, #85 --- .github/workflows/architecture-boundaries.yml | 2 +- .github/workflows/architecture-ports.yml | 2 +- .github/workflows/ci.yml | 18 +- .github/workflows/client-boundary.yml | 2 +- .github/workflows/client-part-bytes.yml | 2 +- .../workflows/collaboration-list-bounds.yml | 2 +- .../workflows/git-transport-boundaries.yml | 2 +- .github/workflows/hosting-authority.yml | 2 +- .github/workflows/hosting-readonly.yml | 2 +- .github/workflows/integration-regressions.yml | 4 +- .github/workflows/issue-recovery.yml | 2 +- .github/workflows/lease-list-bounds.yml | 2 +- .github/workflows/market-boundaries.yml | 2 +- .../workflows/metadata-session-boundary.yml | 2 +- .github/workflows/python-quality.yml | 67 + .github/workflows/recovery-safety.yml | 2 +- .github/workflows/storage-session.yml | 2 +- .github/workflows/token-delivery-boundary.yml | 2 +- .github/workflows/transfer-status.yml | 2 +- .gitignore | 1 + README.md | 2 +- conformance/test_token_budget.py | 85 +- conformance/test_transports.py | 254 +- conftest.py | 53 +- docs/CLIENT_INSTALLATION.md | 4 +- docs/DEPLOYMENT.md | 2 +- docs/LOCAL_VERIFICATION.md | 62 + docs/NEW_ROOT_LEGACY_IMPORT_RUNBOOK.md | 74 +- docs/QUALITY_PROGRESS.md | 65 + pyproject.toml | 6 + scripts/check_client_install.py | 110 +- scripts/check_isolated_sshd.py | 320 +- scripts/check_package_artifacts.py | 183 +- scripts/check_tool_sandbox.py | 39 +- scripts/ci_shards.py | 32 +- scripts/verify_local.py | 252 + src/msg/__init__.py | 3 +- src/msg/admin/backup_retirement.py | 77 +- src/msg/admin/backups.py | 578 +- src/msg/admin/config_check.py | 360 +- src/msg/admin/content_edit_check.py | 81 +- src/msg/admin/custodial_check.py | 114 +- src/msg/admin/diagnostics.py | 1908 ++++--- src/msg/admin/dm_check.py | 40 +- src/msg/admin/git_check.py | 166 +- src/msg/admin/honor_check.py | 168 +- src/msg/admin/legacy_approval.py | 238 +- src/msg/admin/market.py | 130 +- src/msg/admin/market_check.py | 1012 +++- src/msg/admin/money.py | 476 +- src/msg/admin/offer_import.py | 176 +- src/msg/admin/preflight.py | 111 +- src/msg/admin/recovery_policy.py | 139 +- src/msg/admin/recovery_proof.py | 506 +- src/msg/admin/recovery_replay.py | 444 +- src/msg/admin/recovery_state.py | 172 +- src/msg/admin/restore_database.py | 48 +- src/msg/admin/root.py | 885 +++- src/msg/admin/rotation.py | 400 +- src/msg/admin/token_delivery_check.py | 15 +- src/msg/admin/tool_check.py | 73 +- src/msg/admin/transfer_check.py | 140 +- src/msg/admin/upgrade_check.py | 113 +- src/msg/application.py | 228 +- src/msg/atomic_file.py | 15 +- src/msg/bootstrap.py | 962 +++- src/msg/cli.py | 1241 +++-- src/msg/client.py | 1330 +++-- src/msg/client_certificates.py | 239 +- src/msg/client_collaboration.py | 20 +- src/msg/client_content.py | 94 +- src/msg/client_custodial.py | 264 +- src/msg/client_market.py | 201 +- src/msg/client_recovery.py | 375 +- src/msg/client_secrets.py | 97 +- src/msg/client_tokens.py | 68 +- src/msg/client_upgrade.py | 233 +- src/msg/config.py | 572 +- src/msg/config_contracts.py | 71 +- src/msg/constants.py | 21 +- src/msg/core/batching.py | 32 +- src/msg/core/codec.py | 204 +- src/msg/core/contracts.py | 134 +- src/msg/core/cursors.py | 149 +- src/msg/core/email_address.py | 29 +- src/msg/core/errors.py | 29 +- src/msg/core/events.py | 1 + src/msg/core/execution_ports.py | 14 +- src/msg/core/executor.py | 529 +- src/msg/core/models.py | 67 +- src/msg/core/packet.py | 174 +- src/msg/core/permissions.py | 23 +- src/msg/core/query.py | 16 +- src/msg/core/read_query.py | 75 +- src/msg/core/registry.py | 327 +- src/msg/core/requests.py | 74 +- src/msg/core/schema_policy.py | 9 +- src/msg/core/schemas.py | 113 +- src/msg/core/search_query.py | 35 +- src/msg/core/tags.py | 7 +- src/msg/core/template_dsl.py | 130 +- src/msg/core/text_patch.py | 245 +- src/msg/core/tool_execution.py | 9 +- src/msg/core/transfer.py | 377 +- src/msg/daemon.py | 489 +- src/msg/extensions/git_publication.py | 277 +- src/msg/extensions/hosting.py | 620 ++- src/msg/extensions/keystore.py | 248 +- src/msg/extensions/repositories.py | 805 ++- src/msg/extensions/rss.py | 116 +- src/msg/extensions/ssh.py | 286 +- src/msg/extensions/ssh_git.py | 28 +- src/msg/extensions/tools.py | 341 +- src/msg/hosting_runtime.py | 140 +- src/msg/market/arbitration.py | 799 ++- src/msg/market/catalog.py | 21 +- src/msg/market/compatibility.py | 158 +- src/msg/market/delivery.py | 561 +- src/msg/market/delivery_notifications.py | 98 +- src/msg/market/delivery_targets.py | 57 +- src/msg/market/email.py | 212 +- src/msg/market/entitlement_orders.py | 110 +- src/msg/market/escrow.py | 653 ++- src/msg/market/ledger.py | 257 +- src/msg/market/managed_delivery.py | 208 +- src/msg/market/offer_resources.py | 205 +- src/msg/market/order_records.py | 94 +- src/msg/market/order_resources.py | 205 +- src/msg/market/orders.py | 349 +- src/msg/market/policy.py | 387 +- src/msg/market/rationale.py | 94 +- src/msg/market/references.py | 4 +- src/msg/market/targets.py | 207 +- src/msg/plugins/__init__.py | 32 +- src/msg/plugins/achievements.py | 461 +- src/msg/plugins/batch.py | 59 +- src/msg/plugins/bounty.py | 769 ++- src/msg/plugins/collaboration.py | 409 +- src/msg/plugins/collaboration_resources.py | 309 +- src/msg/plugins/common.py | 572 +- src/msg/plugins/communication.py | 2076 +++++--- src/msg/plugins/content.py | 2285 +++++--- src/msg/plugins/custodial_lifecycle.py | 444 +- src/msg/plugins/delivery.py | 164 +- src/msg/plugins/discovery.py | 2918 ++++++---- src/msg/plugins/discussion.py | 436 +- src/msg/plugins/extensions.py | 10 +- src/msg/plugins/features.py | 27 +- src/msg/plugins/files.py | 63 +- src/msg/plugins/following.py | 51 +- src/msg/plugins/hosting_capacity.py | 26 +- src/msg/plugins/identity.py | 4696 +++++++++++------ src/msg/plugins/money.py | 189 +- src/msg/plugins/offers.py | 481 +- src/msg/plugins/orders.py | 352 +- src/msg/plugins/proposals.py | 131 +- src/msg/plugins/read_predicates.py | 28 +- src/msg/plugins/receipts.py | 44 +- src/msg/plugins/recovery.py | 886 +++- src/msg/plugins/saved_queries.py | 179 +- src/msg/plugins/schemas.py | 17 +- src/msg/plugins/sharing.py | 566 +- src/msg/plugins/store.py | 368 +- src/msg/plugins/system.py | 158 +- src/msg/plugins/transfer.py | 599 ++- src/msg/plugins/watches.py | 350 +- src/msg/security/age_keys.py | 115 +- src/msg/security/authentication.py | 361 +- src/msg/security/authorization.py | 582 +- src/msg/security/backup_retirement.py | 143 +- src/msg/security/capabilities.py | 358 +- src/msg/security/certificates.py | 382 +- src/msg/security/crypto.py | 140 +- src/msg/security/custodial_migration.py | 334 +- src/msg/security/custodial_protocol.py | 88 +- src/msg/security/custody_history.py | 211 +- src/msg/security/network.py | 137 +- src/msg/security/policy.py | 70 +- src/msg/security/quarantine.py | 2 + src/msg/security/root_files.py | 35 +- src/msg/security/rotation_journal.py | 110 +- src/msg/security/sealed_box.py | 101 +- src/msg/security/sharing_policy.py | 35 +- src/msg/security/token_delivery.py | 177 +- src/msg/security/trust_files.py | 51 +- src/msg/security/vault.py | 287 +- src/msg/storage/capacity.py | 38 +- src/msg/storage/custodial_migration.py | 11 +- src/msg/storage/git.py | 599 ++- src/msg/storage/ledger_migration.py | 295 +- src/msg/storage/legacy_git_capture.py | 50 +- src/msg/storage/legacy_git_import.py | 299 +- src/msg/storage/legacy_git_rehearsal.py | 178 +- src/msg/storage/legacy_identity_plan.py | 448 +- src/msg/storage/legacy_rehearsal.py | 155 +- src/msg/storage/legacy_resource_import.py | 366 +- src/msg/storage/legacy_sqlite.py | 150 +- src/msg/storage/market_migration.py | 33 +- src/msg/storage/postgres.py | 72 +- src/msg/storage/query.py | 3 + src/msg/storage/read_only.py | 117 +- src/msg/storage/session.py | 630 ++- src/msg/storage/sqlite.py | 103 +- src/msg/storage/topic_event_migration.py | 48 +- src/msg/storage/valkey_bus.py | 4 +- src/msg/transports/client.py | 145 +- src/msg/transports/dictionary.py | 601 ++- src/msg/transports/git_http.py | 818 ++- src/msg/transports/graphql.py | 144 +- src/msg/transports/http.py | 94 +- src/msg/transports/http_common.py | 123 +- src/msg/transports/http_routes.py | 3485 +++++++----- src/msg/transports/legacy_content_http.py | 64 +- src/msg/transports/mcp.py | 239 +- src/msg/transports/mcp_protocol.py | 4 +- src/msg/transports/packet.py | 39 +- src/msg/transports/read_tree_path.py | 139 +- src/msg/transports/stdio.py | 37 +- src/msg/transports/url_safety.py | 54 +- src/msg/tui.py | 140 +- src/msg/workers/effects.py | 460 +- src/msg/workers/leases.py | 1 + src/msg/workers/mail.py | 9 +- src/msg/workers/maintenance.py | 535 +- src/msg/workers/sandbox.py | 106 +- src/msg/workers/sandbox_child.py | 180 +- src/msg/workers/webhook.py | 126 +- tests/conftest.py | 90 +- tests/fixtures/create_legacy_market.py | 113 +- tests/http_read_cache_matrix.py | 112 +- tests/issue_78_80/conftest.py | 341 +- .../test_content_patch_contract.py | 239 +- tests/issue_78_80/test_lfs_integrity.py | 277 +- .../test_lfs_migration_contract.py | 79 +- tests/issue_78_80/test_read_query_tree.py | 296 +- .../issue_78_80/test_read_tree_transports.py | 323 +- .../test_revision_signature_fields.py | 114 +- tests/issue_78_80/test_text_patch_formats.py | 228 +- tests/issue_78_80/test_tree_query_ref.py | 202 +- tests/read_only_evidence.py | 34 +- tests/test_access_log_secret_redaction.py | 29 +- tests/test_achievement_pins.py | 279 +- tests/test_achievements.py | 166 +- tests/test_architecture_ports.py | 96 +- tests/test_attachment_http_cache.py | 87 +- tests/test_authority_snapshot_drift.py | 147 +- tests/test_authorization.py | 453 +- tests/test_authorization_matrix.py | 242 +- .../test_authorization_recovery_boundaries.py | 174 +- tests/test_authorization_sources.py | 184 +- tests/test_backup_daemon.py | 79 +- tests/test_backup_drill.py | 335 +- tests/test_backup_policy.py | 29 +- tests/test_backup_retirement_attestation.py | 343 +- tests/test_backup_retirement_cli.py | 45 +- tests/test_batch.py | 123 +- tests/test_batch_secret_release.py | 156 +- tests/test_bounty.py | 650 ++- tests/test_bounty_snapshot_integrity.py | 66 +- tests/test_branding.py | 34 +- tests/test_ca_depth.py | 102 +- tests/test_canonical_post_paths.py | 18 +- tests/test_capability_matrix.py | 94 +- tests/test_capacity_bounds.py | 52 +- tests/test_checkout_decision_integrity.py | 82 +- tests/test_ci_shards.py | 70 +- tests/test_cli_dm.py | 110 +- tests/test_cli_hosting.py | 154 +- tests/test_cli_legacy.py | 153 +- tests/test_cli_recovery.py | 105 +- tests/test_cli_search_grep.py | 184 +- tests/test_client.py | 154 +- tests/test_client_boundary.py | 117 +- tests/test_client_part_bytes.py | 46 +- tests/test_client_response_boundaries.py | 154 +- tests/test_client_secrets.py | 48 +- tests/test_client_token_delivery.py | 215 +- .../test_collaboration_current_visibility.py | 68 +- tests/test_collaboration_list_bounds.py | 66 +- tests/test_collaboration_list_deadline.py | 13 +- tests/test_collaboration_resource_surfaces.py | 130 +- tests/test_collaboration_resources.py | 232 +- tests/test_collaboration_surfaces.py | 325 +- tests/test_complete_recovery_proof.py | 205 +- tests/test_config_safety.py | 83 +- tests/test_configuration_field_contracts.py | 116 +- tests/test_content_edit_check.py | 16 +- tests/test_credential_delivery_contract.py | 254 +- tests/test_credential_delivery_matrix.py | 73 +- tests/test_credential_recovery_window.py | 208 +- tests/test_credential_upgrade_interleave.py | 17 +- tests/test_custodial_client_upgrade.py | 128 +- tests/test_custodial_historical_migration.py | 266 +- tests/test_custodial_inventory_ack.py | 282 +- tests/test_custodial_lifecycle.py | 451 +- tests/test_custodial_rewrap.py | 287 +- tests/test_custodial_schema_migration.py | 36 +- tests/test_custodial_upgrade.py | 399 +- tests/test_custodial_vault.py | 353 +- tests/test_custody_exit_evidence.py | 153 +- tests/test_delivery.py | 285 +- tests/test_delivery_target_regression.py | 40 +- tests/test_dictionary.py | 380 +- tests/test_dictionary_route.py | 29 +- tests/test_dm.py | 248 +- tests/test_dm_selftest.py | 11 +- tests/test_effect_completion_fencing.py | 107 +- tests/test_effect_lease_deadlines.py | 38 +- tests/test_effect_retry_state.py | 65 +- tests/test_entitlement_orders.py | 272 +- tests/test_error_messages.py | 40 +- tests/test_escrow_decision_migration.py | 45 +- tests/test_escrow_decisions.py | 224 +- tests/test_execution_composition.py | 96 +- tests/test_extensions.py | 299 +- tests/test_feature_manifest.py | 215 +- tests/test_file_copy_integrity.py | 78 +- tests/test_file_operations.py | 256 +- tests/test_fixture_collection.py | 20 +- tests/test_following.py | 237 +- tests/test_foundation.py | 119 +- tests/test_get_only_write.py | 57 +- tests/test_git_http_push.py | 574 +- tests/test_git_lfs.py | 482 +- tests/test_git_selftest.py | 3 +- tests/test_git_transport_boundaries.py | 217 +- tests/test_handoff_lease.py | 295 +- tests/test_honor_diagnostics.py | 5 +- tests/test_hosting_read_permissions.py | 26 +- tests/test_hosting_runtime.py | 379 +- tests/test_hosting_runtime_authority.py | 97 +- tests/test_hosting_runtime_isolation.py | 266 +- tests/test_hosting_same_origin.py | 645 ++- tests/test_http.py | 224 +- tests/test_http_effect_order.py | 46 +- tests/test_identity_keys.py | 130 +- tests/test_independent_share_after_move.py | 34 +- tests/test_lease_list_bounds.py | 80 +- tests/test_lease_list_deadline.py | 6 +- tests/test_ledger_account_isolation.py | 122 +- tests/test_ledger_accounts.py | 218 +- tests/test_ledger_migration_json.py | 38 +- tests/test_ledger_migration_references.py | 179 +- tests/test_legacy_approval.py | 124 +- tests/test_legacy_content_http.py | 112 +- tests/test_legacy_directive.py | 668 ++- tests/test_legacy_git_import.py | 104 +- tests/test_legacy_identity_plan.py | 120 +- tests/test_legacy_ledger_snapshot.py | 244 +- tests/test_legacy_resource_import.py | 84 +- tests/test_legacy_routes_removed.py | 105 +- tests/test_legacy_secret_entrypoints.py | 65 +- tests/test_legacy_sqlite_preservation.py | 98 +- tests/test_legacy_token_release.py | 65 +- tests/test_lexical_search_grep.py | 630 ++- tests/test_lfs_shared_gc_quota.py | 201 +- tests/test_linkset_diff.py | 512 +- tests/test_local_postgres_fixtures.py | 137 + tests/test_local_verification.py | 100 + tests/test_managed_checkout.py | 299 +- tests/test_manifest_feature_linkage.py | 22 +- tests/test_markdown_redirect.py | 98 +- tests/test_market_71.py | 112 +- tests/test_market_72.py | 290 +- tests/test_market_account_scope.py | 191 +- tests/test_market_arbitration.py | 418 +- tests/test_market_boundaries.py | 277 +- tests/test_market_cli_bindings.py | 139 +- tests/test_market_compatibility_reads.py | 199 +- tests/test_market_contracts.py | 292 +- tests/test_market_delivery.py | 448 +- tests/test_market_explicit_acceptance.py | 211 +- tests/test_market_integration.py | 124 +- tests/test_market_lease_deadlines.py | 24 +- tests/test_market_lifecycle.py | 149 +- tests/test_market_manual_flow.py | 162 +- tests/test_market_notification_reuse.py | 119 +- tests/test_market_paths.py | 35 +- tests/test_market_rationale.py | 325 +- tests/test_market_recovery.py | 187 +- tests/test_market_redemption.py | 425 +- tests/test_metadata_session_boundary.py | 141 +- tests/test_money_admin.py | 325 +- tests/test_money_config.py | 177 +- tests/test_money_core.py | 243 +- tests/test_money_paths.py | 82 +- tests/test_native_git_read_only.py | 94 +- tests/test_nested_read_query.py | 447 +- tests/test_nginx_secret_logs.py | 178 +- tests/test_notes_todos.py | 332 +- tests/test_offer_compatibility_boundary.py | 75 +- tests/test_offer_import.py | 108 +- tests/test_offer_resource_writer.py | 139 +- tests/test_offers.py | 62 +- tests/test_online_ca_policy.py | 43 +- tests/test_online_issuer.py | 293 +- tests/test_operations_admin.py | 128 +- tests/test_order_paths.py | 154 +- tests/test_order_resources.py | 163 +- tests/test_orders.py | 516 +- tests/test_organization_governance.py | 115 +- tests/test_pagination.py | 98 +- tests/test_passive_get_guard.py | 293 +- tests/test_passive_ingress_priority.py | 83 +- tests/test_patch_rebase_identity.py | 50 +- tests/test_path_read_query.py | 316 +- tests/test_personal_content_signatures.py | 213 +- tests/test_personal_documents.py | 211 +- tests/test_personal_text_boundaries.py | 56 +- tests/test_pg_config.py | 59 +- tests/test_pg_sql_translation.py | 31 +- tests/test_plugin_lifecycle.py | 138 +- tests/test_post_lifecycle.py | 279 +- tests/test_postgres.py | 110 +- tests/test_presence_claim.py | 89 +- tests/test_proposals.py | 216 +- tests/test_protocol_routes.py | 89 +- tests/test_query_ref.py | 447 +- tests/test_query_ref_lifecycle.py | 153 +- tests/test_read_cursor.py | 264 +- tests/test_read_entry_contract_matrix.py | 92 +- tests/test_read_namespace.py | 101 +- tests/test_read_only_evidence.py | 7 +- .../test_read_projection_cache_transports.py | 70 +- tests/test_read_projection_cache_unit.py | 41 +- tests/test_read_query_cursor.py | 214 +- tests/test_read_query_keyset_boundary.py | 74 +- tests/test_receipt_effect_order.py | 43 + tests/test_recovery_authority_reconcile.py | 125 +- tests/test_recovery_current_authority.py | 224 +- tests/test_recovery_envelope.py | 221 +- tests/test_recovery_metadata.py | 399 +- tests/test_recovery_policy_v2.py | 234 +- tests/test_recovery_runtime_generation.py | 30 +- tests/test_recovery_safety_units.py | 376 +- tests/test_registry_contract_references.py | 63 +- tests/test_registry_operation_rules.py | 40 +- tests/test_registry_references.py | 52 +- tests/test_registry_schema_ownership.py | 23 +- tests/test_registry_template_tools.py | 256 +- tests/test_release_artifacts.py | 137 + tests/test_release_version.py | 12 + tests/test_request_target_safety.py | 254 +- tests/test_resource_path_migration.py | 409 +- tests/test_restore_physical_failure.py | 14 +- tests/test_revision_sources.py | 171 +- tests/test_root_rotation.py | 68 +- tests/test_root_rotation_resume.py | 46 +- tests/test_route_effect_matrix.py | 235 +- tests/test_saved_queries.py | 145 +- tests/test_saved_query_typed_scopes.py | 41 +- tests/test_saved_query_watch.py | 250 +- tests/test_search_source_relation_filters.py | 601 ++- tests/test_search_spelling.py | 75 +- tests/test_search_suggestions.py | 235 +- tests/test_security.py | 92 +- tests/test_security_patch_regressions.py | 278 +- tests/test_security_rollback_fences.py | 33 +- tests/test_security_storage_rollback.py | 66 +- tests/test_security_topic_events.py | 73 +- tests/test_security_transfer_rollback.py | 86 +- tests/test_service.py | 214 +- tests/test_session_boundary.py | 45 +- tests/test_share_grants.py | 308 +- tests/test_share_grants_v2.py | 379 +- tests/test_share_links.py | 233 +- tests/test_share_source_boundaries.py | 130 +- tests/test_share_transport_matrix.py | 153 +- tests/test_single_mailbox_delivery.py | 92 +- tests/test_smtp_socket_acceptance.py | 71 +- tests/test_ssh_process_identity.py | 23 +- tests/test_ssh_rss.py | 319 +- tests/test_stable_resource_views.py | 66 +- tests/test_stdio_cli.py | 195 +- tests/test_storage_commit_faults.py | 68 +- tests/test_storage_layout.py | 16 +- tests/test_store_foundation.py | 382 +- tests/test_subject_read_aliases.py | 357 +- tests/test_subject_receipts.py | 352 +- tests/test_sync_checkpoint.py | 310 +- tests/test_sync_cursor.py | 480 +- tests/test_system_rule_migration.py | 82 +- tests/test_system_rule_retirement.py | 156 +- tests/test_system_rules.py | 184 +- tests/test_tags_index.py | 217 +- tests/test_task_a_custodial_binding.py | 100 +- tests/test_task_a_preflight.py | 25 +- tests/test_task_a_recovery_checkpoint.py | 469 +- tests/test_task_a_root_files.py | 17 +- tests/test_temporary_dual_keys.py | 230 +- tests/test_temporary_legacy_migration.py | 170 +- tests/test_temporary_signed_business.py | 41 +- tests/test_text_patch.py | 259 +- tests/test_text_patch_line_boundaries.py | 55 +- tests/test_text_patch_rebase_batch.py | 246 +- tests/test_todo_due_inbox.py | 118 +- tests/test_token_delivery.py | 363 +- tests/test_token_delivery_boundary.py | 126 +- tests/test_tool_feature_selftest.py | 30 +- tests/test_tool_runner_contract.py | 38 +- tests/test_tool_sandbox_acceptance.py | 69 +- tests/test_tools.py | 530 +- tests/test_tools_directory.py | 27 +- tests/test_topic_governance.py | 378 +- tests/test_transfer.py | 176 +- tests/test_transfer_endpoint.py | 116 +- tests/test_transfer_selftest.py | 4 +- tests/test_transfer_status_cursors.py | 139 +- tests/test_trust_path_compatibility.py | 175 +- tests/test_tui.py | 67 +- tests/test_tui_recovery.py | 91 +- tests/test_tui_transport_boundaries.py | 84 +- tests/test_upgrade_journal.py | 388 +- tests/test_upgrade_journal_file_types.py | 44 +- tests/test_url_secrets.py | 170 +- tests/test_valkey_bus.py | 128 +- tests/test_watch_contracts.py | 117 +- tests/test_watch_corruption.py | 51 +- tests/test_watch_surfaces.py | 30 +- tests/test_webhook_delivery.py | 291 +- tests/test_webhook_domain_events.py | 361 +- tests/test_webhook_identity.py | 51 +- tests/test_webhook_overlap.py | 147 +- tests/test_webhook_overlap_collation.py | 7 +- uv.lock | 46 +- 525 files changed, 76775 insertions(+), 36489 deletions(-) create mode 100644 .github/workflows/python-quality.yml create mode 100644 docs/LOCAL_VERIFICATION.md create mode 100644 docs/QUALITY_PROGRESS.md create mode 100644 scripts/verify_local.py create mode 100644 tests/test_local_postgres_fixtures.py create mode 100644 tests/test_local_verification.py create mode 100644 tests/test_receipt_effect_order.py create mode 100644 tests/test_release_artifacts.py create mode 100644 tests/test_release_version.py diff --git a/.github/workflows/architecture-boundaries.yml b/.github/workflows/architecture-boundaries.yml index 25a9b25a..70501828 100644 --- a/.github/workflows/architecture-boundaries.yml +++ b/.github/workflows/architecture-boundaries.yml @@ -73,7 +73,7 @@ jobs: python -m pytest tests/test_tool_runner_contract.py tests/test_tools.py tests/test_effect_completion_fencing.py --junitxml=artifacts/runner-contract.xml - - uses: actions/upload-artifact@v4 + - uses: actions/upload-artifact@v7.0.1 if: always() with: name: architecture-boundaries diff --git a/.github/workflows/architecture-ports.yml b/.github/workflows/architecture-ports.yml index 7782b432..318f36e8 100644 --- a/.github/workflows/architecture-ports.yml +++ b/.github/workflows/architecture-ports.yml @@ -74,7 +74,7 @@ jobs: tests/test_batch_secret_release.py tests/test_tools.py tests/test_webhook_domain_events.py tests/test_read_projection_cache_transports.py tests/test_read_projection_cache_unit.py tests/issue_78_80 --junitxml=artifacts/architecture-ports.xml - - uses: actions/upload-artifact@v4 + - uses: actions/upload-artifact@v7.0.1 if: always() with: name: architecture-ports-evidence diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5143e5a8..f63db70c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -16,7 +16,7 @@ jobs: strategy: fail-fast: false matrix: - shard: [0, 1, 2, 3] + shard: [0, 1, 2, 3, 4, 5, 6, 7] services: postgres: # Recovery jobs explicitly install matching PostgreSQL 16 client tools. @@ -69,13 +69,17 @@ jobs: bwrap --version sudo apparmor_parser -r /usr/share/apparmor/extra-profiles/bwrap-userns-restrict - name: Install the complete dependency set - run: python -m pip install -e '.[dev]' + run: | + python -m pip install uv==0.12.20 + uv sync --locked --extra dev --python "$(command -v python)" + echo "$GITHUB_WORKSPACE/.venv/bin" >> "$GITHUB_PATH" + uv pip check - name: Check real tool isolation before running shards run: python scripts/check_tool_sandbox.py - name: Compile all new modules - run: python -m compileall -q src + run: python -W error -m compileall -q src - name: Run core, business and security tests - run: python scripts/ci_shards.py run ${{ matrix.shard }} 4 + run: python scripts/ci_shards.py run ${{ matrix.shard }} 8 - name: Require every transport and actual tokenizer if: matrix.shard == 0 env: @@ -86,7 +90,7 @@ jobs: run: | python -m build python scripts/check_package_artifacts.py dist - - uses: actions/upload-artifact@v4 + - uses: actions/upload-artifact@v7.0.1 if: always() with: name: rewrite-verification-${{ matrix.shard }} @@ -100,12 +104,12 @@ jobs: timeout-minutes: 5 steps: - uses: actions/checkout@v7 - - uses: actions/download-artifact@v4 + - uses: actions/download-artifact@v8.0.1 with: pattern: rewrite-verification-* merge-multiple: true - name: Require complete disjoint test evidence - run: python scripts/ci_shards.py check 4 + run: python scripts/ci_shards.py check 8 - name: Require all tests, conformance and build jobs to succeed env: SHARD_RESULT: ${{ needs.shards.result }} diff --git a/.github/workflows/client-boundary.yml b/.github/workflows/client-boundary.yml index 383239b6..db09ab13 100644 --- a/.github/workflows/client-boundary.yml +++ b/.github/workflows/client-boundary.yml @@ -63,7 +63,7 @@ jobs: "$client_python" -I "$GITHUB_WORKSPACE/scripts/check_client_install.py" --minimal-install \ > "$GITHUB_WORKSPACE/artifacts/client-install.json" "$client_python" -m pip list --format=json > "$GITHUB_WORKSPACE/artifacts/client-packages.json" - - uses: actions/upload-artifact@v4 + - uses: actions/upload-artifact@v7.0.1 if: always() with: name: client-boundary diff --git a/.github/workflows/client-part-bytes.yml b/.github/workflows/client-part-bytes.yml index 94162d60..8003aae9 100644 --- a/.github/workflows/client-part-bytes.yml +++ b/.github/workflows/client-part-bytes.yml @@ -68,7 +68,7 @@ jobs: python -m pytest -q tests/test_client_part_bytes.py tests/test_client.py --junitxml=artifacts/client-part-bytes.xml - - uses: actions/upload-artifact@v4 + - uses: actions/upload-artifact@v7.0.1 if: always() with: name: client-part-bytes-evidence diff --git a/.github/workflows/collaboration-list-bounds.yml b/.github/workflows/collaboration-list-bounds.yml index 82f2c388..b45a05a4 100644 --- a/.github/workflows/collaboration-list-bounds.yml +++ b/.github/workflows/collaboration-list-bounds.yml @@ -64,7 +64,7 @@ jobs: tests/test_collaboration_resources.py tests/test_collaboration_resource_surfaces.py tests/test_proposals.py --junitxml=artifacts/work-list-regressions.xml - - uses: actions/upload-artifact@v4 + - uses: actions/upload-artifact@v7.0.1 if: always() with: name: collaboration-list-evidence diff --git a/.github/workflows/git-transport-boundaries.yml b/.github/workflows/git-transport-boundaries.yml index 2f66e02b..9a736678 100644 --- a/.github/workflows/git-transport-boundaries.yml +++ b/.github/workflows/git-transport-boundaries.yml @@ -67,7 +67,7 @@ jobs: - name: Preserve real HTTP, SSH, bundle, LFS and publication fences run: | python -m pytest tests/test_git_http_push.py tests/test_native_git_read_only.py tests/test_git_lfs.py tests/test_lfs_shared_gc_quota.py tests/test_ssh_rss.py tests/test_extensions.py tests/test_storage_layout.py tests/test_effect_completion_fencing.py tests/issue_78_80/test_lfs_integrity.py tests/issue_78_80/test_lfs_migration_contract.py --junitxml=artifacts/git-regressions.xml - - uses: actions/upload-artifact@v4 + - uses: actions/upload-artifact@v7.0.1 if: always() with: name: git-transport-boundaries-${{ github.sha }} diff --git a/.github/workflows/hosting-authority.yml b/.github/workflows/hosting-authority.yml index d5756fc3..1ff159b3 100644 --- a/.github/workflows/hosting-authority.yml +++ b/.github/workflows/hosting-authority.yml @@ -60,7 +60,7 @@ jobs: python -m pip install -e '.[dev]' - name: Exercise revoked keys and restorable ownership rights run: python -m pytest tests/test_hosting_runtime_authority.py --junitxml=artifacts/authority.xml - - uses: actions/upload-artifact@v4 + - uses: actions/upload-artifact@v7.0.1 if: always() with: name: hosting-authority-evidence diff --git a/.github/workflows/hosting-readonly.yml b/.github/workflows/hosting-readonly.yml index 6ed927d9..7b5d68bc 100644 --- a/.github/workflows/hosting-readonly.yml +++ b/.github/workflows/hosting-readonly.yml @@ -67,7 +67,7 @@ jobs: python -m pytest tests/test_hosting*.py tests/test_cli_hosting.py tests/test_extensions.py::test_hosting_publish_is_explicit_versioned_on_service_origin --junitxml=artifacts/hosting.xml - - uses: actions/upload-artifact@v4 + - uses: actions/upload-artifact@v7.0.1 if: always() with: name: hosting-readonly-evidence diff --git a/.github/workflows/integration-regressions.yml b/.github/workflows/integration-regressions.yml index 6d4d96b7..dd494593 100644 --- a/.github/workflows/integration-regressions.yml +++ b/.github/workflows/integration-regressions.yml @@ -92,7 +92,7 @@ jobs: 'requested_database_locale': os.environ['TEST_DATABASE_LOCALE']}, indent=2) + '\n') PY - name: Preserve reviewed public source even when tests fail - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7.0.1 with: name: integration-regression-source-${{ matrix.locale }} path: | @@ -151,7 +151,7 @@ jobs: tests/test_webhook_overlap.py tests/test_webhook_domain_events.py tests/test_webhook_delivery.py tests/test_webhook_identity.py --junitxml=artifacts/integration-regressions.xml - - uses: actions/upload-artifact@v4 + - uses: actions/upload-artifact@v7.0.1 if: always() with: name: integration-regression-evidence-${{ matrix.locale }} diff --git a/.github/workflows/issue-recovery.yml b/.github/workflows/issue-recovery.yml index 97645555..3e6607f2 100644 --- a/.github/workflows/issue-recovery.yml +++ b/.github/workflows/issue-recovery.yml @@ -76,7 +76,7 @@ jobs: tests/test_backup_retirement_cli.py tests/test_recovery_policy_v2.py tests/test_recovery_authority_reconcile.py tests/test_complete_recovery_proof.py --junitxml=artifacts/recovery-current.xml - - uses: actions/upload-artifact@v4 + - uses: actions/upload-artifact@v7.0.1 if: always() with: name: issue-recovery-evidence diff --git a/.github/workflows/lease-list-bounds.yml b/.github/workflows/lease-list-bounds.yml index f9bbac6e..cad6da41 100644 --- a/.github/workflows/lease-list-bounds.yml +++ b/.github/workflows/lease-list-bounds.yml @@ -65,7 +65,7 @@ jobs: tests/test_collaboration_surfaces.py tests/test_collaboration_resources.py tests/test_collaboration_resource_surfaces.py --junitxml=artifacts/lease-list-regressions.xml - - uses: actions/upload-artifact@v4 + - uses: actions/upload-artifact@v7.0.1 if: always() with: name: lease-list-evidence diff --git a/.github/workflows/market-boundaries.yml b/.github/workflows/market-boundaries.yml index 87ee288f..2aadc863 100644 --- a/.github/workflows/market-boundaries.yml +++ b/.github/workflows/market-boundaries.yml @@ -59,7 +59,7 @@ jobs: - name: Preserve all published checkout and recovery semantics run: | python -m pytest tests/test_orders.py tests/test_managed_checkout.py tests/test_checkout_decision_integrity.py tests/test_market_contracts.py tests/test_market_integration.py tests/test_market_recovery.py tests/test_market_account_scope.py tests/test_market_71.py tests/test_market_72.py tests/test_money_core.py tests/test_market_redemption.py tests/test_bounty.py tests/test_store_foundation.py --junitxml=artifacts/market-regressions.xml - - uses: actions/upload-artifact@v4 + - uses: actions/upload-artifact@v7.0.1 if: always() with: name: market-boundaries-${{ github.sha }} diff --git a/.github/workflows/metadata-session-boundary.yml b/.github/workflows/metadata-session-boundary.yml index 5a3bf086..3b446f9e 100644 --- a/.github/workflows/metadata-session-boundary.yml +++ b/.github/workflows/metadata-session-boundary.yml @@ -58,7 +58,7 @@ jobs: - name: Preserve PostgreSQL locks, migration and recovery regressions run: | python -m pytest tests/test_postgres.py tests/test_security_storage_rollback.py tests/test_security_rollback_fences.py tests/test_ledger_accounts.py tests/test_ledger_account_isolation.py tests/test_ledger_migration_json.py tests/test_ledger_migration_references.py tests/test_recovery_safety_units.py tests/test_recovery_metadata.py --junitxml=artifacts/metadata-regressions.xml - - uses: actions/upload-artifact@v4 + - uses: actions/upload-artifact@v7.0.1 if: always() with: name: metadata-session-${{ github.sha }} diff --git a/.github/workflows/python-quality.yml b/.github/workflows/python-quality.yml new file mode 100644 index 00000000..d110442f --- /dev/null +++ b/.github/workflows/python-quality.yml @@ -0,0 +1,67 @@ +name: Python quality +on: + push: + branches: [main] + pull_request: + workflow_dispatch: +permissions: + contents: read +concurrency: + group: python-quality-${{ github.head_ref || github.ref_name }} + cancel-in-progress: true +jobs: + lint: + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@v7 + with: + persist-credentials: false + - uses: actions/setup-python@v7 + with: + python-version: '3.15' + allow-prereleases: true + - name: Install the locked development environment + run: | + python -m pip install uv==0.12.20 + uv lock --check --python "$(command -v python)" + uv sync --locked --extra dev --python "$(command -v python)" + uv pip check + - name: Require zero Ruff errors and warnings, and exact formatting + run: | + mkdir -p artifacts + uv run --locked --extra dev ruff check . --output-format json > artifacts/ruff.json 2> artifacts/ruff.stderr + test ! -s artifacts/ruff.stderr + uv run --locked --extra dev ruff format --check . > artifacts/format.txt 2> artifacts/format.stderr + test ! -s artifacts/format.stderr + uv run --locked --extra dev python -W error -m compileall -q src tests conformance scripts + - name: Record source and toolchain evidence + if: always() + run: | + mkdir -p artifacts + uv run --locked --extra dev python - <<'PYTHON' + import importlib.metadata, json, os, pathlib, platform, subprocess + def git(*args): + return subprocess.check_output(['git', *args], text=True).strip() + pathlib.Path('artifacts/quality-source.json').write_text(json.dumps({ + 'commit': git('rev-parse', 'HEAD'), + 'tree': git('rev-parse', 'HEAD^{tree}'), + 'run_id': os.environ['GITHUB_RUN_ID'], + 'python': platform.python_version(), + 'ruff': importlib.metadata.version('ruff'), + 'uv': subprocess.check_output(['uv', '--version'], text=True).strip(), + 'uv_build_requirement': 'uv_build==0.12.20', + }, indent=2) + '\n') + PYTHON + - name: Preserve a suggested safe Ruff patch after failure + if: failure() + run: | + uv run --locked --extra dev ruff check . --fix --no-unsafe-fixes > artifacts/fix-suggestions.txt 2>&1 || true + uv run --locked --extra dev ruff format . >> artifacts/fix-suggestions.txt 2>&1 + git diff --binary > artifacts/quality-fixes.patch + - uses: actions/upload-artifact@v7.0.1 + if: always() + with: + name: python-quality + path: artifacts/ + if-no-files-found: error diff --git a/.github/workflows/recovery-safety.yml b/.github/workflows/recovery-safety.yml index 2d71e0e2..2ab8562b 100644 --- a/.github/workflows/recovery-safety.yml +++ b/.github/workflows/recovery-safety.yml @@ -69,7 +69,7 @@ jobs: tests/test_backup_drill.py \ tests/test_backup_daemon.py \ --junitxml=artifacts/recovery-safety.xml - - uses: actions/upload-artifact@v4 + - uses: actions/upload-artifact@v7.0.1 if: always() with: name: recovery-safety-verification diff --git a/.github/workflows/storage-session.yml b/.github/workflows/storage-session.yml index b3dd3955..4b6ba9be 100644 --- a/.github/workflows/storage-session.yml +++ b/.github/workflows/storage-session.yml @@ -77,7 +77,7 @@ jobs: tests/test_ledger_migration_references.py tests/test_recovery_safety_units.py tests/test_recovery_metadata.py --junitxml=artifacts/storage-session.xml - - uses: actions/upload-artifact@v4 + - uses: actions/upload-artifact@v7.0.1 if: always() with: name: storage-session-evidence diff --git a/.github/workflows/token-delivery-boundary.yml b/.github/workflows/token-delivery-boundary.yml index 5178b251..5acc125d 100644 --- a/.github/workflows/token-delivery-boundary.yml +++ b/.github/workflows/token-delivery-boundary.yml @@ -74,7 +74,7 @@ jobs: tests/test_credential_recovery_window.py tests/test_client_token_delivery.py tests/test_batch_secret_release.py tests/test_security_patch_regressions.py --junitxml=artifacts/token-delivery-boundary.xml - - uses: actions/upload-artifact@v4 + - uses: actions/upload-artifact@v7.0.1 if: always() with: name: token-delivery-boundary-evidence diff --git a/.github/workflows/transfer-status.yml b/.github/workflows/transfer-status.yml index c37d4f84..d561753a 100644 --- a/.github/workflows/transfer-status.yml +++ b/.github/workflows/transfer-status.yml @@ -73,7 +73,7 @@ jobs: tests/test_transfer_status_cursors.py tests/test_transfer.py tests/test_transfer_endpoint.py tests/test_read_only_evidence.py --junitxml=artifacts/transfer-status.xml - - uses: actions/upload-artifact@v4 + - uses: actions/upload-artifact@v7.0.1 if: always() with: name: transfer-status-evidence diff --git a/.gitignore b/.gitignore index d48dfcff..3c66f7ad 100644 --- a/.gitignore +++ b/.gitignore @@ -15,3 +15,4 @@ dist/ .coverage htmlcov/ .legacy-ledger/ +/artifacts/ diff --git a/README.md b/README.md index 978ca324..5ae103f9 100644 --- a/README.md +++ b/README.md @@ -30,7 +30,7 @@ uv sync ``` 基础安装仅包含签名客户端及传输所需的 Python 依赖，不要求本机运行 PostgreSQL 或 Valkey。 -安装发行包时使用 `python -m pip install msg-lmm-best`。`age` 加密/备份等操作仍需相应系统工具， +安装发行包时使用 `python -m pip install msgctl`。`age` 加密/备份等操作仍需相应系统工具， 并不因为依赖分组而获得空实现或降级执行。 部署完整服务端请使用 `uv sync --extra server`，或在源码目录安装 `python -m pip install '.[server]'`。 diff --git a/conformance/test_token_budget.py b/conformance/test_token_budget.py index faa6e60e..9f3fbe3d 100644 --- a/conformance/test_token_budget.py +++ b/conformance/test_token_budget.py @@ -1,51 +1,74 @@ """Measure actual tokenizer tokens; byte counts are not a token estimate.""" + import json import os from pathlib import Path + import httpx import pytest + +from msg.admin.root import _approve_csr, _provision from msg.application import Application -from msg.admin.root import _provision,_approve_csr +from msg.client import ClientState, MsgClient from msg.config import write_example -from msg.client import ClientState,MsgClient -from msg.core.codec import canonical,wire +from msg.core.codec import canonical, wire from msg.core.executor import result_wire from msg.transports.client import HTTPTransport from msg.transports.http import create_app + @pytest.mark.asyncio -async def test_real_token_and_roundtrip_budget(tmp_path,pg_dsn): +async def test_real_token_and_roundtrip_budget(tmp_path, pg_dsn): import tiktoken - encoding=tiktoken.get_encoding('cl100k_base') - app=Application(write_example(tmp_path/'etc',tmp_path/'data','http://testserver', - postgres_dsn=pg_dsn)) - csr,root=await _provision(app,'token-budget-test-passphrase') - await _approve_csr(app,csr,root,expected_digest=None,operator='conformance-fixture') - records=[] + + encoding = tiktoken.get_encoding('cl100k_base') + app = Application( + write_example(tmp_path / 'etc', tmp_path / 'data', 'http://testserver', postgres_dsn=pg_dsn) + ) + csr, root = await _provision(app, 'token-budget-test-passphrase') + await _approve_csr(app, csr, root, expected_digest=None, operator='conformance-fixture') + records = [] try: - async with httpx.AsyncClient(transport=httpx.ASGITransport(app=create_app(app)),base_url=app.settings.service_url) as http: - transport=HTTPTransport(app.settings.service_url,http=http) - client=MsgClient(ClientState(tmp_path/'client',server=app.settings.service_url),transport) + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=create_app(app)), base_url=app.settings.service_url + ) as http: + transport = HTTPTransport(app.settings.service_url, http=http) + client = MsgClient( + ClientState(tmp_path / 'client', server=app.settings.service_url), transport + ) await client.register('budget-agent') - async def measured(name,args,budget=900): - before=transport.calls - packet=client.prepare(name,args) - result=client.checked(await client.send(packet)) - text=canonical(result_wire(result)).decode() - request_text=canonical(packet).decode() - record={'operation':name,'input_tokens':len(encoding.encode(request_text)), - 'output_tokens':len(encoding.encode(text)),'output_bytes':len(text.encode()), - 'roundtrips':transport.calls-before} - assert record['roundtrips']==1,record - assert record['output_tokens']<=budget,record + + async def measured(name, args, budget=900): + before = transport.calls + packet = client.prepare(name, args) + result = client.checked(await client.send(packet)) + text = canonical(result_wire(result)).decode() + request_text = canonical(packet).decode() + record = { + 'operation': name, + 'input_tokens': len(encoding.encode(request_text)), + 'output_tokens': len(encoding.encode(text)), + 'output_bytes': len(text.encode()), + 'roundtrips': transport.calls - before, + } + assert record['roundtrips'] == 1, record + assert record['output_tokens'] <= budget, record records.append(record) return result - post=await measured('content.post_create',{'parent':'/main','body':'A small coordination message.'}) - await measured('discussion.reply',{'target':wire(post.resources[0]),'body':'Received; ready for the next step.'}) - await measured('discussion.thread',{'id':post.resources[0].id},budget=2500) - await measured('transfer.open',{'direction':'upload','size':0}) - target=Path(os.getenv('MSG_BENCHMARK_PATH',str(tmp_path/'token-budget.json'))) - target.parent.mkdir(parents=True,exist_ok=True) - target.write_text(json.dumps({'tokenizer':'cl100k_base','measurements':records},indent=2)) + + post = await measured( + 'content.post_create', {'parent': '/main', 'body': 'A small coordination message.'} + ) + await measured( + 'discussion.reply', + {'target': wire(post.resources[0]), 'body': 'Received; ready for the next step.'}, + ) + await measured('discussion.thread', {'id': post.resources[0].id}, budget=2500) + await measured('transfer.open', {'direction': 'upload', 'size': 0}) + target = Path(os.getenv('MSG_BENCHMARK_PATH', str(tmp_path / 'token-budget.json'))) + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text( + json.dumps({'tokenizer': 'cl100k_base', 'measurements': records}, indent=2) + ) finally: await app.close() diff --git a/conformance/test_transports.py b/conformance/test_transports.py index 60f0cd6d..399e9fee 100644 --- a/conformance/test_transports.py +++ b/conformance/test_transports.py @@ -3,125 +3,211 @@ No importorskip, fake GraphQL adapter, or expected-failure marker is used here. The local delivery environment cannot execute the GraphQL case; CI must do so. """ + import asyncio import os -from pathlib import Path import socket import sys +from pathlib import Path from uuid import uuid4 -import httpx + import pytest import uvicorn + +from msg.admin.root import _approve_csr, _provision from msg.application import Application -from msg.admin.root import _provision, _approve_csr -from msg.config import write_example from msg.client import ClientState, MsgClient -from msg.core.codec import b64,unb64,canonical,digest,decode,loads,wire -from msg.core.models import OperationResult +from msg.config import write_example +from msg.core.codec import b64, canonical, digest, loads, unb64 from msg.transports.client import TRANSPORTS, HTTPTransport from msg.transports.http import create_app def test_release_dependencies_are_available(): - assert sys.version_info[:2]>=(3,15), 'Release verification requires Python 3.15' + assert sys.version_info[:2] >= (3, 15), 'Release verification requires Python 3.15' import graphql import tiktoken + assert graphql.__version__ and tiktoken.__version__ @pytest.mark.asyncio -@pytest.mark.parametrize('mode',['http','path_get','graphql','mcp_http','cli','mcp_stdio']) -async def test_transfer_conformance_for_every_transport(mode,tmp_path,pg_dsn): - sock=socket.socket();sock.bind(('127.0.0.1',0));sock.setblocking(False) - url='http://127.0.0.1:'+str(sock.getsockname()[1]) - app=Application(write_example(tmp_path/'etc',tmp_path/'data',url,postgres_dsn=pg_dsn)) - csr,root=await _provision(app,'conformance-test-passphrase') - await _approve_csr(app,csr,root,expected_digest=None,operator='conformance-fixture') - server=uvicorn.Server(uvicorn.Config(create_app(app),log_level='critical',access_log=False,ws='none')) - serving=asyncio.create_task(server.serve(sockets=[sock])) +@pytest.mark.parametrize('mode', ['http', 'path_get', 'graphql', 'mcp_http', 'cli', 'mcp_stdio']) +async def test_transfer_conformance_for_every_transport(mode, tmp_path, pg_dsn): + sock = socket.socket() + sock.bind(('127.0.0.1', 0)) + sock.setblocking(False) + url = 'http://127.0.0.1:' + str(sock.getsockname()[1]) + app = Application(write_example(tmp_path / 'etc', tmp_path / 'data', url, postgres_dsn=pg_dsn)) + csr, root = await _provision(app, 'conformance-test-passphrase') + await _approve_csr(app, csr, root, expected_digest=None, operator='conformance-fixture') + server = uvicorn.Server( + uvicorn.Config(create_app(app), log_level='critical', access_log=False, ws='none') + ) + serving = asyncio.create_task(server.serve(sockets=[sock])) for _ in range(100): - if server.started:break - await asyncio.sleep(.02) + if server.started: + break + await asyncio.sleep(0.02) assert server.started - state=ClientState(tmp_path/'client',server=url) - transport=TRANSPORTS.get(mode,HTTPTransport)(url) - client=MsgClient(state,transport) - peer_transport=HTTPTransport(url) - peer=MsgClient(state,peer_transport) + state = ClientState(tmp_path / 'client', server=url) + transport = TRANSPORTS.get(mode, HTTPTransport)(url) + client = MsgClient(state, transport) + peer_transport = HTTPTransport(url) + peer = MsgClient(state, peer_transport) await peer.register('conformance-agent') - process=None - environment={**os.environ,'PYTHONPATH':str(Path(__file__).parents[1]/'src')} - command=[sys.executable,'-m','msg.cli','--config-dir',str(state.directory),'--server',url] - if mode=='mcp_stdio': - process=await asyncio.create_subprocess_exec(*command,'mcp',env=environment, - stdin=asyncio.subprocess.PIPE,stdout=asyncio.subprocess.PIPE,stderr=asyncio.subprocess.PIPE) - process.stdin.write(canonical({'jsonrpc':'2.0','id':'init','method':'initialize', - 'params':{'protocolVersion':'2025-11-25'}})+b'\n') + process = None + environment = {**os.environ, 'PYTHONPATH': str(Path(__file__).parents[1] / 'src')} + command = [ + sys.executable, + '-m', + 'msg.cli', + '--config-dir', + str(state.directory), + '--server', + url, + ] + if mode == 'mcp_stdio': + process = await asyncio.create_subprocess_exec( + *command, + 'mcp', + env=environment, + stdin=asyncio.subprocess.PIPE, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + process.stdin.write( + canonical({ + 'jsonrpc': '2.0', + 'id': 'init', + 'method': 'initialize', + 'params': {'protocolVersion': '2025-11-25'}, + }) + + b'\n' + ) await process.stdin.drain() - assert 'result' in loads(await asyncio.wait_for(process.stdout.readline(),15)) - async def invoke(operation,args,rid=None): - rid=rid or uuid4().hex - if mode=='cli': - child=await asyncio.create_subprocess_exec(*command,'call',operation,canonical(args).decode(), - '--request-id',rid,env=environment,stdout=asyncio.subprocess.PIPE,stderr=asyncio.subprocess.PIPE) - stdout,stderr=await asyncio.wait_for(child.communicate(),20) - assert child.returncode in {0,1},stderr.decode() + assert 'result' in loads(await asyncio.wait_for(process.stdout.readline(), 15)) + + async def invoke(operation, args, rid=None): + rid = rid or uuid4().hex + if mode == 'cli': + child = await asyncio.create_subprocess_exec( + *command, + 'call', + operation, + canonical(args).decode(), + '--request-id', + rid, + env=environment, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + stdout, stderr = await asyncio.wait_for(child.communicate(), 20) + assert child.returncode in {0, 1}, stderr.decode() return loads(stdout) - if mode=='mcp_stdio': - process.stdin.write(canonical({'jsonrpc':'2.0','id':rid,'method':'tools/call','params':{ - 'name':operation,'arguments':args,'_meta':{'msg/request_id':rid}}})+b'\n') + if mode == 'mcp_stdio': + process.stdin.write( + canonical({ + 'jsonrpc': '2.0', + 'id': rid, + 'method': 'tools/call', + 'params': { + 'name': operation, + 'arguments': args, + '_meta': {'msg/request_id': rid}, + }, + }) + + b'\n' + ) await process.stdin.drain() - result=loads(await asyncio.wait_for(process.stdout.readline(),15)) - assert 'result' in result,result + result = loads(await asyncio.wait_for(process.stdout.readline(), 15)) + assert 'result' in result, result return result['result']['structuredContent'] - result=await client.call(operation,args,request_id=rid) + result = await client.call(operation, args, request_id=rid) from msg.core.executor import result_wire + return result_wire(result) + try: - data=b'UTF8-exact\r\n'+bytes(range(128)) - opened=await invoke('transfer.open',{'direction':'upload','size':len(data),'digest':digest(data),'requested_part_bytes':97}) - assert opened['status']=='ok',opened - tid=opened['data']['transfer_id'] - chunks=[(offset,data[offset:offset+97]) for offset in range(0,len(data),97)] - offset,part=chunks[-1] - args={'transfer_id':tid,'offset':offset,'data':b64(part),'digest':digest(part)} - first=await invoke('transfer.part_put',args,'same-chunk') - again=await invoke('transfer.part_put',args,'same-chunk') - assert first['status']=='ok' and again['replayed'] - bad=await invoke('transfer.part_put',{**args,'digest':digest(b'wrong')}) - assert bad['status']=='error' - incomplete=await invoke('transfer.seal',{'transfer_id':tid,'final_size':len(data),'final_digest':digest(data)}) - assert incomplete['status']=='error' - for offset,part in reversed(chunks[:-1]): - piece={'transfer_id':tid,'offset':offset,'data':b64(part),'digest':digest(part)} + data = b'UTF8-exact\r\n' + bytes(range(128)) + opened = await invoke( + 'transfer.open', + { + 'direction': 'upload', + 'size': len(data), + 'digest': digest(data), + 'requested_part_bytes': 97, + }, + ) + assert opened['status'] == 'ok', opened + tid = opened['data']['transfer_id'] + chunks = [(offset, data[offset : offset + 97]) for offset in range(0, len(data), 97)] + offset, part = chunks[-1] + args = {'transfer_id': tid, 'offset': offset, 'data': b64(part), 'digest': digest(part)} + first = await invoke('transfer.part_put', args, 'same-chunk') + again = await invoke('transfer.part_put', args, 'same-chunk') + assert first['status'] == 'ok' and again['replayed'] + bad = await invoke('transfer.part_put', {**args, 'digest': digest(b'wrong')}) + assert bad['status'] == 'error' + incomplete = await invoke( + 'transfer.seal', + {'transfer_id': tid, 'final_size': len(data), 'final_digest': digest(data)}, + ) + assert incomplete['status'] == 'error' + for offset, part in reversed(chunks[:-1]): + piece = { + 'transfer_id': tid, + 'offset': offset, + 'data': b64(part), + 'digest': digest(part), + } # Recover the same subject-bound session using another real adapter. - if offset==0: - assert (await peer.call('transfer.part_put',piece)).status=='ok' + if offset == 0: + assert (await peer.call('transfer.part_put', piece)).status == 'ok' else: - assert (await invoke('transfer.part_put',piece))['status']=='ok' - status=await invoke('transfer.status',{'transfer_id':tid}) - assert status['status']=='ok' - sealed=await invoke('transfer.seal',{'transfer_id':tid,'final_size':len(data),'final_digest':digest(data)}) - assert sealed['status']=='ok',sealed - repeated=await invoke('transfer.seal',{'transfer_id':tid,'final_size':len(data),'final_digest':digest(data)}) - assert repeated['output']==sealed['output'] - download=await invoke('transfer.open',{'direction':'download','target':sealed['output'],'requested_part_bytes':83}) - assert download['status']=='ok' - restored=bytearray() - for offset in range(0,len(data),83): - result=await invoke('transfer.part_get',{'transfer_id':download['data']['transfer_id'],'offset':offset,'length':min(83,len(data)-offset)}) - assert result['status']=='ok',result - chunk=unb64(result['data']['data']) - assert digest(chunk)==result['data']['chunk']['content']['digest'] + assert (await invoke('transfer.part_put', piece))['status'] == 'ok' + status = await invoke('transfer.status', {'transfer_id': tid}) + assert status['status'] == 'ok' + sealed = await invoke( + 'transfer.seal', + {'transfer_id': tid, 'final_size': len(data), 'final_digest': digest(data)}, + ) + assert sealed['status'] == 'ok', sealed + repeated = await invoke( + 'transfer.seal', + {'transfer_id': tid, 'final_size': len(data), 'final_digest': digest(data)}, + ) + assert repeated['output'] == sealed['output'] + download = await invoke( + 'transfer.open', + {'direction': 'download', 'target': sealed['output'], 'requested_part_bytes': 83}, + ) + assert download['status'] == 'ok' + restored = bytearray() + for offset in range(0, len(data), 83): + result = await invoke( + 'transfer.part_get', + { + 'transfer_id': download['data']['transfer_id'], + 'offset': offset, + 'length': min(83, len(data) - offset), + }, + ) + assert result['status'] == 'ok', result + chunk = unb64(result['data']['data']) + assert digest(chunk) == result['data']['chunk']['content']['digest'] restored.extend(chunk) - assert bytes(restored)==data and digest(bytes(restored))==digest(data) - cancelled=await invoke('transfer.cancel',{'transfer_id':download['data']['transfer_id']}) - assert cancelled['status']=='ok' + assert bytes(restored) == data and digest(bytes(restored)) == digest(data) + cancelled = await invoke( + 'transfer.cancel', {'transfer_id': download['data']['transfer_id']} + ) + assert cancelled['status'] == 'ok' finally: if process: process.stdin.close() - await asyncio.wait_for(process.wait(),15) - await transport.close();await peer_transport.close() - server.should_exit=True + await asyncio.wait_for(process.wait(), 15) + await transport.close() + await peer_transport.close() + server.should_exit = True await serving sock.close() diff --git a/conftest.py b/conftest.py index 0a5a6428..a74ac741 100644 --- a/conftest.py +++ b/conftest.py @@ -15,43 +15,60 @@ from psycopg import sql -@pytest.fixture(scope="session") +@pytest.fixture(scope='session') def pg_cluster(): - configured = os.environ.get("MSG_TEST_POSTGRES_URL_TEMPLATE") + configured = os.environ.get('MSG_TEST_POSTGRES_URL_TEMPLATE') if configured: yield configured return - root = Path(tempfile.mkdtemp(prefix="msg-test-pg-", dir="/tmp")) - data = root / "data" - socket = root / "socket" + root = Path(tempfile.mkdtemp(prefix='msg-test-pg-', dir='/tmp')) + data = root / 'data' + socket = root / 'socket' socket.mkdir() try: subprocess.run( - ["initdb", "-D", str(data), "-A", "trust", "--no-instructions"], - check=True, capture_output=True, text=True, + ['initdb', '-D', str(data), '-U', 'msgtest', '-A', 'trust', '--no-instructions'], + check=True, + capture_output=True, + text=True, ) subprocess.run( - ["pg_ctl", "-D", str(data), "-l", str(root / "server.log"), - "-o", f"-k {socket} -h '' -p 5432", "-w", "start"], - check=True, capture_output=True, text=True, + [ + 'pg_ctl', + '-D', + str(data), + '-l', + str(root / 'server.log'), + '-o', + f"-k {socket} -h '' -p 5432", + '-w', + 'start', + ], + check=True, + capture_output=True, + text=True, ) - yield f'postgresql://lightjunction@localhost:5432/{{database}}?host={quote(str(socket),safe="")}' + yield f'postgresql://msgtest@localhost:5432/{{database}}?host={quote(str(socket), safe="")}' finally: if data.exists(): subprocess.run( - ["pg_ctl", "-D", str(data), "-m", "immediate", "-w", "stop"], - check=False, capture_output=True, text=True, + ['pg_ctl', '-D', str(data), '-m', 'immediate', '-w', 'stop'], + check=False, + capture_output=True, + text=True, ) shutil.rmtree(root) @pytest.fixture def pg_dsn(pg_cluster): - name = "msg_test_" + uuid.uuid4().hex - with psycopg.connect(pg_cluster.format(database="postgres"), autocommit=True) as connection: - connection.execute(sql.SQL("CREATE DATABASE {}").format(sql.Identifier(name))) + name = 'msg_test_' + uuid.uuid4().hex + with psycopg.connect(pg_cluster.format(database='postgres'), autocommit=True) as connection: + connection.execute(sql.SQL('CREATE DATABASE {}').format(sql.Identifier(name))) try: yield pg_cluster.format(database=name) finally: - with psycopg.connect(pg_cluster.format(database="postgres"), autocommit=True) as connection: - connection.execute(sql.SQL("DROP DATABASE {} WITH (FORCE)").format(sql.Identifier(name))) + with psycopg.connect(pg_cluster.format(database='postgres'), autocommit=True) as connection: + connection.execute( + sql.SQL('DROP DATABASE {} WITH (FORCE)').format(sql.Identifier(name)) + ) diff --git a/docs/CLIENT_INSTALLATION.md b/docs/CLIENT_INSTALLATION.md index 8b013e83..e9e4b43a 100644 --- a/docs/CLIENT_INSTALLATION.md +++ b/docs/CLIENT_INSTALLATION.md @@ -4,9 +4,9 @@ ## 安装角色 -仅连接远程 msg 服务的客户端，源码安装使用 `python -m pip install .` 或 `uv sync`。安装已经构建的发行文件时，使用 `python -m pip install /path/to/msg_lmm_best--py3-none-any.whl`。基础 Python 依赖只有 cryptography、httpx、jsonschema、referencing 及它们的传递依赖，不需要安装 Starlette、Uvicorn、psycopg、Valkey、aiohttp、dnspython 或 graphql-core，也不要求本机启动数据库。 +仅连接远程 msg 服务的客户端，源码安装使用 `python -m pip install .` 或 `uv sync`。安装已经构建的发行文件时，使用 `python -m pip install /path/to/msgctl--py3-none-any.whl`。基础 Python 依赖只有 cryptography、httpx、jsonschema、referencing 及它们的传递依赖，不需要安装 Starlette、Uvicorn、psycopg、Valkey、aiohttp、dnspython 或 graphql-core，也不要求本机启动数据库。 -完整服务端使用 `python -m pip install '.[server]'` 或 `uv sync --extra server`。发行包安装使用 `python -m pip install 'msg-lmm-best[server]'`，应选用已发布且与服务端部署匹配的版本。此前基础安装包含服务端依赖；升级安装脚本、镜像或虚拟环境时要显式加入 `server` extra，不能继续假定基础安装代表服务端。现有 `.[dev]` 仍拉入完整服务端依赖，原全量 CI 安装路径不变。 +完整服务端使用 `python -m pip install '.[server]'` 或 `uv sync --extra server`。发行包安装使用 `python -m pip install 'msgctl[server]'`，应选用已发布且与服务端部署匹配的版本。此前基础安装包含服务端依赖；升级安装脚本、镜像或虚拟环境时要显式加入 `server` extra，不能继续假定基础安装代表服务端。现有 `.[dev]` 仍拉入完整服务端依赖，原全量 CI 安装路径不变。 `msg`、`msgd` 两个命令入口都保留。`--help` 不要求服务端依赖。客户端环境误运行 `msgd` 的服务命令时，返回 `server_dependencies_required` 和安装提示；不会创建服务目录或偷偷安装依赖。安装 extra 只安装 Python 包，不启动数据库、不部署服务、不授予证书权限。`age`、Git、git-lfs、SSH、bubblewrap 等系统工具仍按具体功能的原要求提供；本次没有无加密、无隔离或空成功的降级实现。 diff --git a/docs/DEPLOYMENT.md b/docs/DEPLOYMENT.md index 5aed900e..9c71d5b0 100644 --- a/docs/DEPLOYMENT.md +++ b/docs/DEPLOYMENT.md @@ -33,7 +33,7 @@ service_keys = "/var/lib/msgd/service" sudo install -d -m 0755 /opt/msgd sudo /absolute/path/to/verified/python3.15 -m venv /opt/msgd/venv # 下面的锁生成只在验收环境运行；目录需由构建操作者可写。 -printf '%s\n' 'msg-lmm-best[server] @ file:///protected/staged/msg_lmm_best-1.0.0a1-py3-none-any.whl' > /protected/staged/server.in +printf '%s\n' 'msgctl[server] @ file:///protected/staged/msgctl-0.1.0a1-py3-none-any.whl' > /protected/staged/server.in uv pip compile --python /opt/msgd/venv/bin/python --generate-hashes \ /protected/staged/server.in --output-file /protected/staged/server.lock sudo uv pip sync --python /opt/msgd/venv/bin/python --require-hashes /protected/staged/server.lock diff --git a/docs/LOCAL_VERIFICATION.md b/docs/LOCAL_VERIFICATION.md new file mode 100644 index 00000000..03adf4a8 --- /dev/null +++ b/docs/LOCAL_VERIFICATION.md @@ -0,0 +1,62 @@ +# 在本地验证未提交代码 + +验证脚本不提交、不推送、不触发 GitHub Actions，不部署生产。 +它会运行项目代码和隔离测试；数据库、Valkey 必须是可销毁的本机测试实例， +不能填写生产连接。源码记录使用未提交文件的真实内容指纹，而不把 HEAD 当作测试源码。 + +## 准备 + +使用 Python 3.15，uv 0.12.20。安装锁定的开发依赖： + +```sh +uv sync --locked --extra dev --python 3.15 +``` + +系统工具包括 PostgreSQL 客户端、initdb/pg_ctl（自行启动临时集群时）、 +Git/Git-LFS、age、OpenSSL、Nginx、bubblewrap。以普通用户运行。 +本机 bwrap 必须能够启动隔离子进程，不能为了通过测试更改生产安全策略。 + +`MSG_TEST_POSTGRES_URL_TEMPLATE` 可指向专用本机测试 PostgreSQL， +库名位置保留 `{database}`，测试将创建并清理随机数据库。 +不提供该变量时，由 fixture 使用 initdb/pg_ctl 启动仅 Unix socket 可访问的临时集群。 +`MSG_TEST_VALKEY_URL` 必须指向专用本机测试 Valkey，不能复用生产缓存实例。 + +旧版本测试需要准确的固定源码，本工作区已附 `.legacy-ledger`； +在其他 checkout 中可以仅从已有 Git 对象恢复，无需提交： + +```sh +mkdir -p .legacy-ledger +git archive ac083b3666b1b5268d90115129e2cc4e0dd11521 | tar -x -C .legacy-ledger +``` + +也可通过 `MSG_TEST_LEGACY_SOURCE` 指定等价目录。 +预检与既有迁移测试会校验 manifest 中的文件哈希。 + +## 运行 + +```sh +uv run --locked --extra dev --python 3.15 python scripts/verify_local.py --preflight-only +uv run --locked --extra dev --python 3.15 python scripts/verify_local.py --shards 8 +``` + +日志位于 `artifacts/local-<时间>/`。目录必须全新，不覆盖旧证据。 +本地八个分片默认串行，避免共享一个 Valkey 时出现跨分片干扰。 +脚本复用 `ci_shards.py` 的实际 pytest node-ID 并集检查，不用测试数量代替身份校验。 +之后执行 conformance、打包校验和独立客户端 wheel 安装/签名传输检查。 +任何阶段失败即返回非零状态；单独预检成功也不会标记整个验证通过。 + +`result.json` 包含基线 commit、dirty 状态、实际文件清单/摘要、命令、日志和退出码。 +验证前后源码改变会使整个结果失败。所有这些结果仍然不等于生产部署验收。 + +## 本次受限环境中的补充测试 + +当前会话只有 Python 3.13，不能运行目标版本业务模块。实际通过的是以下 +工具与配置测试，不加载 PostgreSQL 全局 fixture，也不代替上面的完整命令： + +```sh +PYTEST_DISABLE_PLUGIN_AUTOLOAD=1 python -m pytest --noconftest \ + -p pytest_asyncio.plugin \ + tests/test_release_version.py tests/test_release_artifacts.py \ + tests/test_ci_shards.py tests/test_local_verification.py \ + tests/test_local_postgres_fixtures.py +``` diff --git a/docs/NEW_ROOT_LEGACY_IMPORT_RUNBOOK.md b/docs/NEW_ROOT_LEGACY_IMPORT_RUNBOOK.md index 82b8df82..a8f86f21 100644 --- a/docs/NEW_ROOT_LEGACY_IMPORT_RUNBOOK.md +++ b/docs/NEW_ROOT_LEGACY_IMPORT_RUNBOOK.md @@ -50,29 +50,38 @@ from msg.client import ClientState, MsgClient from msg.transports.client import HTTPTransport from msg.transports.http import create_app + async def main(): app = Application(load_settings(Path(os.environ['CFG']))) try: await app.load() state = ClientState(Path(os.environ['CLIENT']), server=app.settings.service_url) - async with httpx.AsyncClient(transport=httpx.ASGITransport(app=create_app(app)), - base_url=app.settings.service_url) as http: + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=create_app(app)), base_url=app.settings.service_url + ) as http: client = MsgClient(state, HTTPTransport(state.server, http=http)) registered = await client.register(os.environ['NEW_HANDLE']) assert registered.status == 'ok', registered.error print('HOLDER', state.subject) for name in ('legacy-content', 'legacy-git', 'legacy-identities'): - result = await client.call('content.topic_create', {'parent':'/main','name':name}) + result = await client.call( + 'content.topic_create', {'parent': '/main', 'name': name} + ) assert result.status == 'ok', result.error ref = result.resources[0] async with app.metadata.transaction(write=False) as tx: generation = (await tx.resource(ref.id)).generation - result = await client.call('content.chmod', {'id':ref.id,'mode':'0700'}, - expected=((ref.id, generation),)) + result = await client.call( + 'content.chmod', + {'id': ref.id, 'mode': '0700'}, + expected=((ref.id, generation),), + ) assert result.status == 'ok', result.error print(name, ref.id) finally: await app.close() + + asyncio.run(main()) ``` @@ -141,47 +150,70 @@ from msg.config import load_settings from msg.core.codec import digest from msg.admin.legacy_approval import write_new + async def main(): - snapshot = Path(os.environ['MATERIALS'])/'offline-legacy-0221.sqlite' + snapshot = Path(os.environ['MATERIALS']) / 'offline-legacy-0221.sqlite' sha = os.environ['SQLITE_SHA256'] with snapshot.open('rb') as stream: assert hashlib.file_digest(stream, 'sha256').hexdigest() == sha - with sqlite3.connect(snapshot.resolve().as_uri()+'?mode=ro', uri=True) as db: + with sqlite3.connect(snapshot.resolve().as_uri() + '?mode=ro', uri=True) as db: db.execute('PRAGMA query_only=ON') candidates = [ (table, row[0], mode) for table, query, mode in ( - ('posts','SELECT id FROM posts ORDER BY id','0644'), - ('attachments','SELECT a.id FROM attachments a JOIN posts p ON p.id=a.post_id ORDER BY a.id','0644'), - ('boards','SELECT name FROM boards ORDER BY name','0755')) + ('posts', 'SELECT id FROM posts ORDER BY id', '0644'), + ( + 'attachments', + 'SELECT a.id FROM attachments a JOIN posts p ON p.id=a.post_id ORDER BY a.id', + '0644', + ), + ('boards', 'SELECT name FROM boards ORDER BY name', '0755'), + ) for row in db.execute(query) ] app = Application(load_settings(Path(os.environ['CFG']))) try: await app.load() async with app.metadata.transaction(write=False) as tx: - report = tx.setting('legacy-import:'+sha) + report = tx.setting('legacy-import:' + sha) assert report and report['source_sha256'] == sha entries = [] for table, old_id, mode in candidates: - rid = 'legacy_'+sha[:24]+'_'+digest((table,old_id))[7:31] + rid = 'legacy_' + sha[:24] + '_' + digest((table, old_id))[7:31] resource = await tx.resource(rid) assert resource.owner == os.environ['HOLDER'] and resource.state == 'active' - entries.append({'id':rid,'generation':resource.generation,'mode':mode,'source_table':table}) + entries.append({ + 'id': rid, + 'generation': resource.generation, + 'mode': mode, + 'source_table': table, + }) parent = await tx.resource(os.environ['CONTENT_PARENT']) assert parent.owner == os.environ['HOLDER'] and parent.mode == 0o700 for item in entries: assert parent.id in {r.id for r in await tx.ancestors(item['id'])} - entries.append({'id':parent.id,'generation':parent.generation,'mode':'0755','source_table':'publication_parent_last'}) - plan = {'source_sha256':sha,'holder':os.environ['HOLDER'],'entries':entries, - 'restores':'read-only-public-candidates','review_required':True, - 'source_code_commit':'e12eea32764656eec82452fee111e5b4fb6d8a42', - 'classification':'application-anonymous-read-candidate', - 'deployment_acl_review':None} - write_new(Path(os.environ['PUBLICATION_PLAN']),plan) - print({'candidate_count':len(entries)-1,'plan_digest':digest(plan)}) + entries.append({ + 'id': parent.id, + 'generation': parent.generation, + 'mode': '0755', + 'source_table': 'publication_parent_last', + }) + plan = { + 'source_sha256': sha, + 'holder': os.environ['HOLDER'], + 'entries': entries, + 'restores': 'read-only-public-candidates', + 'review_required': True, + 'source_code_commit': 'e12eea32764656eec82452fee111e5b4fb6d8a42', + 'classification': 'application-anonymous-read-candidate', + 'deployment_acl_review': None, + } + write_new(Path(os.environ['PUBLICATION_PLAN']), plan) + print({'candidate_count': len(entries) - 1, 'plan_digest': digest(plan)}) finally: await app.close() + + asyncio.run(main()) ``` diff --git a/docs/QUALITY_PROGRESS.md b/docs/QUALITY_PROGRESS.md new file mode 100644 index 00000000..872f9cd1 --- /dev/null +++ b/docs/QUALITY_PROGRESS.md @@ -0,0 +1,65 @@ +# 本地质量收口 · 2026-09-30 + +## 当前状态 + +本地工作区基于已经合并的 main： +`25cf518a64ca2b6af820367caa4f5bb46dbc2f02`， +原始 tree 为 `61cc47b7abc25ccdcf163d59804e323ea6a10649`。 +已从已有 Git bundle 恢复并核对远端已发布 commit/tree 的准确哈希。 +本轮没有创建业务提交、推送、触发云端工作流或执行生产部署。 +所有代码修改均留在工作区，HEAD 仍是以上基线。 + +此前质量候选只作为本地修复来源，没有套用相对旧 head 的补丁： +先重建准确 main，再重新生成相对该 main 的差异。 +Ruff 的自动修复/格式化来源是此前真实云端运行；后续人工修改 +和本轮修改尚未经过最终 Ruff 0.16.9 验证，不能标记为零诊断。 + +## 本轮新增修复 + +- `src/msg/__init__.py` 的运行时版本从错误的 `1.0.0a1` 对齐至发行版本 `0.1.0a1`。 +- 构建校验从仅检查资源/命令名称扩展到完整包文件集合、逐字节 Python 源码、 + Name/Version/Requires-Python、准确 console target、sdist 当前源码和锁文件。 + 旧代码、额外的已移除模块和错误元数据不能再被校验脚本当作合格产物。 +- 本地 PostgreSQL 测试显式创建并使用 `msgtest` 临时角色，不依赖开发者的系统用户名。 +- 安装测试 fixture 使用 finally 回收应用和临时数据库；初始化失败也清理， + session 级 seed 数据库有明确 teardown，不再永久残留在配置的测试集群中。 +- `scripts/verify_local.py` 提供本地检查入口，记录实际未提交文件指纹、退出码与日志； + 缺依赖、非零退出、要求清空的 stderr、跳过的 conformance、源码漂移都不会标为通过。 +- uv 的版本要求写入 `pyproject.toml`；旧 uv 的真实拒绝结果已记录。 +- 收拢重复的显式 import，保持导入源、名称、别名及绑定顺序，减少 695 行展开代码。 + +## 实际运行结果及范围 + +现有可执行环境是 Python 3.13.5、uv 0.10.0，不是目标环境。 +新增发布回归先得到 8 个失败/2 个通过；修复后通过。 +新增 fixture 生命周期回归先得到 4 个失败；修复后通过。 + +最终本地补充回归：54 passed，0 failed / 0 error / 0 skipped。 +包含 29 个既有 CI 分片校验测试和 25 个新增工具/配置/fixture 编排测试。 +测试通过 `--noconftest` 运行指定的五个不依赖真实数据库的测试文件； +fixture 生命周期测试使用模拟进程/应用检查编排，不是 PostgreSQL 实测。 +产物校验测试使用合成档案，不是本项目真实 wheel/sdist 构建成功的证据。 +没有把这些结果当作 Python 3.15 全量业务测试。 + +附加静态源码审查没有发现未绑定的全局名称或额外语法错误。 +该审查仅为了让 Python 3.13 的解析器读取源码，临时规范化了 +18 个文件中的 Python 3.14+ except 语法，没有修改生产文件，也没有运行这些模块。 +它不代替 Ruff、目标版本编译或业务测试。 + +固定旧版源代码 `ac083b3666b1b5268d90115129e2cc4e0dd11521` +已恢复至本地 `.legacy-ledger`，按既有 manifest 验证了 8 个文件的 SHA-256； +没有因此宣称数据库迁移/回滚已执行。 + +## 仍未完成 + +本环境没有 Python 3.15、Ruff、build、psycopg、graphql-core、valkey、 +tiktoken 和若干系统测试工具。pip/uv 下载及 GitHub 域名解析未成功。 +`verify_local.py --preflight-only` 实际退出 1，结果是 blocked。 + +因此最终 Ruff/format、锁定环境安装、Python 3.15 全量测试、真实 +PostgreSQL/Valkey/Git-LFS/age/bwrap 验收、conformance/tokenizer、 +真实 wheel/sdist 和干净安装仍未完成。当前不能标记为可部署或全部完成。 +生产日志、旧库、Root、本机控制台、恢复授权和备份退役仍有各自验收边界。 + +本地执行入口及步骤见 `LOCAL_VERIFICATION.md`。会话交付包的 evidence/ +保存真实日志、JUnit 和环境阻塞记录；不借用此前其他提交的 CI 成功。 diff --git a/pyproject.toml b/pyproject.toml index 6eae7358..e538fbb2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -41,6 +41,9 @@ dev = [ "tiktoken>=0.12", ] +[tool.uv] +required-version = "==0.12.20" + [tool.uv.build-backend] module-name = "msg" source-include = [ @@ -74,5 +77,8 @@ select = [ ] explicit-preview-rules = true +[tool.ruff.lint.isort] +combine-as-imports = true + [tool.ruff.format] quote-style = "single" diff --git a/scripts/check_client_install.py b/scripts/check_client_install.py index f12ed662..308f9177 100644 --- a/scripts/check_client_install.py +++ b/scripts/check_client_install.py @@ -3,6 +3,7 @@ Run with --minimal-install in a fresh wheel-only virtualenv, outside the source checkout. Network traffic uses httpx.MockTransport, not a remote service. """ + from __future__ import annotations import argparse @@ -11,15 +12,24 @@ import importlib.abc import importlib.util import json -from pathlib import Path import stat import sys import tempfile +from pathlib import Path SERVER_DEPENDENCIES = ('starlette', 'uvicorn', 'psycopg', 'valkey', 'aiohttp', 'dns', 'graphql') -FORBIDDEN = ('msg.application', 'msg.core.executor', 'msg.storage', 'msg.admin', - 'msg.workers', 'msg.security.vault', 'msg.security.custodial_migration', - 'msg.transports.http', 'msg.transports.graphql', 'msg.transports.mcp') +FORBIDDEN = ( + 'msg.application', + 'msg.core.executor', + 'msg.storage', + 'msg.admin', + 'msg.workers', + 'msg.security.vault', + 'msg.security.custodial_migration', + 'msg.transports.http', + 'msg.transports.graphql', + 'msg.transports.mcp', +) class ClientBoundary(importlib.abc.MetaPathFinder): @@ -31,6 +41,7 @@ def find_spec(self, fullname, path=None, target=None): async def check(directory): import httpx + from msg.client import ClientState from msg.core.codec import canonical, loads, wire from msg.core.requests import request_for, signing_bytes @@ -38,8 +49,18 @@ async def check(directory): from msg.transports.client import TRANSPORTS from msg.transports.packet import decode_packet, path_packet - for name in ('cli', 'tui', 'client_certificates', 'client_content', 'client_custodial', - 'client_market', 'client_recovery', 'client_secrets', 'client_tokens', 'client_upgrade'): + for name in ( + 'cli', + 'tui', + 'client_certificates', + 'client_content', + 'client_custodial', + 'client_market', + 'client_recovery', + 'client_secrets', + 'client_tokens', + 'client_upgrade', + ): importlib.import_module('msg.' + name) state = ClientState(directory, server='https://unit.invalid') signer = Ed25519Signer.from_bytes(bytes(range(32))) @@ -51,17 +72,33 @@ async def check(directory): for path in (state.path, state.key_path, state.age_key_path): assert stat.S_IMODE(path.stat().st_mode) == 0o600 assert stat.S_IMODE(directory.stat().st_mode) == 0o700 - request = request_for('content.read', {'id': 'r_test'}, state.server, - subject='u_test', signer=signer, request_id='client-boundary') + request = request_for( + 'content.read', + {'id': 'r_test'}, + state.server, + subject='u_test', + signer=signer, + request_id='client-boundary', + ) calls = [] def respond(incoming): path = incoming.url.path if path == '/_transports': - return httpx.Response(200, json={'version': 1, 'target_service': state.server, - 'operations': {'content.read': 'read'}, 'limits': { - 'max_request_bytes': 1048576, 'max_response_bytes': 1048576, - 'max_path_bytes': 8192, 'encodings': ['j', 'gz']}}) + return httpx.Response( + 200, + json={ + 'version': 1, + 'target_service': state.server, + 'operations': {'content.read': 'read'}, + 'limits': { + 'max_request_bytes': 1048576, + 'max_response_bytes': 1048576, + 'max_path_bytes': 8192, + 'encodings': ['j', 'gz'], + }, + }, + ) if path.startswith('/-/g/'): _, _, _, operation, encoding, encoded = path.split('/') packet = path_packet(encoded, encoding, 1048576) @@ -84,24 +121,38 @@ def respond(incoming): assert canonical(packet) == canonical(request) verify(signer.public_key, signing_bytes(packet), packet.proof.signature, purpose='request') calls.append(kind) - result = {'request_id': packet.request_id, 'operation': packet.operation, 'status': 'ok', - 'actor': 'u_test', 'subject': 'u_test', 'resources': [{'id': 'r_test'}], - 'data': {'content': 'signed client response'}} + result = { + 'request_id': packet.request_id, + 'operation': packet.operation, + 'status': 'ok', + 'actor': 'u_test', + 'subject': 'u_test', + 'resources': [{'id': 'r_test'}], + 'data': {'content': 'signed client response'}, + } if kind == 'graphql': result = {'data': {'call': result}} elif kind == 'mcp_http': - result = {'jsonrpc': '2.0', 'id': packet.request_id, - 'result': {'structuredContent': result}} + result = { + 'jsonrpc': '2.0', + 'id': packet.request_id, + 'result': {'structuredContent': result}, + } return httpx.Response(200, json=result) async with httpx.AsyncClient(transport=httpx.MockTransport(respond)) as http: for name, transport in TRANSPORTS.items(): result = await transport(state.server, http=http).call(request) - assert result.status == 'ok' and result.resources[0].id == 'r_test', (name, wire(result)) + assert result.status == 'ok' and result.resources[0].id == 'r_test', ( + name, + wire(result), + ) assert result.data['content'] == 'signed client response' assert sorted(calls) == sorted(TRANSPORTS) - assert not any(any(name == prefix or name.startswith(prefix + '.') for prefix in FORBIDDEN) - for name in sys.modules) + assert not any( + any(name == prefix or name.startswith(prefix + '.') for prefix in FORBIDDEN) + for name in sys.modules + ) return calls @@ -110,14 +161,23 @@ def main(): parser.add_argument('--minimal-install', action='store_true') args = parser.parse_args() if args.minimal_install: - present = [name for name in SERVER_DEPENDENCIES if importlib.util.find_spec(name) is not None] - assert not present, 'server-only Python dependencies in client environment: ' + ', '.join(present) + present = [ + name for name in SERVER_DEPENDENCIES if importlib.util.find_spec(name) is not None + ] + assert not present, 'server-only Python dependencies in client environment: ' + ', '.join( + present + ) sys.meta_path.insert(0, ClientBoundary()) with tempfile.TemporaryDirectory(prefix='msg-client-install-') as folder: calls = asyncio.run(check(Path(folder) / 'client')) - print(json.dumps({'status': 'ok', 'transports': sorted(calls), - 'server_implementation_imported': False, - 'minimal_install': args.minimal_install})) + print( + json.dumps({ + 'status': 'ok', + 'transports': sorted(calls), + 'server_implementation_imported': False, + 'minimal_install': args.minimal_install, + }) + ) if __name__ == '__main__': diff --git a/scripts/check_isolated_sshd.py b/scripts/check_isolated_sshd.py index a46e0eb3..c1e3537e 100644 --- a/scripts/check_isolated_sshd.py +++ b/scripts/check_isolated_sshd.py @@ -4,11 +4,10 @@ Run with the installed project's Python as an ordinary user with sudo -n. Requires PostgreSQL tools, OpenSSH and Git. No production configuration is read. """ + import asyncio -from datetime import timedelta import json import os -from pathlib import Path import pwd import shlex import shutil @@ -16,19 +15,21 @@ import subprocess import sys import tempfile +from datetime import timedelta +from pathlib import Path from cryptography.hazmat.primitives import serialization from msg.admin.diagnostics import temporary_postgres -from msg.admin.root import _provision, _approve_csr, _revoke +from msg.admin.root import _approve_csr, _provision, _revoke from msg.application import Application from msg.config import write_example +from msg.constants import ROOT_SUBJECT from msg.core.codec import b64, canonical, wire -from msg.core.requests import request_for from msg.core.models import CertificateRequest, Signature -from msg.constants import ROOT_SUBJECT -from msg.security.certificates import csr_body +from msg.core.requests import request_for from msg.security.age_keys import generate_age_key +from msg.security.certificates import csr_body from msg.security.crypto import Ed25519Signer, subject_id @@ -42,9 +43,16 @@ async def acceptance(): sshd = shutil.which('sshd') assert sshd and shutil.which('ssh') and shutil.which('ssh-keygen') checks = [] - with tempfile.TemporaryDirectory(prefix='msg-sshd-check-') as temporary, temporary_postgres() as dsn: + with ( + tempfile.TemporaryDirectory(prefix='msg-sshd-check-') as temporary, + temporary_postgres() as dsn, + ): folder = Path(temporary) - app = Application(write_example(folder/'etc', folder/'data', 'https://sshd-check.invalid', postgres_dsn=dsn)) + app = Application( + write_example( + folder / 'etc', folder / 'data', 'https://sshd-check.invalid', postgres_dsn=dsn + ) + ) assert app.settings.root_private_dir.is_relative_to(folder) server = None master = None @@ -53,65 +61,154 @@ async def acceptance(): csr, root = await _provision(app, 'isolated-sshd-' + os.urandom(24).hex()) await _approve_csr(app, csr, root, expected_digest=None, operator='isolated-sshd-check') # The service account must actually be unable to read root material. - assert run('sudo', '-n', 'chown', '-R', '0:0', str(app.settings.root_private_dir)).returncode == 0 + assert ( + run( + 'sudo', '-n', 'chown', '-R', '0:0', str(app.settings.root_private_dir) + ).returncode + == 0 + ) key = Ed25519Signer.generate() uid = subject_id(key.public_key) async def call(name, args): - packet = request_for(name, args, app.settings.service_url, signer=key, subject=uid, - expires_at=app.clock()+timedelta(seconds=120)) + packet = request_for( + name, + args, + app.settings.service_url, + signer=key, + subject=uid, + expires_at=app.clock() + timedelta(seconds=120), + ) return await app.executor.execute(packet) _, recipient = generate_age_key() - packet = request_for('identity.register', {'handle': 'ssh-check', 'public_key': b64(key.public_key), - 'encryption_recipient': recipient}, app.settings.service_url, signer=key, subject=uid, - contract_version=2, expires_at=app.clock()+timedelta(seconds=120)) + packet = request_for( + 'identity.register', + { + 'handle': 'ssh-check', + 'public_key': b64(key.public_key), + 'encryption_recipient': recipient, + }, + app.settings.service_url, + signer=key, + subject=uid, + contract_version=2, + expires_at=app.clock() + timedelta(seconds=120), + ) assert (await app.executor.execute(packet)).status == 'ok' repo = await call('git.create', {'parent': uid, 'name': 'code.git'}) assert repo.status == 'ok' - client_key = folder/'client-key' - assert run('ssh-keygen', '-q', '-t', 'ed25519', '-N', '', '-f', str(client_key)).returncode == 0 + client_key = folder / 'client-key' + assert ( + run('ssh-keygen', '-q', '-t', 'ed25519', '-N', '', '-f', str(client_key)).returncode + == 0 + ) private = serialization.load_ssh_private_key(client_key.read_bytes(), password=None) - signer = Ed25519Signer.from_bytes(private.private_bytes( - serialization.Encoding.Raw, serialization.PrivateFormat.Raw, serialization.NoEncryption())) + signer = Ed25519Signer.from_bytes( + private.private_bytes( + serialization.Encoding.Raw, + serialization.PrivateFormat.Raw, + serialization.NoEncryption(), + ) + ) line = ' '.join(client_key.with_suffix('.pub').read_text().split()[:2]) - proof = signer.sign(canonical({'subject_id': uid, 'public_key': line}), purpose='ssh-key-add') - grants = [g for g in app.base_grants() if g.capability in {'git.basic', 'discovery.basic'}] - added = await call('identity.ssh_key_add', {'public_key': line, 'proof': wire(proof), - 'ceiling': wire(grants)}) + proof = signer.sign( + canonical({'subject_id': uid, 'public_key': line}), purpose='ssh-key-add' + ) + grants = [ + g for g in app.base_grants() if g.capability in {'git.basic', 'discovery.basic'} + ] + added = await call( + 'identity.ssh_key_add', + {'public_key': line, 'proof': wire(proof), 'ceiling': wire(grants)}, + ) assert added.status == 'ok' - csr = CertificateRequest(resource_id='pending', applicant=uid, subject_id=uid, - requested_issuer=ROOT_SUBJECT, public_key=signer.public_key, kind='capability', - grants=tuple(grants), issuance=None, requested_ttl_seconds=600, - target_service=app.settings.service_url, delegation_depth=0, authority_sources=(), - request_digest='', possession_proof=Signature(key_id=signer.key_id, - algorithm='ed25519', value=b'')) + csr = CertificateRequest( + resource_id='pending', + applicant=uid, + subject_id=uid, + requested_issuer=ROOT_SUBJECT, + public_key=signer.public_key, + kind='capability', + grants=tuple(grants), + issuance=None, + requested_ttl_seconds=600, + target_service=app.settings.service_url, + delegation_depth=0, + authority_sources=(), + request_digest='', + possession_proof=Signature(key_id=signer.key_id, algorithm='ed25519', value=b''), + ) possession = signer.sign(canonical(csr_body(csr)), purpose='csr') - requested = await call('cert.request', {'requested_issuer': ROOT_SUBJECT, - 'public_key': b64(signer.public_key), 'kind': 'capability', 'grants': wire(grants), - 'requested_ttl_seconds': 600, 'possession_proof': wire(possession)}) + requested = await call( + 'cert.request', + { + 'requested_issuer': ROOT_SUBJECT, + 'public_key': b64(signer.public_key), + 'kind': 'capability', + 'grants': wire(grants), + 'requested_ttl_seconds': 600, + 'possession_proof': wire(possession), + }, + ) assert requested.status == 'ok' - certificate = await _approve_csr(app, requested.data['csr_id'], root, - expected_digest=requested.data['request_digest'], operator='isolated-sshd-check') - attached = await call('identity.ssh_certificates', - {'key_id': signer.key_id, 'certificates': [certificate.resource_id]}) + certificate = await _approve_csr( + app, + requested.data['csr_id'], + root, + expected_digest=requested.data['request_digest'], + operator='isolated-sshd-check', + ) + attached = await call( + 'identity.ssh_certificates', + {'key_id': signer.key_id, 'certificates': [certificate.resource_id]}, + ) assert attached.status == 'ok' - host = folder/'host-key' - assert run('ssh-keygen', '-q', '-t', 'ed25519', '-N', '', '-f', str(host)).returncode == 0 - privileged = Path(run('sudo', '-n', 'mktemp', '-d', '/run/msg-sshd-check-XXXXXXXX', check=True).stdout.decode().strip()) - assert privileged.parent == Path('/run') and privileged.name.startswith('msg-sshd-check-') + host = folder / 'host-key' + assert ( + run('ssh-keygen', '-q', '-t', 'ed25519', '-N', '', '-f', str(host)).returncode == 0 + ) + privileged = Path( + run('sudo', '-n', 'mktemp', '-d', '/run/msg-sshd-check-XXXXXXXX', check=True) + .stdout.decode() + .strip() + ) + assert privileged.parent == Path('/run') and privileged.name.startswith( + 'msg-sshd-check-' + ) assert run('sudo', '-n', 'chmod', '755', str(privileged)).returncode == 0 - wrapper = folder/'authorized' - wrapper.write_text('#!/bin/sh\nexec ' + shlex.join([sys.executable, '-m', 'msg.daemon', - '--config-dir', str(folder/'etc'), 'ssh-authorized-key']) + ' "$@"\n') - assert run('sudo', '-n', 'install', '-m', '755', str(wrapper), str(privileged/'authorized')).returncode == 0 + wrapper = folder / 'authorized' + wrapper.write_text( + '#!/bin/sh\nexec ' + + shlex.join([ + sys.executable, + '-m', + 'msg.daemon', + '--config-dir', + str(folder / 'etc'), + 'ssh-authorized-key', + ]) + + ' "$@"\n' + ) + assert ( + run( + 'sudo', + '-n', + 'install', + '-m', + '755', + str(wrapper), + str(privileged / 'authorized'), + ).returncode + == 0 + ) with socket.socket() as probe: probe.bind(('127.0.0.1', 0)) port = probe.getsockname()[1] username = pwd.getpwuid(os.geteuid()).pw_name - pidfile = folder/'sshd.pid' - config = folder/'sshd.conf' - config.write_text(f'''ListenAddress 127.0.0.1 + pidfile = folder / 'sshd.pid' + config = folder / 'sshd.conf' + config.write_text(f"""ListenAddress 127.0.0.1 Port {port} HostKey {host} PidFile {pidfile} @@ -130,83 +227,144 @@ async def call(name, args): PermitUserRC no PermitUserEnvironment no LogLevel ERROR -''') +""") assert run('sudo', '-n', sshd, '-t', '-f', str(config)).returncode == 0 - server = subprocess.Popen(['sudo', '-n', sshd, '-D', '-e', '-f', str(config)], - stdout=subprocess.DEVNULL, stderr=(folder/'sshd.log').open('wb')) + server = subprocess.Popen( + ['sudo', '-n', sshd, '-D', '-e', '-f', str(config)], + stdout=subprocess.DEVNULL, + stderr=(folder / 'sshd.log').open('wb'), + ) for _ in range(100): if pidfile.exists(): break assert server.poll() is None, 'isolated sshd exited' - await asyncio.sleep(.05) + await asyncio.sleep(0.05) assert pidfile.exists() host_fields = host.with_suffix('.pub').read_text().split()[:2] - known = folder/'known-hosts' + known = folder / 'known-hosts' known.write_text(f'[127.0.0.1]:{port} ' + ' '.join(host_fields) + '\n') - ssh = ['ssh', '-F', '/dev/null', '-p', str(port), '-i', str(client_key), - '-o', 'IdentitiesOnly=yes', '-o', 'BatchMode=yes', '-o', 'StrictHostKeyChecking=yes', - '-o', f'UserKnownHostsFile={known}', '-o', 'ConnectTimeout=5'] + ssh = [ + 'ssh', + '-F', + '/dev/null', + '-p', + str(port), + '-i', + str(client_key), + '-o', + 'IdentitiesOnly=yes', + '-o', + 'BatchMode=yes', + '-o', + 'StrictHostKeyChecking=yes', + '-o', + f'UserKnownHostsFile={known}', + '-o', + 'ConnectTimeout=5', + ] target = f'{username}@127.0.0.1' - read = await asyncio.to_thread(run, *ssh, target, 'msg call discovery.get \'{"id":"/main"}\'') - assert read.returncode == 0, (read.stderr.decode(), (folder/'sshd.log').read_text()) + read = await asyncio.to_thread( + run, *ssh, target, 'msg call discovery.get \'{"id":"/main"}\'' + ) + assert read.returncode == 0, (read.stderr.decode(), (folder / 'sshd.log').read_text()) assert json.loads(read.stdout)['status'] == 'ok' checks.append('real_authorized_keys_and_forced_command') for command in ('id', 'sh', 'git-upload-pack /etc/passwd'): denied = await asyncio.to_thread(run, *ssh, target, command) assert denied.returncode != 0 checks.append('shell_and_path_escape_denied') - forwarding = await asyncio.to_thread(run, *ssh, '-o', 'ExitOnForwardFailure=yes', - '-R', '0:127.0.0.1:1', target, 'msg call discovery.get \'{"id":"/main"}\'') - assert forwarding.returncode != 0 and b'remote port forwarding failed' in forwarding.stderr + forwarding = await asyncio.to_thread( + run, + *ssh, + '-o', + 'ExitOnForwardFailure=yes', + '-R', + '0:127.0.0.1:1', + target, + 'msg call discovery.get \'{"id":"/main"}\'', + ) + assert ( + forwarding.returncode != 0 and b'remote port forwarding failed' in forwarding.stderr + ) checks.append('remote_forwarding_denied') - work = folder/'work'; work.mkdir() + work = folder / 'work' + work.mkdir() + def git(*args): return run('git', '-C', str(work), *args) + assert git('init', '--initial-branch=main').returncode == 0 assert git('config', 'user.name', 'Isolated').returncode == 0 assert git('config', 'user.email', 'isolated@example.invalid').returncode == 0 - (work/'README').write_text('real SSH Git acceptance\n') + (work / 'README').write_text('real SSH Git acceptance\n') assert git('add', '.').returncode == 0 assert git('commit', '-m', 'isolated').returncode == 0 assert git('branch', 'second').returncode == 0 remote = f'{target}:/@ssh-check/code.git' - command = ['git', '-C', str(work), '-c', 'core.sshCommand='+shlex.join(ssh)] - pushed = await asyncio.to_thread(run, *command, 'push', '--atomic', remote, 'main', 'second') + command = ['git', '-C', str(work), '-c', 'core.sshCommand=' + shlex.join(ssh)] + pushed = await asyncio.to_thread( + run, *command, 'push', '--atomic', remote, 'main', 'second' + ) assert pushed.returncode == 0, pushed.stderr.decode() fetched = await asyncio.to_thread(run, *command, 'ls-remote', remote) - assert fetched.returncode == 0 and b'refs/heads/main' in fetched.stdout and b'refs/heads/second' in fetched.stdout - clone = folder/'clone' + assert ( + fetched.returncode == 0 + and b'refs/heads/main' in fetched.stdout + and b'refs/heads/second' in fetched.stdout + ) + clone = folder / 'clone' cloned = await asyncio.to_thread(run, *command, 'clone', remote, str(clone)) - assert cloned.returncode == 0 and (clone/'README').read_bytes() == (work/'README').read_bytes() + assert ( + cloned.returncode == 0 + and (clone / 'README').read_bytes() == (work / 'README').read_bytes() + ) checks.append('real_ssh_atomic_multi_ref_push_and_fetch') - control = folder/'control' - master = subprocess.Popen([*ssh, '-M', '-S', str(control), '-N', target], - stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + control = folder / 'control' + master = subprocess.Popen( + [*ssh, '-M', '-S', str(control), '-N', target], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) for _ in range(200): if control.exists(): break assert master.poll() is None, 'SSH master connection failed' - await asyncio.sleep(.05) + await asyncio.sleep(0.05) assert control.exists() - await _revoke(app, certificate.resource_id, root, reason='isolated test', - operator='isolated-sshd-check') - cert_denied = await asyncio.to_thread(run, *ssh, '-S', str(control), target, - 'msg call discovery.get \'{"id":"/main"}\'') - assert cert_denied.returncode != 0 and b'certificate_revoked' in cert_denied.stdout + cert_denied.stderr - new_denied = await asyncio.to_thread(run, *ssh, target, - 'msg call discovery.get \'{"id":"/main"}\'') + await _revoke( + app, + certificate.resource_id, + root, + reason='isolated test', + operator='isolated-sshd-check', + ) + cert_denied = await asyncio.to_thread( + run, *ssh, '-S', str(control), target, 'msg call discovery.get \'{"id":"/main"}\'' + ) + assert ( + cert_denied.returncode != 0 + and b'certificate_revoked' in cert_denied.stdout + cert_denied.stderr + ) + new_denied = await asyncio.to_thread( + run, *ssh, target, 'msg call discovery.get \'{"id":"/main"}\'' + ) assert new_denied.returncode == 255 checks.append('revoked_certificate_denies_new_and_existing_authenticated_connections') # Explicitly detach the revoked optional certificate; never unrevoke it. - detached = await call('identity.ssh_certificates', {'key_id': signer.key_id, 'certificates': []}) + detached = await call( + 'identity.ssh_certificates', {'key_id': signer.key_id, 'certificates': []} + ) assert detached.status == 'ok' revoked = await call('identity.ssh_key_revoke', {'key_id': signer.key_id}) assert revoked.status == 'ok' - reused = await asyncio.to_thread(run, *ssh, '-S', str(control), target, - 'msg call discovery.get \'{"id":"/main"}\'') + reused = await asyncio.to_thread( + run, *ssh, '-S', str(control), target, 'msg call discovery.get \'{"id":"/main"}\'' + ) assert reused.returncode != 0 and b'credential_revoked' in reused.stdout + reused.stderr checks.append('revocation_rechecked_inside_already_authenticated_connection') - denied = await asyncio.to_thread(run, *ssh, target, 'msg call discovery.get \'{"id":"/main"}\'') + denied = await asyncio.to_thread( + run, *ssh, target, 'msg call discovery.get \'{"id":"/main"}\'' + ) assert denied.returncode == 255 and b'Permission denied' in denied.stderr denied_git = await asyncio.to_thread(run, *command, 'ls-remote', remote) assert denied_git.returncode != 0 diff --git a/scripts/check_package_artifacts.py b/scripts/check_package_artifacts.py index 6f3d6f00..8d6ca726 100644 --- a/scripts/check_package_artifacts.py +++ b/scripts/check_package_artifacts.py @@ -1,46 +1,149 @@ -"""Check that the build contains the release-owned rules and installable entry points.""" +"""Reject stale code, changed release resources and mislabeled build artifacts.""" -import sys +import argparse +import configparser +import re import tomllib -from pathlib import Path +from email.parser import BytesParser +from pathlib import Path, PurePosixPath from tarfile import open as open_tar from zipfile import ZipFile -root = Path(__file__).resolve().parents[1] -dist = Path(sys.argv[1]) if len(sys.argv) > 1 else root / "dist" -version = tomllib.loads((root / "pyproject.toml").read_text())["project"]["version"] -wheels = sorted(dist.glob(f"msg_lmm_best-{version}-*.whl")) -sdists = sorted(dist.glob(f"msg_lmm_best-{version}.tar.gz")) -if len(wheels) != 1 or len(sdists) != 1: - raise SystemExit("expected exactly one wheel and one source distribution") - -with ZipFile(wheels[0]) as wheel: - names = set(wheel.namelist()) - for source in (root / "src/msg/data").rglob("*"): - if not source.is_file() or "__pycache__" in source.parts or source.suffix in {".pyc", ".pyo"}: + +def normalized_name(name): + return re.sub(r'[-_.]+', '_', name).lower() + + +def package_sources(root): + package = root / 'src' / 'msg' + result = {} + for source in package.rglob('*'): + if '__pycache__' in source.parts or source.suffix in {'.pyc', '.pyo'}: continue - target = "msg/data/" + source.relative_to(root / "src/msg/data").as_posix() - if target not in names or wheel.read(target) != source.read_bytes(): - raise SystemExit(f"missing or changed wheel resource: {target}") - entry_files = [name for name in names if name.endswith(".dist-info/entry_points.txt")] - if len(entry_files) != 1: - raise SystemExit("missing wheel entry points") - entries = wheel.read(entry_files[0]).decode() - if "msg = msg.cli:main" not in entries or "msgd = msg.daemon:main" not in entries: - raise SystemExit("missing msg or msgd entry point") - -with open_tar(sdists[0]) as sdist: - names = set(sdist.getnames()) - for relative in ( - "src/msg/data/system/AGENTS.md", - "docs/DEPLOYMENT.md", - "deploy/msgd.service", - "tests/test_system_rules.py", - "conformance/test_transports.py", - ): - if not any(name.endswith("/" + relative) for name in names): - raise SystemExit(f"missing sdist file: {relative}") - if any(name.endswith(("/setup.py", "/MANIFEST.in")) for name in names): - raise SystemExit("legacy build files present in sdist") - -print("package artifacts contain the expected code, rules, data, and entry points") + if source.is_symlink(): + raise ValueError(f'package source must not be a symlink: {source}') + if source.is_file(): + result['msg/' + source.relative_to(package).as_posix()] = source.read_bytes() + if 'msg/__init__.py' not in result: + raise ValueError('missing package source') + return result + + +def validate_names(names): + if len(names) != len(set(names)): + raise ValueError('duplicate archive member') + for name in names: + path = PurePosixPath(name) + if path.is_absolute() or '..' in path.parts or '\\' in name: + raise ValueError(f'unsafe archive member: {name}') + + +def check(root, dist): + project = tomllib.loads((root / 'pyproject.toml').read_text())['project'] + version = project['version'] + distribution = normalized_name(project['name']) + prefix = f'{distribution}-{version}' + wheels = sorted(dist.glob(f'{prefix}-*.whl')) + sdists = sorted(dist.glob(f'{prefix}.tar.gz')) + if len(wheels) != 1 or len(sdists) != 1: + raise ValueError('expected exactly one wheel and one source distribution') + expected = package_sources(root) + + with ZipFile(wheels[0]) as wheel: + names = wheel.namelist() + validate_names(names) + packaged = {name for name in names if name.startswith('msg/') and not name.endswith('/')} + if packaged != set(expected): + difference = sorted(packaged ^ set(expected)) + raise ValueError(f'wheel package file set differs from source: {difference}') + for name, content in expected.items(): + if wheel.read(name) != content: + raise ValueError(f'missing or changed wheel source: {name}') + metadata_path = prefix + '.dist-info/METADATA' + metadata_files = [name for name in names if name.endswith('.dist-info/METADATA')] + if metadata_files != [metadata_path]: + raise ValueError('missing or ambiguous wheel metadata') + metadata = BytesParser().parsebytes(wheel.read(metadata_path)) + for field, value in ( + ('Name', project['name']), + ('Version', version), + ('Requires-Python', project['requires-python']), + ): + if metadata.get_all(field) != [value]: + raise ValueError(f'wheel metadata differs from project: {field}') + entry_path = prefix + '.dist-info/entry_points.txt' + entry_files = [name for name in names if name.endswith('.dist-info/entry_points.txt')] + if entry_files != [entry_path]: + raise ValueError('missing or ambiguous wheel entry points') + entries = configparser.ConfigParser(interpolation=None) + entries.read_string(wheel.read(entry_path).decode()) + if not entries.has_section('console_scripts'): + raise ValueError('missing console scripts') + if dict(entries['console_scripts']) != project['scripts']: + raise ValueError('wheel console targets differ from project scripts') + + with open_tar(sdists[0]) as sdist: + names = sdist.getnames() + validate_names(names) + if any(name != prefix and not name.startswith(prefix + '/') for name in names): + raise ValueError('unexpected source distribution root') + if any(name.endswith(('/setup.py', '/MANIFEST.in')) for name in names): + raise ValueError('legacy build files present in sdist') + source_names = { + name.removeprefix(prefix + '/src/') + for name in names + if name.startswith(prefix + '/src/msg/') and sdist.getmember(name).isfile() + } + if source_names != set(expected): + raise ValueError('sdist package file set differs from source') + required = { + **{'src/' + name: data for name, data in expected.items()}, + **{ + name: (root / name).read_bytes() + for name in ( + 'uv.lock', + '.python-version', + 'docs/DEPLOYMENT.md', + 'deploy/msgd.service', + 'tests/test_system_rules.py', + 'conformance/test_transports.py', + ) + }, + } + for relative, content in required.items(): + name = prefix + '/' + relative + if name not in names or not sdist.getmember(name).isfile(): + raise ValueError(f'missing or non-regular sdist file: {relative}') + stream = sdist.extractfile(name) + if stream is None: + raise ValueError(f'unreadable sdist file: {relative}') + with stream: + if stream.read() != content: + raise ValueError(f'changed sdist source: {relative}') + project_path = prefix + '/pyproject.toml' + if project_path not in names or not sdist.getmember(project_path).isfile(): + raise ValueError('missing sdist project metadata') + stream = sdist.extractfile(project_path) + if stream is None: + raise ValueError('unreadable sdist project metadata') + with stream: + packaged_project = tomllib.loads(stream.read().decode())['project'] + for field in ('name', 'version', 'requires-python', 'scripts'): + if packaged_project.get(field) != project[field]: + raise ValueError(f'sdist project metadata differs from source: {field}') + + +def main(): + root = Path(__file__).resolve().parents[1] + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('dist', nargs='?', type=Path, default=root / 'dist') + args = parser.parse_args() + try: + check(root, args.dist) + except ValueError as exc: + parser.exit(1, str(exc) + '\n') + print('package artifacts match current code, rules, metadata, lockfile, and entry points') + + +if __name__ == '__main__': + main() diff --git a/scripts/check_tool_sandbox.py b/scripts/check_tool_sandbox.py index 4ee1e9d6..840345d0 100644 --- a/scripts/check_tool_sandbox.py +++ b/scripts/check_tool_sandbox.py @@ -3,6 +3,7 @@ Only this operator-invoked probe captures subprocess stderr. The packet contains no user input or credentials, and production errors keep their stable safe code. """ + from __future__ import annotations import asyncio @@ -22,14 +23,30 @@ async def check(): print(f'Python prefix: {sys.prefix}; base prefix: {sys.base_prefix}', flush=True) setting = Path('/proc/sys/kernel/apparmor_restrict_unprivileged_userns') if setting.exists(): - print(f'AppArmor unprivileged userns restriction: {setting.read_text().strip()}', flush=True) - policy = NetworkPolicy(schemes=frozenset({'https'}), hosts=(), ports=frozenset({443}), - methods=frozenset({'GET'}), allow_private=False, - timeout_ms=1000, max_response_bytes=1024, max_redirects=0) - app = SimpleNamespace(settings=SimpleNamespace(server=SimpleNamespace( - limits=SimpleNamespace(max_request_bytes=1024)))) + print( + f'AppArmor unprivileged userns restriction: {setting.read_text().strip()}', flush=True + ) + policy = NetworkPolicy( + schemes=frozenset({'https'}), + hosts=(), + ports=frozenset({443}), + methods=frozenset({'GET'}), + allow_private=False, + timeout_ms=1000, + max_response_bytes=1024, + max_redirects=0, + ) + app = SimpleNamespace( + settings=SimpleNamespace( + server=SimpleNamespace(limits=SimpleNamespace(max_request_bytes=1024)) + ) + ) spawn = asyncio.create_subprocess_exec - with TemporaryDirectory(prefix='msg-sandbox-preflight-') as directory, TemporaryFile() as diagnostics: + with ( + TemporaryDirectory(prefix='msg-sandbox-preflight-') as directory, + TemporaryFile() as diagnostics, + ): + async def capture(*args, **kwargs): kwargs['stderr'] = diagnostics return await spawn(*args, **kwargs) @@ -37,8 +54,12 @@ async def capture(*args, **kwargs): try: with patch('asyncio.create_subprocess_exec', capture): try: - await BubblewrapRunner(app)(SimpleNamespace(executor_key='curl'), - {'url': 'file:///not-a-network-target'}, (policy,), Path(directory)) + await BubblewrapRunner(app)( + SimpleNamespace(executor_key='curl'), + {'url': 'file:///not-a-network-target'}, + (policy,), + Path(directory), + ) except Failure as exc: if exc.code != 'network_policy_denied': raise diff --git a/scripts/ci_shards.py b/scripts/ci_shards.py index 8b45b623..1d010f15 100644 --- a/scripts/ci_shards.py +++ b/scripts/ci_shards.py @@ -3,6 +3,7 @@ Adapted from PR #104. Every runner collects the complete suite; the gate rejects skips, duplicate IDs, count-only substitutions and missing or failed evidence. """ + from __future__ import annotations import argparse @@ -18,14 +19,22 @@ def partition(nodeids, index, count): raise ValueError('invalid shard coordinates') if len(nodeids) != len(set(nodeids)): raise ValueError('duplicate test node ID') - return [node for node in nodeids - if int.from_bytes(hashlib.sha256(node.encode()).digest()[:8], 'big') % count == index] + return [ + node + for node in nodeids + if int.from_bytes(hashlib.sha256(node.encode()).digest()[:8], 'big') % count == index + ] def evidence(nodeids, index, count): all_nodes = sorted(nodeids) - return {'version': 2, 'index': index, 'count': count, 'all': all_nodes, - 'selected': partition(all_nodes, index, count)} + return { + 'version': 2, + 'index': index, + 'count': count, + 'all': all_nodes, + 'selected': partition(all_nodes, index, count), + } def report_nodes(path): @@ -34,8 +43,11 @@ def report_nodes(path): raise ValueError('JUnit result contains failed or skipped tests') nodes = [] for case in root.findall('.//testcase'): - ids = [p.get('value') for p in case.findall('./properties/property') - if p.get('name') == 'msg.nodeid'] + ids = [ + p.get('value') + for p in case.findall('./properties/property') + if p.get('name') == 'msg.nodeid' + ] if len(ids) != 1 or not ids[0]: raise ValueError('JUnit result is missing an unambiguous test node ID') nodes.append(ids[0]) @@ -85,10 +97,12 @@ def pytest_collection_modifyitems(self, config, items): item.user_properties.append(('msg.nodeid', item.nodeid)) config.hook.pytest_deselected(items=rejected) (directory / f'shard-{index}.json').write_text( - json.dumps(manifest, ensure_ascii=False, indent=2) + '\n') + json.dumps(manifest, ensure_ascii=False, indent=2) + '\n' + ) - return pytest.main(['tests', f'--junitxml={directory / f"tests-{index}.xml"}'], - plugins=[Collector()]) + return pytest.main( + ['tests', f'--junitxml={directory / f"tests-{index}.xml"}'], plugins=[Collector()] + ) def main(): diff --git a/scripts/verify_local.py b/scripts/verify_local.py new file mode 100644 index 00000000..b3be884b --- /dev/null +++ b/scripts/verify_local.py @@ -0,0 +1,252 @@ +"""Verify an uncommitted checkout without GitHub writes or production deployment. + +Use the locked Python 3.15 development environment. Every stage records its +actual result; missing tools, incomplete tests and source drift fail closed. +""" + +from __future__ import annotations + +import argparse +import hashlib +import importlib.metadata +import json +import os +import platform +import shutil +import subprocess +import sys +import tempfile +import time +import tomllib +from datetime import UTC, datetime +from pathlib import Path +from urllib.parse import urlsplit + +PINNED_TOOLS = {'ruff': '0.16.9', 'uv': '0.12.20'} +REQUIRED_PACKAGES = ( + 'pytest', 'pytest-asyncio', 'build', 'cryptography', 'httpx', 'jsonschema', + 'referencing', 'starlette', 'uvicorn', 'aiohttp', 'dnspython', 'graphql-core', + 'psycopg', 'valkey', 'tiktoken', +) +REQUIRED_COMMANDS = ('git', 'git-lfs', 'age', 'openssl', 'nginx', 'bwrap', 'pg_dump', 'pg_restore') + + +def content_fingerprint(root, names): + """Hash the actual files, not the last commit; never follow source symlinks.""" + digest = hashlib.sha256() + records = [] + for name in sorted(set(names)): + path = root / name + if Path(name).is_absolute() or '..' in Path(name).parts: + raise ValueError('invalid source path') + if path.is_symlink(): + kind = 'symlink' + data = os.fsencode(os.readlink(path)) + elif not path.exists(): + continue + elif path.is_file(): + kind = 'executable' if path.stat().st_mode & 0o111 else 'file' + data = path.read_bytes() + else: + raise ValueError(f'source entry is not a file: {name}') + item = {'path': name, 'kind': kind, 'sha256': hashlib.sha256(data).hexdigest()} + digest.update(json.dumps(item, sort_keys=True, ensure_ascii=True).encode() + b'\n') + records.append(item) + if not records: + raise ValueError('empty source inventory') + return {'sha256': digest.hexdigest(), 'files': records} + + +def source_snapshot(root): + def git(*args): + return subprocess.check_output(['git', *args], cwd=root) + + tracked = git('ls-files', '-z', '--cached', '--others', '--exclude-standard') + names = [os.fsdecode(name) for name in tracked.split(b'\0') if name] + result = content_fingerprint(root, names) + result['base_commit'] = git('rev-parse', 'HEAD').decode().strip() + result['dirty'] = bool(git('status', '--porcelain', '--untracked-files=normal').strip()) + return result + + +def assert_same_source(before, after): + if before['sha256'] != after['sha256']: + raise ValueError('source changed during verification; results belong to different code') + + +def preflight(root): + blockers = [] + tools = {'python': platform.python_version()} + if sys.version_info[:2] != (3, 15): + blockers.append('Python 3.15 is required; do not substitute an older interpreter') + project = tomllib.loads((root / 'pyproject.toml').read_text()) + if project['project']['requires-python'] != '>=3.15': + blockers.append('project requires-python is not >=3.15') + if project['build-system']['requires'] != ['uv_build==0.12.20']: + blockers.append('uv_build is not pinned to 0.12.20') + for name in ('ruff', *REQUIRED_PACKAGES): + try: + version = importlib.metadata.version(name) + except importlib.metadata.PackageNotFoundError: + blockers.append(f'missing Python package: {name}') + continue + tools[name] = version + if name in PINNED_TOOLS and version != PINNED_TOOLS[name]: + blockers.append(f'{name} must be {PINNED_TOOLS[name]}; found {version}') + for name in REQUIRED_COMMANDS: + executable = shutil.which(name) + if executable is None: + blockers.append(f'missing executable: {name}') + else: + tools[name] = executable + if shutil.which('uv') is None: + blockers.append('missing uv executable') + else: + try: + version = subprocess.check_output(['uv', '--version'], text=True, timeout=10).strip() + except (OSError, subprocess.SubprocessError) as exc: + blockers.append('uv version check failed: ' + type(exc).__name__) + else: + tools['uv'] = version + if version.split()[:2] != ['uv', PINNED_TOOLS['uv']]: + blockers.append(f'uv must be {PINNED_TOOLS["uv"]}; found {version}') + for name in ('MSG_TEST_POSTGRES_URL_TEMPLATE', 'MSG_TEST_VALKEY_URL'): + configured = os.environ.get(name) + if configured: + parsed = urlsplit(configured) + if parsed.hostname not in {'localhost', '127.0.0.1', '::1'}: + blockers.append(f'{name} must point to a disposable local test service') + if not os.environ.get('MSG_TEST_POSTGRES_URL_TEMPLATE'): + for name in ('initdb', 'pg_ctl'): + if shutil.which(name) is None: + blockers.append(f'missing local PostgreSQL executable: {name}') + if hasattr(os, 'geteuid') and os.geteuid() == 0: + blockers.append('initdb requires a non-root user, or configure disposable local PostgreSQL') + if not os.environ.get('MSG_TEST_VALKEY_URL'): + blockers.append('MSG_TEST_VALKEY_URL must identify disposable local Valkey') + legacy = os.environ.get('MSG_TEST_LEGACY_SOURCE', str(root / '.legacy-ledger')) + if not (Path(legacy) / 'src').is_dir(): + blockers.append('missing pinned legacy source; set MSG_TEST_LEGACY_SOURCE') + else: + manifest = json.loads((root / 'tests/fixtures/legacy_market_source.json').read_text()) + for name, expected in manifest['sha256'].items(): + path = Path(legacy) / name + if not path.is_file() or hashlib.sha256(path.read_bytes()).hexdigest() != expected: + blockers.append('pinned legacy source mismatch: ' + name) + return {'tools': tools, 'blockers': blockers} + + +def run_step(name, command, directory, *, cwd, env=None, clean_stderr=False, timeout=1200): + started = time.monotonic() + stdout = directory / (name + '.stdout') + stderr = directory / (name + '.stderr') + result = {'name': name, 'command': command, 'status': 'failed'} + try: + with stdout.open('wb') as output, stderr.open('wb') as errors: + completed = subprocess.run( + command, cwd=cwd, env=env, stdout=output, stderr=errors, + check=False, timeout=timeout, + ) + result['returncode'] = completed.returncode + result['stderr_bytes'] = stderr.stat().st_size + if completed.returncode == 0 and (not clean_stderr or not result['stderr_bytes']): + result['status'] = 'passed' + except OSError as exc: + result['error'] = type(exc).__name__ + except subprocess.TimeoutExpired: + result['error'] = 'timeout' + result['duration_seconds'] = round(time.monotonic() - started, 3) + return result + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--shards', type=int, default=8) + parser.add_argument('--directory', type=Path) + parser.add_argument('--preflight-only', action='store_true') + args = parser.parse_args(argv) + if not 1 <= args.shards <= 64: + parser.error('--shards must be between 1 and 64') + root = Path(__file__).resolve().parents[1] + stamp = datetime.now(UTC).strftime('%Y%m%dT%H%M%S.%fZ') + directory = (args.directory or root / 'artifacts' / ('local-' + stamp)).resolve() + directory.mkdir(parents=True, exist_ok=False) + report = {'status': 'blocked', 'stages': [], 'scope': 'local source verification; not deployment'} + report_path = directory / 'result.json' + + def save(): + report_path.write_text(json.dumps(report, indent=2) + '\n') + + def execute(name, command, *, cwd=root, env=None, clean_stderr=False): + stage = run_step(name, command, directory, cwd=cwd, env=env, clean_stderr=clean_stderr) + report['stages'].append(stage) + save() + if stage['status'] != 'passed': + raise ValueError(f'{name} failed; see {directory}') + + try: + report['source_before'] = source_snapshot(root) + report['preflight'] = preflight(root) + save() + if report['preflight']['blockers']: + print('\n'.join(report['preflight']['blockers'])) + return 1 + if args.preflight_only: + report['status'] = 'preflight_passed' + save() + return 0 + python = sys.executable + execute('lock', ['uv', 'lock', '--check', '--python', python]) + execute('dependencies', ['uv', 'pip', 'check', '--python', python]) + execute('ruff', [python, '-m', 'ruff', 'check', '.', '--output-format', 'json'], clean_stderr=True) + execute('format', [python, '-m', 'ruff', 'format', '--check', '.'], clean_stderr=True) + execute('compile', [python, '-W', 'error', '-m', 'compileall', '-q', 'src', 'tests', 'scripts', 'conformance', 'conftest.py'], clean_stderr=True) + execute('tool-sandbox', [python, 'scripts/check_tool_sandbox.py']) + # Sequential shards keep one local Valkey service from cross-shard races. + for index in range(args.shards): + execute(f'shard-{index}', [python, 'scripts/ci_shards.py', 'run', str(index), str(args.shards), '--directory', str(directory)]) + execute('test-union', [python, 'scripts/ci_shards.py', 'check', str(args.shards), '--directory', str(directory)]) + environment = {**os.environ, 'MSG_BENCHMARK_PATH': str(directory / 'token-budget.json')} + execute('conformance', [python, '-m', 'pytest', 'conformance', f'--junitxml={directory / "conformance.xml"}'], env=environment) + # Pytest exits zero for skips too; enforce the same no-skips acceptance here. + import xml.etree.ElementTree as ET + + conformance = ET.parse(directory / 'conformance.xml').getroot() + if not conformance.findall('.//testcase') or any( + conformance.findall('.//' + tag) for tag in ('failure', 'error', 'skipped') + ): + raise ValueError('conformance evidence is incomplete, failed or skipped') + dist = directory / 'dist' + execute('build', [python, '-m', 'build', '--outdir', str(dist)]) + execute('artifacts', [python, 'scripts/check_package_artifacts.py', str(dist)]) + requirements = directory / 'client-requirements.txt' + execute('client-lock', ['uv', 'export', '--locked', '--no-dev', '--no-emit-project', '--format', 'requirements-txt', '--output-file', str(requirements)]) + with tempfile.TemporaryDirectory(prefix='msg-verify-client-') as temporary: + folder = Path(temporary) + venv = folder / 'venv' + execute('client-env', ['uv', 'venv', '--python', python, str(venv)]) + client_python = str(venv / ('Scripts/python.exe' if os.name == 'nt' else 'bin/python')) + wheels = list(dist.glob('*.whl')) + if len(wheels) != 1: + raise ValueError('ambiguous client wheel') + execute('client-install', ['uv', 'pip', 'install', '--python', client_python, '--constraint', str(requirements), str(wheels[0])]) + execute('client-dependencies', ['uv', 'pip', 'check', '--python', client_python]) + environment = dict(os.environ) + environment.pop('PYTHONPATH', None) + execute('client-transports', [client_python, str(root / 'scripts/check_client_install.py'), '--minimal-install'], cwd=folder, env=environment) + report['source_after'] = source_snapshot(root) + assert_same_source(report['source_before'], report['source_after']) + report['status'] = 'passed' + save() + print(f'Local verification passed: {report_path}') + return 0 + except (OSError, ValueError, subprocess.SubprocessError) as exc: + report['status'] = 'failed' + report['error'] = str(exc) + save() + print(str(exc), file=sys.stderr) + return 1 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/src/msg/__init__.py b/src/msg/__init__.py index d72073ba..89b5deec 100644 --- a/src/msg/__init__.py +++ b/src/msg/__init__.py @@ -1,2 +1,3 @@ """Atomic communication for sandboxed agents.""" -__version__ = "1.0.0a1" + +__version__ = '0.1.0a1' diff --git a/src/msg/admin/backup_retirement.py b/src/msg/admin/backup_retirement.py index f91e764c..5724e1db 100644 --- a/src/msg/admin/backup_retirement.py +++ b/src/msg/admin/backup_retirement.py @@ -5,22 +5,24 @@ databases. The record is verified again on every read, so this import is not a trusted cache of a success flag. """ + from __future__ import annotations -from msg.constants import ROOT_SUBJECT import os -from pathlib import Path import tempfile +from pathlib import Path +from msg.constants import ROOT_SUBJECT from msg.core.codec import canonical, digest, loads from msg.core.errors import Failure, require from msg.core.models import AuditEvent, Event, ResourceRef from msg.plugins.common import new_id -from msg.security.backup_retirement import FIELDS, DOMAIN, check, root_verifier, setting_key +from msg.security.backup_retirement import DOMAIN, FIELDS, check, root_verifier, setting_key def read_statement(path): from msg.security.root_files import read_private + return loads(read_private(path, limit=1024 * 1024)) @@ -49,41 +51,70 @@ def write_record(path, record): def unsigned_statement(value): - require(isinstance(value, dict) and set(value) == FIELDS and value.get('domain') == DOMAIN, - 'backup_retirement_invalid') + require( + isinstance(value, dict) and set(value) == FIELDS and value.get('domain') == DOMAIN, + 'backup_retirement_invalid', + ) return value def completed_migration(tx, subject_id, old_identity_key_id): """The one completed migration of this subject that retired exactly this old key.""" - matches = [loads(body) for (body,) in tx.rows( - "SELECT body FROM custodial_upgrades WHERE subject=? AND status='completed' ORDER BY id", - (subject_id,)) if loads(body).get('old_identity_key_id') == old_identity_key_id] + matches = [ + loads(body) + for (body,) in tx.rows( + "SELECT body FROM custodial_upgrades WHERE subject=? AND status='completed' ORDER BY id", + (subject_id,), + ) + if loads(body).get('old_identity_key_id') == old_identity_key_id + ] require(len(matches) == 1, 'backup_retirement_migration_not_found') return matches[0] async def import_record(tx, record, *, now, operator): body = record.get('statement') if isinstance(record, dict) else None - require(isinstance(body, dict) and isinstance(body.get('subject_id'), str) and - isinstance(body.get('old_identity_key_id'), str), 'backup_retirement_invalid') + require( + isinstance(body, dict) + and isinstance(body.get('subject_id'), str) + and isinstance(body.get('old_identity_key_id'), str), + 'backup_retirement_invalid', + ) subject = body['subject_id'] details = completed_migration(tx, subject, body['old_identity_key_id']) attestation = check(record, await root_verifier(tx, now=now), subject, details, now=now) key = setting_key(subject, attestation.old_identity_key_id) previous = tx.setting(key) tx.set_setting(key, record) - data = {'subject_id': subject, 'old_identity_key_id': attestation.old_identity_key_id, - 'old_encryption_key_id': attestation.old_encryption_key_id, - 'backup_sets': list(attestation.backup_sets), 'record_digest': attestation.record_digest, - 'signer_key_id': attestation.signer_key_id, 'operator': operator, - 'scope': 'listed_backup_sets_only'} - event = Event(id=new_id('audit'), type='root.custodial.backup_retirement', time=now, - request_id=new_id('local'), actor=ROOT_SUBJECT, subject=subject, - resources=(ResourceRef(id=subject),), data=data) - await tx.append_audit(AuditEvent(event=event, - authority=(ResourceRef(id=tx.setting('active_root_certificate')),), - before_digest=digest(previous) if previous is not None else None, - after_digest=attestation.record_digest, previous_digest=None, entry_digest='', - result='imported')) + data = { + 'subject_id': subject, + 'old_identity_key_id': attestation.old_identity_key_id, + 'old_encryption_key_id': attestation.old_encryption_key_id, + 'backup_sets': list(attestation.backup_sets), + 'record_digest': attestation.record_digest, + 'signer_key_id': attestation.signer_key_id, + 'operator': operator, + 'scope': 'listed_backup_sets_only', + } + event = Event( + id=new_id('audit'), + type='root.custodial.backup_retirement', + time=now, + request_id=new_id('local'), + actor=ROOT_SUBJECT, + subject=subject, + resources=(ResourceRef(id=subject),), + data=data, + ) + await tx.append_audit( + AuditEvent( + event=event, + authority=(ResourceRef(id=tx.setting('active_root_certificate')),), + before_digest=digest(previous) if previous is not None else None, + after_digest=attestation.record_digest, + previous_digest=None, + entry_digest='', + result='imported', + ) + ) return attestation diff --git a/src/msg/admin/backups.py b/src/msg/admin/backups.py index c3437724..dc09abdf 100644 --- a/src/msg/admin/backups.py +++ b/src/msg/admin/backups.py @@ -1,20 +1,24 @@ """Explicit local data backup; encrypted root material is a separate operation.""" + from __future__ import annotations + import asyncio import hashlib import os -from pathlib import Path import shutil +import subprocess import tempfile import tomllib import zipfile +from pathlib import Path + import psycopg -from psycopg.conninfo import conninfo_to_dict,make_conninfo -import subprocess -from msg.core.codec import canonical,loads -from msg.core.errors import Failure,require -from msg.config import load_settings,write_example +from psycopg.conninfo import conninfo_to_dict, make_conninfo + from msg.admin.restore_database import restore_dump +from msg.config import load_settings, write_example +from msg.core.codec import canonical, loads +from msg.core.errors import require from msg.storage.git import durable_write _FORMAT = 'msg-data-backup-v4' @@ -26,320 +30,438 @@ def _regular_tree(path): require(path.is_dir() and not path.is_symlink(), 'backup_source_missing') for root, directories, files in os.walk(path, followlinks=False): for name in (*directories, *files): - entry=Path(root)/name + entry = Path(root) / name require(not entry.is_symlink(), 'backup_symlink_forbidden') require(entry.is_dir() or entry.is_file(), 'backup_special_file_forbidden') def _db_refs(connection): """Revision and transfer chunk SQL pointers into Git/CAS.""" - contents={} - revisions=[] + contents = {} + revisions = [] for revision_id, raw in connection.execute('SELECT id,body FROM revisions ORDER BY id'): - body=loads(raw) - content=body['content'] + body = loads(raw) + content = body['content'] require(content['digest'].startswith('sha256:'), 'backup_reference_invalid') - contents[content['digest'][7:]]=content['size'] + contents[content['digest'][7:]] = content['size'] revisions.append(revision_id) for (raw,) in connection.execute('SELECT body FROM chunks ORDER BY transfer_id,"offset"'): - content=loads(raw)['content'] + content = loads(raw)['content'] require(content['digest'].startswith('sha256:'), 'backup_reference_invalid') - contents[content['digest'][7:]]=content['size'] + contents[content['digest'][7:]] = content['size'] from msg.market.references import content_references + for content in content_references(connection.execute): require(content['digest'].startswith('sha256:'), 'backup_reference_invalid') - contents[content['digest'][7:]]=content['size'] - live={row[0] for row in connection.execute('SELECT revision FROM resources WHERE revision IS NOT NULL')} + contents[content['digest'][7:]] = content['size'] + live = { + row[0] + for row in connection.execute('SELECT revision FROM resources WHERE revision IS NOT NULL') + } require(live.issubset(set(revisions)), 'backup_revision_missing') - return {'contents':contents, 'revisions':revisions} + return {'contents': contents, 'revisions': revisions} def _verify_storage(directory, refs, *, settings=None, repair_git_layout=True): """Verify archived bytes, Git objects and the SQL-referenced content index.""" - content=(settings.server.content_dir if settings is not None else directory/'content') - blobs=(settings.server.blob_dir if settings is not None else directory/'blobs') - repos=(settings.server.repositories_dir if settings is not None else directory/'repositories') - require(content.is_dir() and (content/'private.git').is_dir() and - (content/'index').is_dir() and blobs.is_dir(), - 'backup_storage_missing') - index_files={path.name:path for path in (content/'index').iterdir() if path.is_file()} + content = settings.server.content_dir if settings is not None else directory / 'content' + blobs = settings.server.blob_dir if settings is not None else directory / 'blobs' + repos = settings.server.repositories_dir if settings is not None else directory / 'repositories' + require( + content.is_dir() + and (content / 'private.git').is_dir() + and (content / 'index').is_dir() + and blobs.is_dir(), + 'backup_storage_missing', + ) + index_files = {path.name: path for path in (content / 'index').iterdir() if path.is_file()} require(set(refs['contents']).issubset(index_files), 'backup_content_missing') - for key,index in index_files.items(): - entry=loads(index.read_bytes()) - size=refs['contents'].get(key,entry.get('size')) - require(len(key)==64 and all(c in '0123456789abcdef' for c in key) and - type(size) is int and size>=0, 'backup_reference_invalid') - require(entry.get('size')==size, 'backup_content_size_mismatch') - if entry.get('kind')=='binary': - target=blobs/key - require(target.is_file() and target.stat().st_size==size and - _hash(target)==key, 'backup_content_digest_mismatch') + for key, index in index_files.items(): + entry = loads(index.read_bytes()) + size = refs['contents'].get(key, entry.get('size')) + require( + len(key) == 64 + and all(c in '0123456789abcdef' for c in key) + and type(size) is int + and size >= 0, + 'backup_reference_invalid', + ) + require(entry.get('size') == size, 'backup_content_size_mismatch') + if entry.get('kind') == 'binary': + target = blobs / key + require( + target.is_file() and target.stat().st_size == size and _hash(target) == key, + 'backup_content_digest_mismatch', + ) else: - require(entry.get('kind')=='git', 'backup_content_kind_invalid') - oid=entry.get('oid') - require(type(oid) is str and len(oid) in (40,64) and - all(c in '0123456789abcdef' for c in oid), 'backup_git_oid_invalid') - status,actual_size,actual_hash=_git_blob_digest(content/'private.git',oid) - require(status==0 and actual_size==size and actual_hash==key, - 'backup_git_content_mismatch') + require(entry.get('kind') == 'git', 'backup_content_kind_invalid') + oid = entry.get('oid') + require( + type(oid) is str + and len(oid) in (40, 64) + and all(c in '0123456789abcdef' for c in oid), + 'backup_git_oid_invalid', + ) + status, actual_size, actual_hash = _git_blob_digest(content / 'private.git', oid) + require( + status == 0 and actual_size == size and actual_hash == key, + 'backup_git_content_mismatch', + ) for revision in refs['revisions']: - require(type(revision) is str and '/' not in revision and revision not in ('.','..'), - 'backup_revision_invalid') - pointer=content/'revisions'/revision + require( + type(revision) is str and '/' not in revision and revision not in ('.', '..'), + 'backup_revision_invalid', + ) + pointer = content / 'revisions' / revision require(pointer.is_file(), 'backup_revision_content_missing') - oid=pointer.read_text().strip() - require(len(oid) in (40,64) and all(c in '0123456789abcdef' for c in oid), - 'backup_revision_oid_invalid') - process=subprocess.run(['git','--git-dir',str(content/'private.git'), - 'cat-file','-e',oid+'^{commit}'],capture_output=True,check=False) - require(process.returncode==0, 'backup_revision_content_missing') - for repository in (repos.iterdir() if repos.exists() else ()): - require(repository.is_dir() and repository.name.endswith('.git'), - 'backup_repository_invalid') + oid = pointer.read_text().strip() + require( + len(oid) in (40, 64) and all(c in '0123456789abcdef' for c in oid), + 'backup_revision_oid_invalid', + ) + process = subprocess.run( + ['git', '--git-dir', str(content / 'private.git'), 'cat-file', '-e', oid + '^{commit}'], + capture_output=True, + check=False, + ) + require(process.returncode == 0, 'backup_revision_content_missing') + for repository in repos.iterdir() if repos.exists() else (): + require( + repository.is_dir() and repository.name.endswith('.git'), 'backup_repository_invalid' + ) # ZIP stores files only. An empty bare repository still needs these # structural directories; recreating them changes no Git object/ref. - for name in ('objects/info','objects/pack','refs/heads','refs/tags'): + for name in ('objects/info', 'objects/pack', 'refs/heads', 'refs/tags'): if repair_git_layout: - (repository/name).mkdir(parents=True,exist_ok=True) + (repository / name).mkdir(parents=True, exist_ok=True) else: - require((repository/name).is_dir(), 'backup_git_layout_missing') - process=subprocess.run(['git','--git-dir',str(repository),'fsck','--full','--no-reflogs'], - capture_output=True,check=False) - require(process.returncode==0, 'backup_git_repository_invalid') - objects=repository/'lfs'/'objects' + require((repository / name).is_dir(), 'backup_git_layout_missing') + process = subprocess.run( + ['git', '--git-dir', str(repository), 'fsck', '--full', '--no-reflogs'], + capture_output=True, + check=False, + ) + require(process.returncode == 0, 'backup_git_repository_invalid') + objects = repository / 'lfs' / 'objects' if not objects.exists(): continue for path in objects.rglob('*'): if not path.is_file(): continue - oid=path.name - require(len(oid)==64 and all(c in '0123456789abcdef' for c in oid) and - path.relative_to(objects).parts==(oid[:2],oid[2:4],oid), - 'backup_lfs_path_invalid') - canonical_blob=blobs/oid - require(canonical_blob.is_file() and _hash(path)==oid and - path.stat().st_size==canonical_blob.stat().st_size and - _hash(canonical_blob)==oid, 'backup_lfs_digest_mismatch') + oid = path.name + require( + len(oid) == 64 + and all(c in '0123456789abcdef' for c in oid) + and path.relative_to(objects).parts == (oid[:2], oid[2:4], oid), + 'backup_lfs_path_invalid', + ) + canonical_blob = blobs / oid + require( + canonical_blob.is_file() + and _hash(path) == oid + and path.stat().st_size == canonical_blob.stat().st_size + and _hash(canonical_blob) == oid, + 'backup_lfs_digest_mismatch', + ) def _relink_lfs(data_dir, settings): """ZIP cannot retain hardlinks; restore canonical shared-CAS link counts.""" - repositories=settings.server.repositories_dir + repositories = settings.server.repositories_dir if not repositories.exists(): return for repository in repositories.iterdir(): - objects=repository/'lfs'/'objects' + objects = repository / 'lfs' / 'objects' if not objects.exists(): continue for path in objects.rglob('*'): if not path.is_file(): continue - shared=settings.server.blob_dir/path.name - require(path.stat().st_dev==shared.stat().st_dev, - 'restore_lfs_shared_volume_required') - temporary=path.with_name('.relink-'+path.name) + shared = settings.server.blob_dir / path.name + require( + path.stat().st_dev == shared.stat().st_dev, 'restore_lfs_shared_volume_required' + ) + temporary = path.with_name('.relink-' + path.name) try: - os.link(shared,temporary) - os.replace(temporary,path) + os.link(shared, temporary) + os.replace(temporary, path) finally: temporary.unlink(missing_ok=True) def _write_restored_config(source, settings, postgres_dsn): """Retain policy, but bind only to the new isolated storage and loopback.""" - raw=tomllib.loads(source.read_text()) - server={'service_url':settings.service_url,'listen':'127.0.0.1','port':8042} - for name in ('public_web_origin','temporary_ttl','transfer_ttl'): - if name in raw.get('server',{}): - server[name]=raw['server'][name] - storage={'postgres_dsn':postgres_dsn, - 'content':str(settings.server.content_dir), - 'repositories':str(settings.server.repositories_dir), - 'blobs':str(settings.server.blob_dir), - 'staging':str(settings.server.staging_dir), - 'service_keys':str(settings.service_keys)} - sections={'server':server,'storage':storage} - for name in ('identity','money','limits','plugins','tools'): + raw = tomllib.loads(source.read_text()) + server = {'service_url': settings.service_url, 'listen': '127.0.0.1', 'port': 8042} + for name in ('public_web_origin', 'temporary_ttl', 'transfer_ttl'): + if name in raw.get('server', {}): + server[name] = raw['server'][name] + storage = { + 'postgres_dsn': postgres_dsn, + 'content': str(settings.server.content_dir), + 'repositories': str(settings.server.repositories_dir), + 'blobs': str(settings.server.blob_dir), + 'staging': str(settings.server.staging_dir), + 'service_keys': str(settings.service_keys), + } + sections = {'server': server, 'storage': storage} + for name in ('identity', 'money', 'limits', 'plugins', 'tools'): if name in raw: - sections[name]=raw[name] - lines=[] - for section,values in sections.items(): + sections[name] = raw[name] + lines = [] + for section, values in sections.items(): lines.append(f'[{section}]') - for name,value in values.items(): + for name, value in values.items(): lines.append(f'{name} = {canonical(value).decode()}') lines.append('') - recovery=raw.get('recovery',{}) + recovery = raw.get('recovery', {}) if recovery: lines.append('[recovery]') - for name,value in recovery.items(): - if name!='custodians': + for name, value in recovery.items(): + if name != 'custodians': lines.append(f'{name} = {canonical(value).decode()}') lines.append('') - for custodian in recovery.get('custodians',()): + for custodian in recovery.get('custodians', ()): lines.append('[[recovery.custodians]]') - for name,value in custodian.items(): + for name, value in custodian.items(): lines.append(f'{name} = {canonical(value).decode()}') lines.append('') - path=settings.config_dir/'msgd.toml' + path = settings.config_dir / 'msgd.toml' path.write_text('\n'.join(lines)) - os.chmod(path,0o600) + os.chmod(path, 0o600) def _hash(path): - value=hashlib.sha256() + value = hashlib.sha256() with path.open('rb') as source: - while chunk:=source.read(65536):value.update(chunk) + while chunk := source.read(65536): + value.update(chunk) return value.hexdigest() def _git_blob_digest(repository, oid): - value=hashlib.sha256() - size=0 - with subprocess.Popen(['git','--git-dir',str(repository),'cat-file','blob',oid], - stdout=subprocess.PIPE,stderr=subprocess.DEVNULL) as process: - while chunk:=process.stdout.read(1024*1024): - value.update(chunk);size+=len(chunk) - status=process.wait() - return status,size,value.hexdigest() + value = hashlib.sha256() + size = 0 + with subprocess.Popen( + ['git', '--git-dir', str(repository), 'cat-file', 'blob', oid], + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, + ) as process: + while chunk := process.stdout.read(1024 * 1024): + value.update(chunk) + size += len(chunk) + status = process.wait() + return status, size, value.hexdigest() -async def backup(app,destination): - destination=Path(destination) - require(not destination.exists() and not destination.is_symlink(),'backup_destination_exists') - destination.parent.mkdir(parents=True,exist_ok=True) - with tempfile.TemporaryDirectory(prefix='msg-backup-',dir=destination.parent) as tmp: - directory=Path(tmp) - async with app.metadata.transaction(write=True) as tx: +async def backup(app, destination): + destination = Path(destination) + require(not destination.exists() and not destination.is_symlink(), 'backup_destination_exists') + destination.parent.mkdir(parents=True, exist_ok=True) + with tempfile.TemporaryDirectory(prefix='msg-backup-', dir=destination.parent) as tmp: + directory = Path(tmp) + async with app.metadata.transaction(write=True): # PostgreSQL's transaction-scoped deployment advisory lock excludes # other msgd writers while pg_dump uses its own MVCC connection. # Storage objects are published before their SQL pointers commit. - await asyncio.to_thread(app.metadata.backup,directory/'metadata.dump') + await asyncio.to_thread(app.metadata.backup, directory / 'metadata.dump') with psycopg.connect(app.settings.server.postgres_dsn) as connection: - refs=_db_refs(connection) - for source,name in ((app.settings.server.content_dir,'content'), - (app.settings.server.repositories_dir,'repositories'), - (app.settings.server.blob_dir,'blobs'), - (app.settings.server.staging_dir,'staging')): + refs = _db_refs(connection) + for source, name in ( + (app.settings.server.content_dir, 'content'), + (app.settings.server.repositories_dir, 'repositories'), + (app.settings.server.blob_dir, 'blobs'), + (app.settings.server.staging_dir, 'staging'), + ): if source.exists(): _regular_tree(source) - await asyncio.to_thread(shutil.copytree,source,directory/name) + await asyncio.to_thread(shutil.copytree, source, directory / name) else: - (directory/name).mkdir() + (directory / name).mkdir() _regular_tree(app.settings.service_keys) - await asyncio.to_thread(shutil.copytree,app.settings.service_keys,directory/'service') - require(all((directory/'service'/name).is_file() for name in _KEYS), - 'service_keys_missing') - require(app.settings.trust_file.is_file() and not app.settings.trust_file.is_symlink(), - 'backup_trust_missing') - shutil.copy2(app.settings.trust_file,directory/'root-public.json') + await asyncio.to_thread( + shutil.copytree, app.settings.service_keys, directory / 'service' + ) + require( + all((directory / 'service' / name).is_file() for name in _KEYS), + 'service_keys_missing', + ) + require( + app.settings.trust_file.is_file() and not app.settings.trust_file.is_symlink(), + 'backup_trust_missing', + ) + shutil.copy2(app.settings.trust_file, directory / 'root-public.json') # Keep policy/configuration for review. Restore rewrites storage paths, # disables Valkey and does not copy mail credentials. - config_source=(app.settings.config_dir/'msgd.toml' - if (app.settings.config_dir/'msgd.toml').exists() - else app.settings.config_dir/'server.toml') - require(config_source.is_file() and not config_source.is_symlink(), - 'backup_configuration_missing') - shutil.copy2(config_source,directory/'source-config.toml') - _verify_storage(directory,refs) - files={str(path.relative_to(directory)):_hash(path) for path in directory.rglob('*') if path.is_file()} - manifest={'format':_FORMAT,'service_url':app.settings.service_url,'files':files, - 'references':refs,'root_private_key_included':False, - 'mail_credentials_included':False} - (directory/'manifest.json').write_bytes(canonical(manifest)) - partial=directory/'archive.zip' - with zipfile.ZipFile(partial,'x',compression=zipfile.ZIP_DEFLATED) as archive: - for name in ('manifest.json',*sorted(files)): - archive.write(directory/name,name) - os.chmod(partial,0o600) - with partial.open('rb') as stream:os.fsync(stream.fileno()) - os.link(partial,destination) - return {'status':'backup_created','path':str(destination),'sha256':_hash(destination), - 'root_private_key_included':False} + config_source = ( + app.settings.config_dir / 'msgd.toml' + if (app.settings.config_dir / 'msgd.toml').exists() + else app.settings.config_dir / 'server.toml' + ) + require( + config_source.is_file() and not config_source.is_symlink(), + 'backup_configuration_missing', + ) + shutil.copy2(config_source, directory / 'source-config.toml') + _verify_storage(directory, refs) + files = { + str(path.relative_to(directory)): _hash(path) + for path in directory.rglob('*') + if path.is_file() + } + manifest = { + 'format': _FORMAT, + 'service_url': app.settings.service_url, + 'files': files, + 'references': refs, + 'root_private_key_included': False, + 'mail_credentials_included': False, + } + (directory / 'manifest.json').write_bytes(canonical(manifest)) + partial = directory / 'archive.zip' + with zipfile.ZipFile(partial, 'x', compression=zipfile.ZIP_DEFLATED) as archive: + for name in ('manifest.json', *sorted(files)): + archive.write(directory / name, name) + os.chmod(partial, 0o600) + with partial.open('rb') as stream: + os.fsync(stream.fileno()) + os.link(partial, destination) + return { + 'status': 'backup_created', + 'path': str(destination), + 'sha256': _hash(destination), + 'root_private_key_included': False, + } -def restore(source,config_dir,data_dir,*,postgres_dsn='service=msgd'): +def restore(source, config_dir, data_dir, *, postgres_dsn='service=msgd'): """Restore only into new locations and an empty PostgreSQL database.""" - config_dir,data_dir=Path(config_dir),Path(data_dir) - require(not config_dir.exists() and not data_dir.exists(),'restore_target_exists') - config_dir.parent.mkdir(parents=True,exist_ok=True) - data_dir.parent.mkdir(parents=True,exist_ok=True) - with tempfile.TemporaryDirectory(prefix='msg-restore-',dir=data_dir.parent) as temp: - directory=Path(temp) + config_dir, data_dir = Path(config_dir), Path(data_dir) + require(not config_dir.exists() and not data_dir.exists(), 'restore_target_exists') + config_dir.parent.mkdir(parents=True, exist_ok=True) + data_dir.parent.mkdir(parents=True, exist_ok=True) + with tempfile.TemporaryDirectory(prefix='msg-restore-', dir=data_dir.parent) as temp: + directory = Path(temp) with zipfile.ZipFile(source) as archive: - names=archive.namelist() - require(len(names)==len(set(names)) and 'manifest.json' in names,'invalid_backup') - manifest=loads(archive.read('manifest.json')) - require(manifest.get('format')==_FORMAT and manifest.get('root_private_key_included') is False, - 'invalid_backup') - require(set(names)==set(manifest['files'])|{'manifest.json'},'backup_file_mismatch') + names = archive.namelist() + require(len(names) == len(set(names)) and 'manifest.json' in names, 'invalid_backup') + manifest = loads(archive.read('manifest.json')) + require( + manifest.get('format') == _FORMAT + and manifest.get('root_private_key_included') is False, + 'invalid_backup', + ) + require( + set(names) == set(manifest['files']) | {'manifest.json'}, 'backup_file_mismatch' + ) for info in archive.infolist(): - name=info.filename - require(not name.startswith('/') and '\\' not in name and - all(part not in {'','..','.'} for part in name.split('/')),'unsafe_backup_path') - require((info.external_attr>>16)&0o170000 not in {0o120000,0o060000,0o020000},'unsafe_backup_entry') - target=directory/name;target.parent.mkdir(mode=0o700,parents=True,exist_ok=True) - with archive.open(info) as incoming,target.open('xb') as output: - shutil.copyfileobj(incoming,output,length=65536) - os.chmod(target,0o600) - if name!='manifest.json':require(_hash(target)==manifest['files'][name],'backup_digest_mismatch') + name = info.filename + require( + not name.startswith('/') + and '\\' not in name + and all(part not in {'', '..', '.'} for part in name.split('/')), + 'unsafe_backup_path', + ) + require( + (info.external_attr >> 16) & 0o170000 not in {0o120000, 0o060000, 0o020000}, + 'unsafe_backup_entry', + ) + target = directory / name + target.parent.mkdir(mode=0o700, parents=True, exist_ok=True) + with archive.open(info) as incoming, target.open('xb') as output: + shutil.copyfileobj(incoming, output, length=65536) + os.chmod(target, 0o600) + if name != 'manifest.json': + require(_hash(target) == manifest['files'][name], 'backup_digest_mismatch') # Empty storage directories have no ZIP file entries. - for name in ('content','repositories','blobs','staging'): - (directory/name).mkdir(exist_ok=True) - require((directory/'metadata.dump').is_file() and - (directory/'root-public.json').is_file() and - (directory/'source-config.toml').is_file() and - all((directory/'service'/name).is_file() for name in _KEYS), - 'backup_required_file_missing') - _verify_storage(directory,manifest['references']) - fields=conninfo_to_dict(postgres_dsn) - password=fields.pop('password',None) - env=os.environ.copy() - if password is not None:env['PGPASSWORD']=password - safe_dsn=make_conninfo(**fields) + for name in ('content', 'repositories', 'blobs', 'staging'): + (directory / name).mkdir(exist_ok=True) + require( + (directory / 'metadata.dump').is_file() + and (directory / 'root-public.json').is_file() + and (directory / 'source-config.toml').is_file() + and all((directory / 'service' / name).is_file() for name in _KEYS), + 'backup_required_file_missing', + ) + _verify_storage(directory, manifest['references']) + fields = conninfo_to_dict(postgres_dsn) + password = fields.pop('password', None) + env = os.environ.copy() + if password is not None: + env['PGPASSWORD'] = password + safe_dsn = make_conninfo(**fields) with psycopg.connect(postgres_dsn) as connection: - require(connection.execute( - "SELECT 1 FROM information_schema.tables WHERE table_schema=current_schema() LIMIT 1" - ).fetchone() is None,'restore_database_not_empty') - quarantine={'format':'msg-recovery-quarantine-v1','outbound_enabled':False, - 'source_backup_sha256':_hash(Path(source)), - 'revocation_replay':'required','authority':'health_only'} - restore_dump(directory/'metadata.dump',safe_dsn=safe_dsn,env=env, - quarantine=quarantine) + require( + connection.execute( + 'SELECT 1 FROM information_schema.tables WHERE table_schema=current_schema() LIMIT 1' + ).fetchone() + is None, + 'restore_database_not_empty', + ) + quarantine = { + 'format': 'msg-recovery-quarantine-v1', + 'outbound_enabled': False, + 'source_backup_sha256': _hash(Path(source)), + 'revocation_replay': 'required', + 'authority': 'health_only', + } + restore_dump(directory / 'metadata.dump', safe_dsn=safe_dsn, env=env, quarantine=quarantine) with psycopg.connect(postgres_dsn) as connection: - require(_db_refs(connection)==manifest['references'], - 'backup_database_reference_mismatch') - row=connection.execute("SELECT value FROM settings WHERE key='runtime_config'").fetchone() - runtime=loads(row[0]) if row is not None else {} - require(type(runtime) is dict,'backup_runtime_config_invalid') - runtime['accept_writes']=False - runtime['cleanup_enabled']=False - connection.execute('''INSERT INTO settings(key,value) VALUES('runtime_config',%s) - ON CONFLICT(key) DO UPDATE SET value=EXCLUDED.value''', - (canonical(runtime).decode(),)) + require( + _db_refs(connection) == manifest['references'], 'backup_database_reference_mismatch' + ) + row = connection.execute( + "SELECT value FROM settings WHERE key='runtime_config'" + ).fetchone() + runtime = loads(row[0]) if row is not None else {} + require(type(runtime) is dict, 'backup_runtime_config_invalid') + runtime['accept_writes'] = False + runtime['cleanup_enabled'] = False + connection.execute( + """INSERT INTO settings(key,value) VALUES('runtime_config',%s) + ON CONFLICT(key) DO UPDATE SET value=EXCLUDED.value""", + (canonical(runtime).decode(),), + ) data_dir.mkdir(mode=0o700) - settings=write_example(config_dir,data_dir,manifest['service_url'],postgres_dsn=postgres_dsn) + settings = write_example( + config_dir, data_dir, manifest['service_url'], postgres_dsn=postgres_dsn + ) # Install the fail-closed marker as soon as a runnable config exists, # including if a later filesystem verification fails. - durable_write(config_dir/'recovery-drill.json',canonical({ - **quarantine,'format':'msg-recovery-drill-v1'}),mode=0o600) - _write_restored_config(directory/'source-config.toml',settings,postgres_dsn) - settings=load_settings(config_dir) - for name,target in (('content',settings.server.content_dir), - ('repositories',settings.server.repositories_dir), - ('blobs',settings.server.blob_dir), - ('staging',settings.server.staging_dir)): - if (directory/name).exists(): - target.parent.mkdir(parents=True,exist_ok=True) - shutil.move(str(directory/name),target) - settings.service_keys.parent.mkdir(parents=True,exist_ok=True) - shutil.move(str(directory/'service'),settings.service_keys) - os.chmod(settings.service_keys,0o700) - settings.trust_file.parent.mkdir(parents=True,exist_ok=True) + durable_write( + config_dir / 'recovery-drill.json', + canonical({**quarantine, 'format': 'msg-recovery-drill-v1'}), + mode=0o600, + ) + _write_restored_config(directory / 'source-config.toml', settings, postgres_dsn) + settings = load_settings(config_dir) + for name, target in ( + ('content', settings.server.content_dir), + ('repositories', settings.server.repositories_dir), + ('blobs', settings.server.blob_dir), + ('staging', settings.server.staging_dir), + ): + if (directory / name).exists(): + target.parent.mkdir(parents=True, exist_ok=True) + shutil.move(str(directory / name), target) + settings.service_keys.parent.mkdir(parents=True, exist_ok=True) + shutil.move(str(directory / 'service'), settings.service_keys) + os.chmod(settings.service_keys, 0o700) + settings.trust_file.parent.mkdir(parents=True, exist_ok=True) from msg.security.trust_files import write_trust - write_trust(settings.config_dir,loads((directory/'root-public.json').read_bytes()), - writer=durable_write) - _relink_lfs(data_dir,settings) - _verify_storage(data_dir,manifest['references'],settings=settings) - return {'status':'restored','root_admin_material':'restore_separate_encrypted_root_backup_locally', - 'mail':'disabled_until_configured','outbound':'disabled_recovery_drill', - 'revocation_replay':'required','promotion':'blocked'} + + write_trust( + settings.config_dir, + loads((directory / 'root-public.json').read_bytes()), + writer=durable_write, + ) + _relink_lfs(data_dir, settings) + _verify_storage(data_dir, manifest['references'], settings=settings) + return { + 'status': 'restored', + 'root_admin_material': 'restore_separate_encrypted_root_backup_locally', + 'mail': 'disabled_until_configured', + 'outbound': 'disabled_recovery_drill', + 'revocation_replay': 'required', + 'promotion': 'blocked', + } diff --git a/src/msg/admin/config_check.py b/src/msg/admin/config_check.py index fe9940b9..7d3fc385 100644 --- a/src/msg/admin/config_check.py +++ b/src/msg/admin/config_check.py @@ -4,15 +4,15 @@ Selftest writes temporary TOML files and calls the real loader for each option. It does not connect to SMTP, bind listeners, read a credential, or provision keys. """ -from copy import deepcopy + import json +from copy import deepcopy from pathlib import Path from tempfile import TemporaryDirectory from msg.config_contracts import CONFIGURATION_FIELDS from msg.core.errors import Failure, require - _REQUIRED = object() _DISABLED = object() @@ -20,95 +20,188 @@ def _vectors(): from msg.plugins import BUILTINS from msg.security.age_keys import recipient_from_public - recipient = recipient_from_public(bytes.fromhex('09'+'00'*31)) - custodian = {'id':'test-custodian','name':'Isolated custodian','recipient':recipient} + + recipient = recipient_from_public(bytes.fromhex('09' + '00' * 31)) + custodian = {'id': 'test-custodian', 'name': 'Isolated custodian', 'recipient': recipient} # These are independent acceptance expectations, not configuration defaults # used by the server. Each case asserts the loader's effective Settings value. rows = { - 'server.service_url': ('https://msg.lmm.best','https://config.invalid','file:///tmp/x','invalid_service_url'), - 'server.listen': ('127.0.0.1','::1','','invalid_listen_address'), - 'server.port': (8042,65535,0,'invalid_listen_port'), - 'server.public_web_origin': (None,'https://msg.lmm.best','https://elsewhere.invalid','hosting_origin_must_match_service'), - 'server.temporary_ttl': (3600,1,0,'invalid_ttl'), - 'server.transfer_ttl': (86400,1,0,'invalid_ttl'), - 'storage.postgres_dsn': (_REQUIRED,'service=isolated','sqlite:///tmp/x','invalid_postgres_dsn'), - 'storage.valkey_url': (None,'redis://localhost:6379/0','http://localhost:6379','invalid_valkey_url'), - 'storage.content': ('/var/lib/msgd/git/content','/tmp/config-content','relative','storage_paths_must_be_absolute'), - 'storage.repositories': ('/var/lib/msgd/git/repos','/tmp/config-repos','relative','storage_paths_must_be_absolute'), - 'storage.blobs': ('/var/lib/msgd/blobs/sha256','/tmp/config-blobs','relative','storage_paths_must_be_absolute'), - 'storage.staging': ('/var/lib/msgd/transfers/staging','/tmp/config-staging','relative','storage_paths_must_be_absolute'), - 'storage.service_keys': ('/var/lib/msgd/service','/tmp/config-service','relative','storage_paths_must_be_absolute'), - 'limits.request_bytes': (1048576,256,255,'invalid_limit'), - 'limits.response_bytes': (1048576,256,255,'invalid_limit'), - 'limits.path_bytes': (8192,256,255,'invalid_limit'), - 'limits.part_bytes': (65536,256,255,'invalid_limit'), - 'plugins.enabled': (list(BUILTINS),['identity'],['arbitrary'],'unknown_plugin'), - 'tools.isolation': ('bubblewrap','bubblewrap','none','unsafe_tool_worker'), - 'tools.timeout_ms': (10000,1,0,'invalid_tool_limit'), - 'tools.max_response_bytes': (4194304,1,0,'invalid_tool_limit'), - 'tools.methods': (['GET','HEAD'],['GET'],['EXEC'],'invalid_tool_methods'), - 'tools.ports': ([80,443],[65535],[0],'invalid_tool_ports'), - 'identity.credential_delivery_recovery_window': (900,'60m','61m','invalid_credential_delivery_recovery_window'), - 'money.enabled': (True,True,False,'invalid_money_configuration'), - 'money.currency_id': ('primary','primary','other','invalid_money_configuration'), - 'money.display_name': ('MSG','Community credits','','invalid_money_display_name'), - 'money.code': ('MSG','CREDIT','msg','invalid_money_code'), - 'money.scale': (6,6,7,'invalid_money_configuration'), - 'money.transfer_fee': (0,0,1,'invalid_money_configuration'), - 'money.allow_overdraft': (False,False,True,'invalid_money_configuration'), - 'hosting.base_capacity_bytes': (10485760,1024**4,0,'invalid_hosting_capacity'), - 'hosting.recovery_marker': (None,'/tmp/config-marker','relative','invalid_hosting_recovery_marker'), - 'hosting.content_group_read': (False,True,'true','invalid_content_group_read'), - 'recovery.custodians': ([],[custodian],[custodian]*17,'invalid_recovery_custodians'), - 'mail.enabled': (_DISABLED,False,'false','invalid_mail_enabled'), - 'mail.host': (_DISABLED,'smtp.example.test','bad host','invalid_mail_host'), - 'mail.port': (_DISABLED,465,65536,'invalid_mail_port'), - 'mail.tls': (_DISABLED,'tls','plaintext','mail_tls_required'), - 'mail.sender': (_DISABLED,'Message service ','sender\r\ninjected','invalid_mail_sender'), - 'mail.credential_file': (_DISABLED,'/tmp/unread-smtp-credentials','relative','invalid_mail_credential_file'), - 'recovery.custodians.id': (_DISABLED,'other-custodian','INVALID','invalid_recovery_custodian'), - 'recovery.custodians.name': (_DISABLED,'Other custodian','','invalid_recovery_custodian'), - 'recovery.custodians.recipient': (_DISABLED,recipient,'invalid','invalid_age_recipient'), - 'recovery.custodians.description': (_DISABLED,'Description','x'*501,'invalid_recovery_custodian'), - 'recovery.custodians.policy_ref': (_DISABLED,'/wiki/recovery-policy','x'*501,'invalid_recovery_custodian'), + 'server.service_url': ( + 'https://msg.lmm.best', + 'https://config.invalid', + 'file:///tmp/x', + 'invalid_service_url', + ), + 'server.listen': ('127.0.0.1', '::1', '', 'invalid_listen_address'), + 'server.port': (8042, 65535, 0, 'invalid_listen_port'), + 'server.public_web_origin': ( + None, + 'https://msg.lmm.best', + 'https://elsewhere.invalid', + 'hosting_origin_must_match_service', + ), + 'server.temporary_ttl': (3600, 1, 0, 'invalid_ttl'), + 'server.transfer_ttl': (86400, 1, 0, 'invalid_ttl'), + 'storage.postgres_dsn': ( + _REQUIRED, + 'service=isolated', + 'sqlite:///tmp/x', + 'invalid_postgres_dsn', + ), + 'storage.valkey_url': ( + None, + 'redis://localhost:6379/0', + 'http://localhost:6379', + 'invalid_valkey_url', + ), + 'storage.content': ( + '/var/lib/msgd/git/content', + '/tmp/config-content', + 'relative', + 'storage_paths_must_be_absolute', + ), + 'storage.repositories': ( + '/var/lib/msgd/git/repos', + '/tmp/config-repos', + 'relative', + 'storage_paths_must_be_absolute', + ), + 'storage.blobs': ( + '/var/lib/msgd/blobs/sha256', + '/tmp/config-blobs', + 'relative', + 'storage_paths_must_be_absolute', + ), + 'storage.staging': ( + '/var/lib/msgd/transfers/staging', + '/tmp/config-staging', + 'relative', + 'storage_paths_must_be_absolute', + ), + 'storage.service_keys': ( + '/var/lib/msgd/service', + '/tmp/config-service', + 'relative', + 'storage_paths_must_be_absolute', + ), + 'limits.request_bytes': (1048576, 256, 255, 'invalid_limit'), + 'limits.response_bytes': (1048576, 256, 255, 'invalid_limit'), + 'limits.path_bytes': (8192, 256, 255, 'invalid_limit'), + 'limits.part_bytes': (65536, 256, 255, 'invalid_limit'), + 'plugins.enabled': (list(BUILTINS), ['identity'], ['arbitrary'], 'unknown_plugin'), + 'tools.isolation': ('bubblewrap', 'bubblewrap', 'none', 'unsafe_tool_worker'), + 'tools.timeout_ms': (10000, 1, 0, 'invalid_tool_limit'), + 'tools.max_response_bytes': (4194304, 1, 0, 'invalid_tool_limit'), + 'tools.methods': (['GET', 'HEAD'], ['GET'], ['EXEC'], 'invalid_tool_methods'), + 'tools.ports': ([80, 443], [65535], [0], 'invalid_tool_ports'), + 'identity.credential_delivery_recovery_window': ( + 900, + '60m', + '61m', + 'invalid_credential_delivery_recovery_window', + ), + 'money.enabled': (True, True, False, 'invalid_money_configuration'), + 'money.currency_id': ('primary', 'primary', 'other', 'invalid_money_configuration'), + 'money.display_name': ('MSG', 'Community credits', '', 'invalid_money_display_name'), + 'money.code': ('MSG', 'CREDIT', 'msg', 'invalid_money_code'), + 'money.scale': (6, 6, 7, 'invalid_money_configuration'), + 'money.transfer_fee': (0, 0, 1, 'invalid_money_configuration'), + 'money.allow_overdraft': (False, False, True, 'invalid_money_configuration'), + 'hosting.base_capacity_bytes': (10485760, 1024**4, 0, 'invalid_hosting_capacity'), + 'hosting.recovery_marker': ( + None, + '/tmp/config-marker', + 'relative', + 'invalid_hosting_recovery_marker', + ), + 'hosting.content_group_read': (False, True, 'true', 'invalid_content_group_read'), + 'recovery.custodians': ([], [custodian], [custodian] * 17, 'invalid_recovery_custodians'), + 'mail.enabled': (_DISABLED, False, 'false', 'invalid_mail_enabled'), + 'mail.host': (_DISABLED, 'smtp.example.test', 'bad host', 'invalid_mail_host'), + 'mail.port': (_DISABLED, 465, 65536, 'invalid_mail_port'), + 'mail.tls': (_DISABLED, 'tls', 'plaintext', 'mail_tls_required'), + 'mail.sender': ( + _DISABLED, + 'Message service ', + 'sender\r\ninjected', + 'invalid_mail_sender', + ), + 'mail.credential_file': ( + _DISABLED, + '/tmp/unread-smtp-credentials', + 'relative', + 'invalid_mail_credential_file', + ), + 'recovery.custodians.id': ( + _DISABLED, + 'other-custodian', + 'INVALID', + 'invalid_recovery_custodian', + ), + 'recovery.custodians.name': ( + _DISABLED, + 'Other custodian', + '', + 'invalid_recovery_custodian', + ), + 'recovery.custodians.recipient': (_DISABLED, recipient, 'invalid', 'invalid_age_recipient'), + 'recovery.custodians.description': ( + _DISABLED, + 'Description', + 'x' * 501, + 'invalid_recovery_custodian', + ), + 'recovery.custodians.policy_ref': ( + _DISABLED, + '/wiki/recovery-policy', + 'x' * 501, + 'invalid_recovery_custodian', + ), } return rows, custodian def _toml(value): - if isinstance(value,dict): - return '{'+', '.join(key+' = '+_toml(item) for key,item in value.items())+'}' - if isinstance(value,(list,tuple)): - return '['+', '.join(_toml(item) for item in value)+']' - return json.dumps(value,ensure_ascii=False) + if isinstance(value, dict): + return '{' + ', '.join(key + ' = ' + _toml(item) for key, item in value.items()) + '}' + if isinstance(value, (list, tuple)): + return '[' + ', '.join(_toml(item) for item in value) + ']' + return json.dumps(value, ensure_ascii=False) def _write(directory, data, mail): - (directory/'msgd.toml').write_text('\n'.join( - '['+section+']\n'+'\n'.join(key+' = '+_toml(value) for key,value in values.items()) - for section,values in data.items())) - path=directory/'mail.toml' + (directory / 'msgd.toml').write_text( + '\n'.join( + '[' + + section + + ']\n' + + '\n'.join(key + ' = ' + _toml(value) for key, value in values.items()) + for section, values in data.items() + ) + ) + path = directory / 'mail.toml' if mail is None: path.unlink(missing_ok=True) else: - path.write_text('\n'.join(key+' = '+_toml(value) for key,value in mail.items())) + path.write_text('\n'.join(key + ' = ' + _toml(value) for key, value in mail.items())) def _effective(settings, field): - value=settings + value = settings for part in CONFIGURATION_FIELDS[field]['setting'].split('.'): if value is None: return None if part.isdigit(): - value=value[int(part)] if value else None + value = value[int(part)] if value else None else: - value=getattr(value,part) - if isinstance(value,Path): + value = getattr(value, part) + if isinstance(value, Path): return str(value) - if isinstance(value,tuple): - if field=='recovery.custodians': - return [{key:getattr(item,key) for key in ('id','name','recipient')} - for item in value] + if isinstance(value, tuple): + if field == 'recovery.custodians': + return [ + {key: getattr(item, key) for key in ('id', 'name', 'recipient')} for item in value + ] return list(value) return value @@ -116,74 +209,107 @@ def _effective(settings, field): def configuration_selftest(): """Return only status/code per field; never return loaded configuration values.""" from msg.config import load_settings - vectors,custodian=_vectors() - require(set(vectors)==set(CONFIGURATION_FIELDS),'configuration_diagnostic_coverage_missing') - results={} + + vectors, custodian = _vectors() + require(set(vectors) == set(CONFIGURATION_FIELDS), 'configuration_diagnostic_coverage_missing') + results = {} with TemporaryDirectory(prefix='msg-config-selftest-') as temporary: - directory=Path(temporary) - baseline={'storage':{'postgres_dsn':'service=msgd'}} - for field,(default,good,bad,error_code) in vectors.items(): + directory = Path(temporary) + baseline = {'storage': {'postgres_dsn': 'service=msgd'}} + for field, (default, good, bad, error_code) in vectors.items(): try: - _write(directory,baseline,None) - defaults=load_settings(directory) + _write(directory, baseline, None) + defaults = load_settings(directory) if default is _REQUIRED: - _write(directory,{},None) + _write(directory, {}, None) try: load_settings(directory) except Failure as exc: - require(exc.code==error_code,'configuration_default_mismatch') + require(exc.code == error_code, 'configuration_default_mismatch') else: raise Failure('configuration_default_mismatch') - default_status='required' + default_status = 'required' elif default is _DISABLED: - require(_effective(defaults,field) is None,'configuration_default_mismatch') - default_status='disabled' + require(_effective(defaults, field) is None, 'configuration_default_mismatch') + default_status = 'disabled' else: - require(_effective(defaults,field)==default,'configuration_default_mismatch') - default_status='pass' - for accepted,value in ((True,good),(False,bad)): - data=deepcopy(baseline) - mail=None + require( + _effective(defaults, field) == default, 'configuration_default_mismatch' + ) + default_status = 'pass' + for accepted, value in ((True, good), (False, bad)): + data = deepcopy(baseline) + mail = None if field.startswith('mail.'): - mail={'enabled':True,'host':'smtp.example.test','tls':'starttls', - 'sender':'sender@example.test'} - mail[field.split('.')[1]]=value + mail = { + 'enabled': True, + 'host': 'smtp.example.test', + 'tls': 'starttls', + 'sender': 'sender@example.test', + } + mail[field.split('.')[1]] = value elif field.startswith('recovery.custodians.'): - item=dict(custodian) - item[field.rsplit('.',1)[1]]=value - data['recovery']={'custodians':[item]} + item = dict(custodian) + item[field.rsplit('.', 1)[1]] = value + data['recovery'] = {'custodians': [item]} else: - section,key=field.split('.') - data.setdefault(section,{})[key]=value - _write(directory,data,mail) + section, key = field.split('.') + data.setdefault(section, {})[key] = value + _write(directory, data, mail) try: - loaded=load_settings(directory) + loaded = load_settings(directory) except Failure as exc: - require(not accepted and exc.code==error_code, - 'configuration_rejection_mismatch') + require( + not accepted and exc.code == error_code, + 'configuration_rejection_mismatch', + ) else: - require(accepted,'configuration_rejection_missing') - expected=(int(good[:-1])*60 if field=='identity.credential_delivery_recovery_window' - else None if field=='mail.enabled' else good) - require(_effective(loaded,field)==expected,'configuration_effective_value_mismatch') - results[field]={'status':'pass','default':default_status, - 'accepted':'pass','rejected':'pass'} - except (Failure,ValueError,TypeError,KeyError,OSError,AttributeError) as exc: - results[field]={'status':'fail','code':getattr(exc,'code','configuration_check_failed')} + require(accepted, 'configuration_rejection_missing') + expected = ( + int(good[:-1]) * 60 + if field == 'identity.credential_delivery_recovery_window' + else None + if field == 'mail.enabled' + else good + ) + require( + _effective(loaded, field) == expected, + 'configuration_effective_value_mismatch', + ) + results[field] = { + 'status': 'pass', + 'default': default_status, + 'accepted': 'pass', + 'rejected': 'pass', + } + except (Failure, ValueError, TypeError, KeyError, OSError, AttributeError) as exc: + results[field] = { + 'status': 'fail', + 'code': getattr(exc, 'code', 'configuration_check_failed'), + } return results def configuration_doctor(observations): """Parsing success does not prove listener, SMTP, OS permissions or live limits.""" - result={} - for field,spec in CONFIGURATION_FIELDS.items(): - row={'status':'pass' if observations.get('configuration',{}).get('ok') is True else 'fail', - 'scope':'configuration_loading','selftest_case':field} - check=spec['doctor_check'] - value=observations.get(check,{}) if check else {} - row['related_check']={'check':check,'status': - 'disabled' if value.get('status')=='disabled' else - 'pass' if value.get('ok') is True else - 'fail' if value.get('ok') is False else 'not_checked'} - result[field]=row + result = {} + for field, spec in CONFIGURATION_FIELDS.items(): + row = { + 'status': 'pass' if observations.get('configuration', {}).get('ok') is True else 'fail', + 'scope': 'configuration_loading', + 'selftest_case': field, + } + check = spec['doctor_check'] + value = observations.get(check, {}) if check else {} + row['related_check'] = { + 'check': check, + 'status': 'disabled' + if value.get('status') == 'disabled' + else 'pass' + if value.get('ok') is True + else 'fail' + if value.get('ok') is False + else 'not_checked', + } + result[field] = row return result diff --git a/src/msg/admin/content_edit_check.py b/src/msg/admin/content_edit_check.py index 5e08bd9a..f528f221 100644 --- a/src/msg/admin/content_edit_check.py +++ b/src/msg/admin/content_edit_check.py @@ -1,4 +1,5 @@ """Bounded, real-operation content editing vector for the isolated selftest.""" + from msg.core.codec import b64 from msg.core.errors import require from msg.core.models import ResourceRef @@ -9,20 +10,32 @@ async def check_content_editing(app, call, register, test_path): other_key, other = await register('content-edit-outsider') original = '# Selftest\n\nKeep this line.\nReplace this word.\n' expected_text = original.replace('word', 'value') - created = await call('file.create', { - 'parent': test_path + '/@content-edit-owner/files', 'name': 'edit.md', - 'data': b64(original.encode()), 'media_type': 'text/markdown', - }, owner_key, owner) + created = await call( + 'file.create', + { + 'parent': test_path + '/@content-edit-owner/files', + 'name': 'edit.md', + 'data': b64(original.encode()), + 'media_type': 'text/markdown', + }, + owner_key, + owner, + ) require(created.status == 'ok', 'selftest_content_create_failed') rid = created.resources[0].id old_ref = created.resources[0] async with app.metadata.transaction(write=False) as tx: original_revision = await tx.revision(old_ref) patch = {'kind': 'exact', 'exact': 'word', 'replacement': 'value'} - args = {'id': rid, 'base_revision': old_ref.revision, - 'base_generation': created.data['generation'], 'patch': patch} - patched = await call('file.patch', args, owner_key, owner, - expected=((rid, created.data['generation']),)) + args = { + 'id': rid, + 'base_revision': old_ref.revision, + 'base_generation': created.data['generation'], + 'patch': patch, + } + patched = await call( + 'file.patch', args, owner_key, owner, expected=((rid, created.data['generation']),) + ) require(patched.status == 'ok', 'selftest_content_patch_failed') generation = patched.data['generation'] tables = ('resources', 'revisions', 'events', 'results', 'audit', 'jobs') @@ -30,27 +43,43 @@ async def check_content_editing(app, call, register, test_path): before = tuple(tx.one('SELECT COUNT(*) FROM ' + table)[0] for table in tables) current_before = await tx.resource(rid) # Current generation lets this request reach the stale base_revision check. - stale = await call('file.patch', args, owner_key, owner, - expected=((rid, generation),)) - denied = await call('file.patch', { - **args, 'base_revision': patched.resources[0].revision, - 'base_generation': generation, - 'patch': {'kind': 'exact', 'exact': 'value', 'replacement': 'unauthorized'}, - }, other_key, other, expected=((rid, generation),)) + stale = await call('file.patch', args, owner_key, owner, expected=((rid, generation),)) + denied = await call( + 'file.patch', + { + **args, + 'base_revision': patched.resources[0].revision, + 'base_generation': generation, + 'patch': {'kind': 'exact', 'exact': 'value', 'replacement': 'unauthorized'}, + }, + other_key, + other, + expected=((rid, generation),), + ) current = await call('file.read', {'id': rid}, owner_key, owner) previous = await call('file.read', {'id': rid, 'revision': old_ref.revision}, owner_key, owner) history = await call('file.read', {'id': rid, 'view': 'history', 'limit': 10}, owner_key, owner) async with app.metadata.transaction(write=False) as tx: after = tuple(tx.one('SELECT COUNT(*) FROM ' + table)[0] for table in tables) - unchanged = (await tx.resource(rid) == current_before and - await tx.revision(old_ref) == original_revision) + unchanged = ( + await tx.resource(rid) == current_before + and await tx.revision(old_ref) == original_revision + ) revision = await tx.revision(ResourceRef(id=rid)) - return (stale.error is not None and stale.error.code == 'revision_conflict' and - denied.error is not None and denied.error.code == 'permission_denied' and - all(result.status == 'ok' for result in (current, previous, history)) and - current.data['content'] == expected_text and previous.data['content'] == original and - {row['id'] for row in history.data['revisions']} == - {old_ref.revision, patched.resources[0].revision} and - len(history.data['revisions']) == 2 and - revision.parents == (old_ref.revision,) and revision.author == owner and - generation == created.data['generation'] + 1 and before == after and unchanged) + return ( + stale.error is not None + and stale.error.code == 'revision_conflict' + and denied.error is not None + and denied.error.code == 'permission_denied' + and all(result.status == 'ok' for result in (current, previous, history)) + and current.data['content'] == expected_text + and previous.data['content'] == original + and {row['id'] for row in history.data['revisions']} + == {old_ref.revision, patched.resources[0].revision} + and len(history.data['revisions']) == 2 + and revision.parents == (old_ref.revision,) + and revision.author == owner + and generation == created.data['generation'] + 1 + and before == after + and unchanged + ) diff --git a/src/msg/admin/custodial_check.py b/src/msg/admin/custodial_check.py index 04873034..773072e7 100644 --- a/src/msg/admin/custodial_check.py +++ b/src/msg/admin/custodial_check.py @@ -1,4 +1,5 @@ """Read-only lifecycle inspection and an isolated real-age history selftest.""" + from dataclasses import replace from pathlib import Path from tempfile import TemporaryDirectory @@ -12,20 +13,33 @@ async def inspect_custodial_history(tx): from msg.security.custodial_migration import snapshot + constraint = tx.one("""SELECT pg_get_constraintdef(oid) FROM pg_constraint WHERE conrelid='custodial_vault'::regclass AND conname='custodial_vault_lifecycle_check'""") - require(constraint is not None and 'decrypt_only' in constraint[0], 'custodial_lifecycle_schema_missing') + require( + constraint is not None and 'decrypt_only' in constraint[0], + 'custodial_lifecycle_schema_missing', + ) states = [] for subject, status, challenge, body in tx.rows( - 'SELECT subject,status,challenge,body FROM custodial_upgrades ORDER BY id'): + 'SELECT subject,status,challenge,body FROM custodial_upgrades ORDER BY id' + ): if status == 'failed': continue view = (await snapshot(tx, subject, status, loads(challenge), loads(body))).data - states.append({'subject': subject, 'challenge_id': view['challenge_id'], 'phase': view['phase'], - 'unresolved_count': len(view['unresolved_revisions']), 'inventory_changed': view['inventory_changed'], - 'retirement': view['retirement']}) - return {'migrations': states, 'backup_retirement_verified': False, - 'verification_scope': 'online_state_only'} + states.append({ + 'subject': subject, + 'challenge_id': view['challenge_id'], + 'phase': view['phase'], + 'unresolved_count': len(view['unresolved_revisions']), + 'inventory_changed': view['inventory_changed'], + 'retirement': view['retirement'], + }) + return { + 'migrations': states, + 'backup_retirement_verified': False, + 'verification_scope': 'online_state_only', + } async def check_custodial_history(app, now): @@ -36,42 +50,90 @@ async def check_custodial_history(app, now): from msg.core.models import Resource, Revision from msg.transports.client import HTTPTransport from msg.transports.http import create_app + require(app.selftest_run_id is not None, 'isolated_selftest_required') with TemporaryDirectory(prefix='msg-history-selftest-') as directory: async with httpx.AsyncClient(transport=httpx.ASGITransport(app=create_app(app))) as http: - client = MsgClient(ClientState(Path(directory), server=app.settings.service_url), - HTTPTransport(app.settings.service_url, http=http), clock=lambda: now) + client = MsgClient( + ClientState(Path(directory), server=app.settings.service_url), + HTTPTransport(app.settings.service_url, http=http), + clock=lambda: now, + ) created = client.checked(await client.custodial('history-' + uuid4().hex[:12])) subject = client.state.subject - ciphertext = _age('--encrypt', '--recipient', created.data['encryption_recipient'], - input_data=b'isolated historical plaintext') + ciphertext = _age( + '--encrypt', + '--recipient', + created.data['encryption_recipient'], + input_data=b'isolated historical plaintext', + ) # Explicit fixture fact, not a new privileged network operation. The # normal custodial credential ceiling does not grant keystore writes. async with app.metadata.transaction(write=True) as tx: - folder = tx.one("SELECT id FROM resources WHERE parent=? AND name='keystore'", (subject,))[0] - resource = Resource(id='ks_' + uuid4().hex, type='keystore', type_version=1, - name='history.age', parent=folder, owner=subject, group='g_public', mode=0o600, - generation=0, revision=None, state='active', created_at=now, created_by=subject, - modified_at=now, modified_by=subject) + folder = tx.one( + "SELECT id FROM resources WHERE parent=? AND name='keystore'", (subject,) + )[0] + resource = Resource( + id='ks_' + uuid4().hex, + type='keystore', + type_version=1, + name='history.age', + parent=folder, + owner=subject, + group='g_public', + mode=0o600, + generation=0, + revision=None, + state='active', + created_at=now, + created_by=subject, + modified_at=now, + modified_by=subject, + ) await tx.insert(resource) blob = await app.contents.put_bytes(ciphertext, 'application/octet-stream') - revision = Revision(format_version=1, id='v_' + uuid4().hex, resource_id=resource.id, - parents=(), content=blob, relations=(), actor=subject, subject=subject, author=subject, - created_at=now, manifest_digest='') - revision = replace(revision, manifest_digest=digest({k: v for k, v in wire(revision).items() - if k not in {'manifest_digest', 'signature'}})) + revision = Revision( + format_version=1, + id='v_' + uuid4().hex, + resource_id=resource.id, + parents=(), + content=blob, + relations=(), + actor=subject, + subject=subject, + author=subject, + created_at=now, + manifest_digest='', + ) + revision = replace( + revision, + manifest_digest=digest({ + k: v + for k, v in wire(revision).items() + if k not in {'manifest_digest', 'signature'} + }), + ) await app.contents.pin(blob, revision.id) await app.contents.commit_revision(folder, revision) await tx.append_revision(revision) await tx.replace(replace(resource, generation=1, revision=revision.id), 0) tx.set_setting('keystore_format:' + revision.id, 'age') - pending = client.checked(await client.upgrade_custodial('history-self-' + uuid4().hex[:12], - external_ciphertexts_migrated=True)) + pending = client.checked( + await client.upgrade_custodial( + 'history-self-' + uuid4().hex[:12], external_ciphertexts_migrated=True + ) + ) if pending.data['status'] != 'pending_rewrap': return False result = await migrate(client) if not result['inventory']['history_recoverable']: return False - done = client.checked(await transition(client, 'finalize', external_ciphertexts_migrated=True)) - return (done.data['history_recoverable'] and done.data['retirement']['online_encryption_key_deleted'] - and not done.data['retirement']['backup_retired'] and client.state.token is None) + done = client.checked( + await transition(client, 'finalize', external_ciphertexts_migrated=True) + ) + return ( + done.data['history_recoverable'] + and done.data['retirement']['online_encryption_key_deleted'] + and not done.data['retirement']['backup_retired'] + and client.state.token is None + ) diff --git a/src/msg/admin/diagnostics.py b/src/msg/admin/diagnostics.py index f9d4abac..1e2a4e96 100644 --- a/src/msg/admin/diagnostics.py +++ b/src/msg/admin/diagnostics.py @@ -1,66 +1,95 @@ """Read-only deployment inspection and disposable, real-operation self-tests.""" + from __future__ import annotations + import asyncio -from contextlib import contextmanager -from datetime import UTC, datetime, timedelta -from importlib.util import find_spec import os -from pathlib import Path import shutil -import psycopg import stat import subprocess import sys import tempfile import uuid +from contextlib import contextmanager +from datetime import UTC, datetime, timedelta +from importlib.util import find_spec +from pathlib import Path from urllib.parse import quote + +import psycopg from psycopg import sql -from msg.core.codec import canonical, decode, digest, loads, unb64, wire, b64 -from msg.core.errors import Failure, require -from msg.core.models import Certificate, Resource, ResourceRef, Subject, AuditEvent from msg.constants import ROOT_SUBJECT +from msg.core.codec import b64, canonical, decode, digest, loads, unb64, wire +from msg.core.errors import Failure, require +from msg.core.models import Certificate, ResourceRef @contextmanager def temporary_postgres(): """Run selftest against a new local cluster, never an installed database.""" - configured=os.environ.get('MSG_TEST_POSTGRES_URL_TEMPLATE') + configured = os.environ.get('MSG_TEST_POSTGRES_URL_TEMPLATE') if configured: - name='msg_selftest_'+uuid.uuid4().hex - admin=configured.format(database='postgres') - with psycopg.connect(admin,autocommit=True) as connection: + name = 'msg_selftest_' + uuid.uuid4().hex + admin = configured.format(database='postgres') + with psycopg.connect(admin, autocommit=True) as connection: connection.execute(sql.SQL('CREATE DATABASE {}').format(sql.Identifier(name))) try: yield configured.format(database=name) finally: - with psycopg.connect(admin,autocommit=True) as connection: - connection.execute(sql.SQL('DROP DATABASE {} WITH (FORCE)').format(sql.Identifier(name))) + with psycopg.connect(admin, autocommit=True) as connection: + connection.execute( + sql.SQL('DROP DATABASE {} WITH (FORCE)').format(sql.Identifier(name)) + ) return - with tempfile.TemporaryDirectory(prefix='msg-selftest-pg-',dir='/tmp') as temporary: - root=Path(temporary) - data=root/'data' - socket=root/'socket';socket.mkdir() - subprocess.run(['initdb','-D',str(data),'-A','trust','--no-instructions'], - check=True,capture_output=True,text=True) + with tempfile.TemporaryDirectory(prefix='msg-selftest-pg-', dir='/tmp') as temporary: + root = Path(temporary) + data = root / 'data' + socket = root / 'socket' + socket.mkdir() + subprocess.run( + ['initdb', '-D', str(data), '-A', 'trust', '--no-instructions'], + check=True, + capture_output=True, + text=True, + ) try: - subprocess.run(['pg_ctl','-D',str(data),'-l',str(root/'postgres.log'), - '-o',f"-k {socket} -h '' -p 5432",'-w','start'], - check=True,capture_output=True,text=True) - yield f'postgresql://localhost:5432/postgres?host={quote(str(socket),safe="")}' + subprocess.run( + [ + 'pg_ctl', + '-D', + str(data), + '-l', + str(root / 'postgres.log'), + '-o', + f"-k {socket} -h '' -p 5432", + '-w', + 'start', + ], + check=True, + capture_output=True, + text=True, + ) + yield f'postgresql://localhost:5432/postgres?host={quote(str(socket), safe="")}' finally: - subprocess.run(['pg_ctl','-D',str(data),'-m','immediate','-w','stop'], - check=False,capture_output=True,text=True) + subprocess.run( + ['pg_ctl', '-D', str(data), '-m', 'immediate', '-w', 'stop'], + check=False, + capture_output=True, + text=True, + ) class ReadOnlyStore: """Inspect PostgreSQL without schema creation or repair.""" - def __init__(self,dsn): - self.dsn=dsn + + def __init__(self, dsn): + self.dsn = dsn async def inspect(self, callback): from msg.storage.postgres import PostgresMetadataStore - store=PostgresMetadataStore(self.dsn,initialize=False) + + store = PostgresMetadataStore(self.dsn, initialize=False) async with store.transaction(write=False) as session: return await callback(session) @@ -68,26 +97,32 @@ async def inspect(self, callback): def feature_results(features, observations, field, *, enabled_plugins=None): """Link manifest entries to checks actually run; never synthesize a pass.""" from msg.plugins.features import FEATURE_SOURCES - result={} + + result = {} for feature in features: - feature_id=feature['feature_id'] - name=feature[field] - if (not feature['enabled_by_default'] or - (enabled_plugins is not None and - FEATURE_SOURCES[feature_id][0] not in enabled_plugins)): - result[feature_id]={'status':'disabled','check':name} + feature_id = feature['feature_id'] + name = feature[field] + if not feature['enabled_by_default'] or ( + enabled_plugins is not None and FEATURE_SOURCES[feature_id][0] not in enabled_plugins + ): + result[feature_id] = {'status': 'disabled', 'check': name} elif name is None: - result[feature_id]={'status':'skip','check':None} + result[feature_id] = {'status': 'skip', 'check': None} elif name not in observations: - result[feature_id]={'status':'fail','check':name,'reason':'check_missing'} + result[feature_id] = {'status': 'fail', 'check': name, 'reason': 'check_missing'} else: - value=observations[name] - if isinstance(value,dict): - status=('skip' if value.get('status')=='disabled' else - 'pass' if value.get('ok') is True else 'fail') + value = observations[name] + if isinstance(value, dict): + status = ( + 'skip' + if value.get('status') == 'disabled' + else 'pass' + if value.get('ok') is True + else 'fail' + ) else: - status='pass' if value is True else 'fail' - result[feature_id]={'status':status,'check':name} + status = 'pass' if value is True else 'fail' + result[feature_id] = {'status': status, 'check': name} return result @@ -96,70 +131,117 @@ def inspect_recovery_checkpoint(): from importlib.resources import files from msg.admin.recovery_replay import FORMAT, SUPPORTED_FACTS - schema=loads(files('msg.data').joinpath('recovery-checkpoint.schema.json').read_bytes()) - checkpoint=schema['properties']['checkpoint']['properties'] - kinds=checkpoint['entries']['items']['properties']['kind']['enum'] - require(checkpoint['format'].get('const')==FORMAT and len(kinds)==len(set(kinds)) and - set(kinds)==SUPPORTED_FACTS,'recovery_checkpoint_schema_drift') - return {'fact_kinds':len(kinds),'promotion':'blocked'} + + schema = loads(files('msg.data').joinpath('recovery-checkpoint.schema.json').read_bytes()) + checkpoint = schema['properties']['checkpoint']['properties'] + kinds = checkpoint['entries']['items']['properties']['kind']['enum'] + require( + checkpoint['format'].get('const') == FORMAT + and len(kinds) == len(set(kinds)) + and set(kinds) == SUPPORTED_FACTS, + 'recovery_checkpoint_schema_drift', + ) + return {'fact_kinds': len(kinds), 'promotion': 'blocked'} def inspect_lexical_search(app): - specs=[spec for spec in app.registry.operations() if spec.name=='discovery.lexical_search'] - require(bool(specs) and all(spec.effect=='read' for spec in specs),'lexical_search_contract_invalid') - return {'versions':sorted(spec.version for spec in specs),'read_only':True} + specs = [spec for spec in app.registry.operations() if spec.name == 'discovery.lexical_search'] + require( + bool(specs) and all(spec.effect == 'read' for spec in specs), + 'lexical_search_contract_invalid', + ) + return {'versions': sorted(spec.version for spec in specs), 'read_only': True} def inspect_hosting(app): - for name in ('hosting.create','hosting.deploy'): - require(app.registry.operation(name).require_signature,'hosting_contract_invalid') + for name in ('hosting.create', 'hosting.deploy'): + require(app.registry.operation(name).require_signature, 'hosting_contract_invalid') app.registry.resource_type('website') - return {'operations':['hosting.create@1','hosting.deploy@1']} + return {'operations': ['hosting.create@1', 'hosting.deploy@1']} -async def _selftest_search_hosting(app,call,register,test_path): +async def _selftest_search_hosting(app, call, register, test_path): """Exercise search and publishing only inside the disposable test namespace.""" - handle='test-features' - key,uid=await register(handle) - marker='selftestsearch'+uuid.uuid4().hex[:12] - search=False + handle = 'test-features' + key, uid = await register(handle) + marker = 'selftestsearch' + uuid.uuid4().hex[:12] + search = False try: - post=await call('content.post_create',{'parent':test_path+'/tmp','body':'isolated '+marker},key,uid) - require(post.status=='ok','selftest_search_post_failed') - found=await call('discovery.lexical_search',{'scope':test_path+'/tmp','terms':marker}, - key,uid,contract_version=4) - denied=await call('discovery.lexical_search',{'scope':test_path+'/private','terms':marker}, - key,uid,contract_version=4) - search=(found.status=='ok' and - [item['id'] for item in found.data['items']]==[post.resources[0].id] and - denied.error is not None and denied.error.code=='permission_denied') + post = await call( + 'content.post_create', + {'parent': test_path + '/tmp', 'body': 'isolated ' + marker}, + key, + uid, + ) + require(post.status == 'ok', 'selftest_search_post_failed') + found = await call( + 'discovery.lexical_search', + {'scope': test_path + '/tmp', 'terms': marker}, + key, + uid, + contract_version=4, + ) + denied = await call( + 'discovery.lexical_search', + {'scope': test_path + '/private', 'terms': marker}, + key, + uid, + contract_version=4, + ) + search = ( + found.status == 'ok' + and [item['id'] for item in found.data['items']] == [post.resources[0].id] + and denied.error is not None + and denied.error.code == 'permission_denied' + ) except Failure: - search=False - hosting=False + search = False + hosting = False try: - page=b'

isolated selftest

' - source=await call('content.file_put',{'parent':test_path+'/@'+handle+'/files','name':'index.html', - 'data':b64(page),'media_type':'text/html'},key,uid) - require(source.status=='ok','selftest_hosting_source_failed') - site=await call('hosting.create',{'parent':test_path+'/@'+handle,'name':'site'},key,uid) - require(site.status=='ok','selftest_hosting_create_failed') - website_id=site.resources[0].id - deployed=await call('hosting.deploy',{'id':website_id, - 'entries':[{'path':'index.html','source':wire(source.resources[0])}]},key,uid, - expected=((website_id,site.data['generation']),)) - require(deployed.status=='ok','selftest_hosting_deploy_failed') + page = b'

isolated selftest

' + source = await call( + 'content.file_put', + { + 'parent': test_path + '/@' + handle + '/files', + 'name': 'index.html', + 'data': b64(page), + 'media_type': 'text/html', + }, + key, + uid, + ) + require(source.status == 'ok', 'selftest_hosting_source_failed') + site = await call( + 'hosting.create', {'parent': test_path + '/@' + handle, 'name': 'site'}, key, uid + ) + require(site.status == 'ok', 'selftest_hosting_create_failed') + website_id = site.resources[0].id + deployed = await call( + 'hosting.deploy', + { + 'id': website_id, + 'entries': [{'path': 'index.html', 'source': wire(source.resources[0])}], + }, + key, + uid, + expected=((website_id, site.data['generation']),), + ) + require(deployed.status == 'ok', 'selftest_hosting_deploy_failed') async with app.metadata.transaction(write=False) as tx: - website=await tx.resource(website_id) - revision=await tx.revision(ResourceRef(id=website_id,revision=website.revision)) - manifest=loads(await app.contents.read_bytes(revision.content)) - entry=decode(ResourceRef,manifest['entries']['index.html']) - published=await app.contents.read_bytes((await tx.revision(entry)).content) - site_path=await tx.path(website_id) - hosting=(site_path.startswith(test_path+'/') and set(manifest['entries'])=={'index.html'} and - published==page) - except (Failure,KeyError,TypeError,ValueError): - hosting=False - return {'lexical_search_access':search,'hosting_deploy':hosting} + website = await tx.resource(website_id) + revision = await tx.revision(ResourceRef(id=website_id, revision=website.revision)) + manifest = loads(await app.contents.read_bytes(revision.content)) + entry = decode(ResourceRef, manifest['entries']['index.html']) + published = await app.contents.read_bytes((await tx.revision(entry)).content) + site_path = await tx.path(website_id) + hosting = ( + site_path.startswith(test_path + '/') + and set(manifest['entries']) == {'index.html'} + and published == page + ) + except Failure, KeyError, TypeError, ValueError: + hosting = False + return {'lexical_search_access': search, 'hosting_deploy': hosting} async def authority_snapshot_drift(app, root, online, tx): @@ -171,28 +253,38 @@ async def authority_snapshot_drift(app, root, online, tx): well; a narrower old grant must not be described as covering a broad one. """ from msg.security.policy import constraints_subset, scope_subset - require(root.issuance is not None and online.issuance is not None,'issuer_not_ca') + + require(root.issuance is not None and online.issuance is not None, 'issuer_not_ca') async def missing(expected, actual): - gaps=[] + gaps = [] for desired in expected: - covered=set() + covered = set() for grant in actual: - if (grant.capability==desired.capability and grant.version==desired.version and - await scope_subset(desired.scope,grant.scope,tx) and - constraints_subset(desired.constraints,grant.constraints)): + if ( + grant.capability == desired.capability + and grant.version == desired.version + and await scope_subset(desired.scope, grant.scope, tx) + and constraints_subset(desired.constraints, grant.constraints) + ): covered.update(grant.operations) - operations=sorted(desired.operations-covered) + operations = sorted(desired.operations - covered) if operations: - gaps.append({'capability':desired.capability,'version':desired.version, - 'scope':wire(desired.scope),'operations':operations}) + gaps.append({ + 'capability': desired.capability, + 'version': desired.version, + 'scope': wire(desired.scope), + 'operations': operations, + }) return gaps - primary=app.primary_ceiling() - base=app.base_grants() - return {'root_use':await missing(primary,root.grants), - 'root_issue':await missing(primary,root.issuance.issue_grants), - 'online_issue':await missing(base,online.issuance.issue_grants)} + primary = app.primary_ceiling() + base = app.base_grants() + return { + 'root_use': await missing(primary, root.grants), + 'root_issue': await missing(primary, root.issuance.issue_grants), + 'online_issue': await missing(base, online.issuance.issue_grants), + } def doctor(config_dir=Path('/etc/msgd'), *, clock=None): @@ -200,650 +292,1242 @@ def doctor(config_dir=Path('/etc/msgd'), *, clock=None): # Running the read-only coroutine in a short-lived thread avoids nesting an # asyncio.run() inside adapters/tests with an already running event loop. from concurrent.futures import ThreadPoolExecutor + with ThreadPoolExecutor(max_workers=1) as pool: - return pool.submit(lambda: asyncio.run(_doctor(config_dir,clock))).result() + return pool.submit(lambda: asyncio.run(_doctor(config_dir, clock))).result() -async def _doctor(config_dir,clock): +async def _doctor(config_dir, clock): from msg.application import Application + from msg.bootstrap import feature_manifest, manifest from msg.config import load_settings - from msg.bootstrap import manifest,feature_manifest from msg.security.certificates import CertificateValidator - checks={} - warnings=[] - def success(name,**data):checks[name]={'ok':True,**data} - def failed(name,code,**data):checks[name]={'ok':False,'code':code,**data} - path=Path(config_dir) + + checks = {} + warnings = [] + + def success(name, **data): + checks[name] = {'ok': True, **data} + + def failed(name, code, **data): + checks[name] = {'ok': False, 'code': code, **data} + + path = Path(config_dir) try: - settings=load_settings(path) - app=Application(settings,clock=clock) + settings = load_settings(path) + app = Application(settings, clock=clock) success('configuration') - success('money_config',enabled=settings.money.enabled, - currency_id=settings.money.currency_id, - display_name=settings.money.display_name,code=settings.money.code, - scale=settings.money.scale,transfer_fee=settings.money.transfer_fee, - allow_overdraft=settings.money.allow_overdraft) - except (Failure,ValueError,KeyError,OSError) as exc: - failed('configuration',getattr(exc,'code','invalid_configuration')) + success( + 'money_config', + enabled=settings.money.enabled, + currency_id=settings.money.currency_id, + display_name=settings.money.display_name, + code=settings.money.code, + scale=settings.money.scale, + transfer_fee=settings.money.transfer_fee, + allow_overdraft=settings.money.allow_overdraft, + ) + except (Failure, ValueError, KeyError, OSError) as exc: + failed('configuration', getattr(exc, 'code', 'invalid_configuration')) from msg.admin.config_check import configuration_doctor - return {'ok':False,'root_id':ROOT_SUBJECT,'checks':checks, - 'configuration_fields':configuration_doctor(checks)} + + return { + 'ok': False, + 'root_id': ROOT_SUBJECT, + 'checks': checks, + 'configuration_fields': configuration_doctor(checks), + } # This read-only phase checks installed contracts; real behavior belongs to # the separate isolated selftest vectors below. - for feature,owner,operations in ( - ('content_editing','file',('file.create','file.patch','file.read')), - ('private_dm','communication',('communication.dm_request','communication.dm_accept', - 'communication.dm_send','communication.dm_list')), - ('transfer','transfer',('transfer.open','transfer.part_put','transfer.part_get', - 'transfer.status','transfer.seal'))): + for feature, owner, operations in ( + ('content_editing', 'file', ('file.create', 'file.patch', 'file.read')), + ( + 'private_dm', + 'communication', + ( + 'communication.dm_request', + 'communication.dm_accept', + 'communication.dm_send', + 'communication.dm_list', + ), + ), + ( + 'transfer', + 'transfer', + ( + 'transfer.open', + 'transfer.part_put', + 'transfer.part_get', + 'transfer.status', + 'transfer.seal', + ), + ), + ): if owner not in app.registry._plugins: - checks[feature]={'ok':True,'status':'disabled'} + checks[feature] = {'ok': True, 'status': 'disabled'} continue try: for name in operations: - operation=app.registry.operation(name) + operation = app.registry.operation(name) app.registry.schema(operation.input_schema) app.registry.schema(operation.output_schema) - success(feature,scope='registered_contracts',operations=list(operations)) - except Failure as exc:failed(feature,exc.code) - if sys.version_info[:2]>=(3,15):success('python',version='.'.join(map(str,sys.version_info[:3]))) - else:failed('python','python_315_required',actual='.'.join(map(str,sys.version_info[:3]))) - missing=[name for name in ('cryptography','starlette','uvicorn','httpx','jsonschema','aiohttp','dns','graphql','psycopg','valkey') if find_spec(name) is None] - if not missing:success('dependencies') - else:failed('dependencies','dependency_unavailable',missing=missing) - if shutil.which('git'):success('git') - else:failed('git','git_missing') + success(feature, scope='registered_contracts', operations=list(operations)) + except Failure as exc: + failed(feature, exc.code) + # Doctor must diagnose an unsupported interpreter rather than report success. + minimum_python = (3, 15) + if sys.version_info[:2] >= minimum_python: + success('python', version='.'.join(map(str, sys.version_info[:3]))) + else: + failed('python', 'python_315_required', actual='.'.join(map(str, sys.version_info[:3]))) + missing = [ + name + for name in ( + 'cryptography', + 'starlette', + 'uvicorn', + 'httpx', + 'jsonschema', + 'aiohttp', + 'dns', + 'graphql', + 'psycopg', + 'valkey', + ) + if find_spec(name) is None + ] + if not missing: + success('dependencies') + else: + failed('dependencies', 'dependency_unavailable', missing=missing) + if shutil.which('git'): + success('git') + else: + failed('git', 'git_missing') if 'extensions' in settings.server.plugins: from msg.admin.tool_check import inspect_tool_sandbox - try:success('tool_sandbox',**await inspect_tool_sandbox(app)) - except (Failure,OSError) as exc: - failed('tool_sandbox',getattr(exc,'code','tool_isolation_failed')) + + try: + success('tool_sandbox', **await inspect_tool_sandbox(app)) + except (Failure, OSError) as exc: + failed('tool_sandbox', getattr(exc, 'code', 'tool_isolation_failed')) else: - checks['tool_sandbox']={'ok':True,'status':'disabled'} + checks['tool_sandbox'] = {'ok': True, 'status': 'disabled'} try: - trust=loads(settings.trust_file.read_bytes()) - root=decode(Certificate,trust['certificate']) - validator=CertificateValidator(app.registry,root,unb64(trust['public_key']),settings.service_url, - clock or (lambda:datetime.now(UTC))) + trust = loads(settings.trust_file.read_bytes()) + root = decode(Certificate, trust['certificate']) + validator = CertificateValidator( + app.registry, + root, + unb64(trust['public_key']), + settings.service_url, + clock or (lambda: datetime.now(UTC)), + ) + async def inspect(tx): - require(tx.one('SELECT version FROM schema_version')[0]==1,'schema_version_unknown') + require(tx.one('SELECT version FROM schema_version')[0] == 1, 'schema_version_unknown') success('storage') try: from msg.admin.custodial_check import inspect_custodial_history + success('custodial_history', **await inspect_custodial_history(tx)) - except Failure as exc:failed('custodial_history',exc.code) + except Failure as exc: + failed('custodial_history', exc.code) if 'orders' in settings.server.plugins: try: from msg.admin.market_check import inspect_market - success('market',**(await inspect_market(tx))) - except (Failure,KeyError,ValueError,psycopg.Error) as exc: - failed('market',getattr(exc,'code','market_inspection_failed')) + + success('market', **(await inspect_market(tx))) + except (Failure, KeyError, ValueError, psycopg.Error) as exc: + failed('market', getattr(exc, 'code', 'market_inspection_failed')) else: - checks['market']={'ok':True,'status':'disabled'} + checks['market'] = {'ok': True, 'status': 'disabled'} try: from msg.admin.market_check import inspect_bounty_contracts, inspect_clearing - clearing = inspect_clearing(app,tx) - await inspect_bounty_contracts(app,tx) - success('market_clearing',**clearing) - except (Failure,OSError,ValueError,KeyError,psycopg.Error) as exc: - failed('market_clearing',getattr(exc,'code','market_inspection_failed')) + + clearing = inspect_clearing(app, tx) + await inspect_bounty_contracts(app, tx) + success('market_clearing', **clearing) + except (Failure, OSError, ValueError, KeyError, psycopg.Error) as exc: + failed('market_clearing', getattr(exc, 'code', 'market_inspection_failed')) try: from msg.core.requests import SECRET_DELIVERY_MIN_VERSION - columns={row[0] for row in tx.rows("SELECT column_name FROM information_schema.columns " - "WHERE table_schema='public' AND table_name='token_deliveries'")} - require({'credential_id','subject','request_id','request_digest','recovery_verifier', - 'recovery_expires_at','claimed_at','consumed_at'} <= columns, - 'credential_delivery_schema_missing') - for operation,version in SECRET_DELIVERY_MIN_VERSION.items(): - require(app.registry.operation(operation,version).effect=='transaction', - 'credential_delivery_contract_invalid') - success('credential_delivery',recovery_window_seconds=settings.credential_delivery_recovery_window, - release='once',secret_url=False,versions=SECRET_DELIVERY_MIN_VERSION) - except Failure as exc:failed('credential_delivery',exc.code) + + columns = { + row[0] + for row in tx.rows( + 'SELECT column_name FROM information_schema.columns ' + "WHERE table_schema='public' AND table_name='token_deliveries'" + ) + } + require( + { + 'credential_id', + 'subject', + 'request_id', + 'request_digest', + 'recovery_verifier', + 'recovery_expires_at', + 'claimed_at', + 'consumed_at', + } + <= columns, + 'credential_delivery_schema_missing', + ) + for operation, version in SECRET_DELIVERY_MIN_VERSION.items(): + require( + app.registry.operation(operation, version).effect == 'transaction', + 'credential_delivery_contract_invalid', + ) + success( + 'credential_delivery', + recovery_window_seconds=settings.credential_delivery_recovery_window, + release='once', + secret_url=False, + versions=SECRET_DELIVERY_MIN_VERSION, + ) + except Failure as exc: + failed('credential_delivery', exc.code) if 'achievements' in settings.server.plugins: try: from msg.admin.honor_check import inspect_honors - success('honors', **await inspect_honors(app,tx)) - except (Failure,OSError,ValueError,KeyError,psycopg.Error) as exc: - failed('honors',getattr(exc,'code','honor_inspection_failed')) + + success('honors', **await inspect_honors(app, tx)) + except (Failure, OSError, ValueError, KeyError, psycopg.Error) as exc: + failed('honors', getattr(exc, 'code', 'honor_inspection_failed')) else: - checks['honors']={'ok':True,'status':'disabled'} + checks['honors'] = {'ok': True, 'status': 'disabled'} if 'communication' in settings.server.plugins: try: - spec=app.registry.operation('communication.following') - require(spec.effect=='read','following_contract_invalid') - columns={row[0] for row in tx.rows("SELECT column_name FROM information_schema.columns " - "WHERE table_schema='public' AND table_name='watches'")} - require({'subject','resource'}<=columns,'following_schema_missing') - success('following',default='empty',read_only=True,max_page_size=100) - except Failure as exc:failed('following',exc.code) + spec = app.registry.operation('communication.following') + require(spec.effect == 'read', 'following_contract_invalid') + columns = { + row[0] + for row in tx.rows( + 'SELECT column_name FROM information_schema.columns ' + "WHERE table_schema='public' AND table_name='watches'" + ) + } + require({'subject', 'resource'} <= columns, 'following_schema_missing') + success('following', default='empty', read_only=True, max_page_size=100) + except Failure as exc: + failed('following', exc.code) try: - for kind in ('request','offer','checkpoint','proposal'): - require(app.registry.operation('communication.'+kind+'_create').effect=='transaction', - 'collaboration_contract_invalid') - require(app.registry.operation('communication.'+kind+'_get').effect=='read' and - app.registry.operation('communication.'+kind+'_list').effect=='read', - 'collaboration_contract_invalid') - success('collaboration',default='empty',storage='resource_revision', - kinds=['request','offer','checkpoint','proposal']) - except Failure as exc:failed('collaboration',exc.code) + for kind in ('request', 'offer', 'checkpoint', 'proposal'): + require( + app.registry.operation('communication.' + kind + '_create').effect + == 'transaction', + 'collaboration_contract_invalid', + ) + require( + app.registry.operation('communication.' + kind + '_get').effect + == 'read' + and app.registry.operation('communication.' + kind + '_list').effect + == 'read', + 'collaboration_contract_invalid', + ) + success( + 'collaboration', + default='empty', + storage='resource_revision', + kinds=['request', 'offer', 'checkpoint', 'proposal'], + ) + except Failure as exc: + failed('collaboration', exc.code) else: - checks['following']={'ok':True,'status':'disabled'} - checks['collaboration']={'ok':True,'status':'disabled'} + checks['following'] = {'ok': True, 'status': 'disabled'} + checks['collaboration'] = {'ok': True, 'status': 'disabled'} try: - await validator.validate(root.resource_id,tx) - require((await tx.subject(ROOT_SUBJECT)).local_only,'root_policy_corrupt') - success('root_trust',fingerprint=digest(unb64(trust['public_key']))) - except Failure as exc:failed('root_trust',exc.code) + await validator.validate(root.resource_id, tx) + require((await tx.subject(ROOT_SUBJECT)).local_only, 'root_policy_corrupt') + success('root_trust', fingerprint=digest(unb64(trust['public_key']))) + except Failure as exc: + failed('root_trust', exc.code) try: - online=tx.setting('online_ca_certificate') - require(online is not None,'issuer_not_ready') - online_certificate=await validator.validate(online,tx) + online = tx.setting('online_ca_certificate') + require(online is not None, 'issuer_not_ready') + online_certificate = await validator.validate(online, tx) from msg.security.crypto import Ed25519Signer - online_key=Ed25519Signer.from_bytes((settings.service_keys/'online.key').read_bytes()) - require(online_certificate.key_id==online_key.key_id,'issuer_key_mismatch') + + online_key = Ed25519Signer.from_bytes( + (settings.service_keys / 'online.key').read_bytes() + ) + require(online_certificate.key_id == online_key.key_id, 'issuer_key_mismatch') success('online_ca') - except (Failure,OSError,ValueError) as exc:failed('online_ca',getattr(exc,'code','issuer_key_missing')) + except (Failure, OSError, ValueError) as exc: + failed('online_ca', getattr(exc, 'code', 'issuer_key_missing')) try: - online_id=tx.setting('online_ca_certificate') - require(online_id is not None,'issuer_not_ready') - online_certificate=await tx.certificate(online_id) - gaps=await authority_snapshot_drift(app,root,online_certificate,tx) + online_id = tx.setting('online_ca_certificate') + require(online_id is not None, 'issuer_not_ready') + online_certificate = await tx.certificate(online_id) + gaps = await authority_snapshot_drift(app, root, online_certificate, tx) if any(gaps.values()): - failed('authority_snapshot','signed_authority_outdated',missing=gaps, - action='No automatic authority expansion. Preserve this chain and keep the new ' - 'operation unavailable, or explicitly rotate the root and reissue ' - 'certificates; rotation invalidates old chains.') + failed( + 'authority_snapshot', + 'signed_authority_outdated', + missing=gaps, + action='No automatic authority expansion. Preserve this chain and keep the new ' + 'operation unavailable, or explicitly rotate the root and reissue ' + 'certificates; rotation invalidates old chains.', + ) else: success('authority_snapshot') except Failure as exc: - failed('authority_snapshot',exc.code) - drift=[] + failed('authority_snapshot', exc.code) + drift = [] for expected in manifest()['resources']: try: - resource=await tx.resource(expected['id']) - for field in ('type','parent','owner','group','mode','name'): - desired=int(expected[field],8) if field=='mode' else expected[field] - if getattr(resource,field)!=desired:drift.append({'id':resource.id,'field':field}) - except Failure:drift.append({'id':expected['id'],'field':'missing'}) - if drift:failed('bootstrap','bootstrap_drift',drift=drift) - else:success('bootstrap') - previous=None - for raw, in tx.execute('SELECT body FROM audit ORDER BY seq'): - audit=loads(raw) - entry_digest=audit.pop('entry_digest') - require(audit.get('previous_digest')==previous and digest(audit)==entry_digest,'audit_chain_mismatch') - previous=entry_digest - success('audit',head=previous) + resource = await tx.resource(expected['id']) + for field in ('type', 'parent', 'owner', 'group', 'mode', 'name'): + desired = int(expected[field], 8) if field == 'mode' else expected[field] + if getattr(resource, field) != desired: + drift.append({'id': resource.id, 'field': field}) + except Failure: + drift.append({'id': expected['id'], 'field': 'missing'}) + if drift: + failed('bootstrap', 'bootstrap_drift', drift=drift) + else: + success('bootstrap') + previous = None + for (raw,) in tx.execute('SELECT body FROM audit ORDER BY seq'): + audit = loads(raw) + entry_digest = audit.pop('entry_digest') + require( + audit.get('previous_digest') == previous and digest(audit) == entry_digest, + 'audit_chain_mismatch', + ) + previous = entry_digest + success('audit', head=previous) + await ReadOnlyStore(settings.server.postgres_dsn).inspect(inspect) - except (Failure,OSError,ValueError,KeyError,psycopg.Error) as exc: - failed('inspection',getattr(exc,'code','inspection_failed')) + except (Failure, OSError, ValueError, KeyError, psycopg.Error) as exc: + failed('inspection', getattr(exc, 'code', 'inspection_failed')) if settings.server.valkey_url: try: from msg.storage.valkey_bus import ValkeyOutboxSignal + await asyncio.to_thread(ValkeyOutboxSignal(settings.server.valkey_url).client.ping) success('valkey') except Exception: - failed('valkey','valkey_unavailable') + failed('valkey', 'valkey_unavailable') else: - checks['valkey']={'ok':True,'status':'disabled'} - content=settings.server.content_dir - if (content/'private.git'/'HEAD').is_file():success('content_layout') - else:failed('content_layout','content_missing') + checks['valkey'] = {'ok': True, 'status': 'disabled'} + content = settings.server.content_dir + if (content / 'private.git' / 'HEAD').is_file(): + success('content_layout') + else: + failed('content_layout', 'content_missing') if settings.server.staging_dir.exists(): - disk=shutil.disk_usage(settings.server.staging_dir) - success('capacity',free_bytes=disk.free) + disk = shutil.disk_usage(settings.server.staging_dir) + success('capacity', free_bytes=disk.free) from msg.admin.root import root_envelope - protected=root_envelope(path).parent - if protected==path/'root': - warnings.append({'code':'legacy_root_storage','effect':'migrate_to_var_lib_msgd_root'}) + + protected = root_envelope(path).parent + if protected == path / 'root': + warnings.append({'code': 'legacy_root_storage', 'effect': 'migrate_to_var_lib_msgd_root'}) try: - require(not ((path/'root'/'key.json').exists() and - (settings.root_private_dir/'key.json').exists()), - 'duplicate_root_material') - mode=stat.S_IMODE(protected.stat().st_mode) - require(mode==0o700 and protected.stat().st_uid==0,'unsafe_root_permissions') - secret=protected/'key.json' + require( + not ( + (path / 'root' / 'key.json').exists() + and (settings.root_private_dir / 'key.json').exists() + ), + 'duplicate_root_material', + ) + mode = stat.S_IMODE(protected.stat().st_mode) + require(mode == 0o700 and protected.stat().st_uid == 0, 'unsafe_root_permissions') + secret = protected / 'key.json' try: - require(stat.S_IMODE(secret.stat().st_mode)==0o600 and secret.stat().st_uid==0,'unsafe_root_permissions') + require( + stat.S_IMODE(secret.stat().st_mode) == 0o600 and secret.stat().st_uid == 0, + 'unsafe_root_permissions', + ) except PermissionError: # The network service being unable to inspect a root-owned 0700 # directory is the intended deployment boundary. pass success('root_private_boundary') - except (Failure,OSError) as exc:failed('root_private_boundary',getattr(exc,'code','root_material_missing')) - try:success('recovery_checkpoint',**inspect_recovery_checkpoint()) - except (Failure,OSError,ValueError,KeyError,TypeError,AttributeError) as exc: - failed('recovery_checkpoint',getattr(exc,'code','recovery_checkpoint_schema_invalid')) - for name,plugin,inspect_feature in (('lexical_search','discovery',inspect_lexical_search), - ('hosting','extensions',inspect_hosting)): + except (Failure, OSError) as exc: + failed('root_private_boundary', getattr(exc, 'code', 'root_material_missing')) + try: + success('recovery_checkpoint', **inspect_recovery_checkpoint()) + except (Failure, OSError, ValueError, KeyError, TypeError, AttributeError) as exc: + failed('recovery_checkpoint', getattr(exc, 'code', 'recovery_checkpoint_schema_invalid')) + for name, plugin, inspect_feature in ( + ('lexical_search', 'discovery', inspect_lexical_search), + ('hosting', 'extensions', inspect_hosting), + ): if plugin not in settings.server.plugins: - checks[name]={'ok':True,'status':'disabled'} + checks[name] = {'ok': True, 'status': 'disabled'} continue - try:success(name,**inspect_feature(app)) - except Failure as exc:failed(name,exc.code) + try: + success(name, **inspect_feature(app)) + except Failure as exc: + failed(name, exc.code) from msg.admin.config_check import configuration_doctor - features=feature_results(feature_manifest(),checks,'doctor_check', - enabled_plugins=tuple(app.registry._plugins)) - return {'ok':all(c['ok'] for c in checks.values()) and - all(row['status']!='fail' for row in features.values()), - 'root_id':ROOT_SUBJECT,'checks':checks,'features':features,'warnings':warnings, - 'configuration_fields':configuration_doctor(checks)} + + features = feature_results( + feature_manifest(), checks, 'doctor_check', enabled_plugins=tuple(app.registry._plugins) + ) + return { + 'ok': all(c['ok'] for c in checks.values()) + and all(row['status'] != 'fail' for row in features.values()), + 'root_id': ROOT_SUBJECT, + 'checks': checks, + 'features': features, + 'warnings': warnings, + 'configuration_fields': configuration_doctor(checks), + } -async def _selftest_ca_chain(app,root,call,register,now): +async def _selftest_ca_chain(app, root, call, register, now): """Publish a real, narrowly scoped CA chain in the disposable test instance.""" from dataclasses import replace + + from msg.admin.root import _approve_csr from msg.core.models import CertificateRequest, IssuancePolicy, Scope, Signature from msg.plugins.common import new_id from msg.security.capabilities import grant_for from msg.security.certificates import csr_body, sign_certificate - from msg.admin.root import _approve_csr - scope=Scope(resource_id=app.namespace_root,descendants=True) - issue=grant_for(app.registry.capability('cert.issue'),scope=scope, - operations=('cert.publish@1',)) - ca_issue=grant_for(app.registry.capability('cert.ca.issue'),scope=scope, - operations=('cert.publish@1',)) - read=grant_for(app.registry.capability('discovery.basic'),scope=scope, - operations=('discovery.get@1',)) - bounded_tool=replace(grant_for(app.registry.capability('tool.use'), - scope=Scope(resource_id='t_tools'),operations=('tool.run@1',)), - constraints={'hosts':['example.org']}) - ca_grants=(issue,ca_issue) - issuable=(issue,ca_issue,read,bounded_tool) - - async def request_certificate(key,uid,issuer,kind,grants,policy=None,ttl=600): - csr=CertificateRequest(resource_id='pending',applicant=uid,subject_id=uid, - requested_issuer=issuer,public_key=key.public_key,kind=kind,grants=tuple(grants), - issuance=policy,requested_ttl_seconds=ttl,target_service=app.settings.service_url, - delegation_depth=0,authority_sources=(),request_digest='', - possession_proof=Signature(key_id=key.key_id,algorithm='ed25519',value=b'')) - proof=key.sign(canonical(csr_body(csr)),purpose='csr') - result=await call('cert.request',{'requested_issuer':issuer,'public_key':b64(key.public_key), - 'kind':kind,'grants':wire(tuple(grants)),'issuance':wire(policy), - 'requested_ttl_seconds':ttl,'possession_proof':wire(proof)},key,uid) - require(result.status=='ok',result.error.code if result.error else 'selftest_csr_failed') + scope = Scope(resource_id=app.namespace_root, descendants=True) + issue = grant_for( + app.registry.capability('cert.issue'), scope=scope, operations=('cert.publish@1',) + ) + ca_issue = grant_for( + app.registry.capability('cert.ca.issue'), scope=scope, operations=('cert.publish@1',) + ) + read = grant_for( + app.registry.capability('discovery.basic'), scope=scope, operations=('discovery.get@1',) + ) + bounded_tool = replace( + grant_for( + app.registry.capability('tool.use'), + scope=Scope(resource_id='t_tools'), + operations=('tool.run@1',), + ), + constraints={'hosts': ['example.org']}, + ) + ca_grants = (issue, ca_issue) + issuable = (issue, ca_issue, read, bounded_tool) + + async def request_certificate(key, uid, issuer, kind, grants, policy=None, ttl=600): + csr = CertificateRequest( + resource_id='pending', + applicant=uid, + subject_id=uid, + requested_issuer=issuer, + public_key=key.public_key, + kind=kind, + grants=tuple(grants), + issuance=policy, + requested_ttl_seconds=ttl, + target_service=app.settings.service_url, + delegation_depth=0, + authority_sources=(), + request_digest='', + possession_proof=Signature(key_id=key.key_id, algorithm='ed25519', value=b''), + ) + proof = key.sign(canonical(csr_body(csr)), purpose='csr') + result = await call( + 'cert.request', + { + 'requested_issuer': issuer, + 'public_key': b64(key.public_key), + 'kind': kind, + 'grants': wire(tuple(grants)), + 'issuance': wire(policy), + 'requested_ttl_seconds': ttl, + 'possession_proof': wire(proof), + }, + key, + uid, + ) + require(result.status == 'ok', result.error.code if result.error else 'selftest_csr_failed') return result - async def publish(parent,parent_key,parent_uid,key,uid,kind,grants,policy=None, - *,ttl=600,target_service=None): - requested=await request_certificate(key,uid,parent_uid,kind,grants,policy,ttl) - cert=Certificate(resource_id=new_id('cert'),serial=new_id('serial'),subject_id=uid, - key_id=key.key_id,issuer_id=parent_uid,parent_certificate_id=parent.resource_id, - authority_sources=(),kind=kind,grants=tuple(grants),not_before=now, - expires_at=now+timedelta(seconds=ttl), + async def publish( + parent, + parent_key, + parent_uid, + key, + uid, + kind, + grants, + policy=None, + *, + ttl=600, + target_service=None, + ): + requested = await request_certificate(key, uid, parent_uid, kind, grants, policy, ttl) + cert = Certificate( + resource_id=new_id('cert'), + serial=new_id('serial'), + subject_id=uid, + key_id=key.key_id, + issuer_id=parent_uid, + parent_certificate_id=parent.resource_id, + authority_sources=(), + kind=kind, + grants=tuple(grants), + not_before=now, + expires_at=now + timedelta(seconds=ttl), target_service=target_service or app.settings.service_url, - delegation_depth=0,issuance=policy, - signature=Signature(key_id=parent_key.key_id,algorithm='ed25519',value=b'')) - cert=sign_certificate(cert,parent_key) - result=await call('cert.publish',{'csr_id':requested.data['csr_id'], - 'certificate':wire(cert)},parent_key,parent_uid,(parent.resource_id,)) - return result,cert,requested - - l1_key,l1_uid=await register('test-l1') - l1_policy=IssuancePolicy(issue_grants=issuable,max_cert_ttl_seconds=600, - max_child_ca_depth=2,max_delegation_depth=0) - l1_request=await request_certificate(l1_key,l1_uid,ROOT_SUBJECT,'ca',ca_grants,l1_policy) - l1=await _approve_csr(app,l1_request.data['csr_id'],root, - expected_digest=l1_request.data['request_digest'],operator='isolated-selftest') - l2_key,l2_uid=await register('test-l2') - l2_policy=IssuancePolicy(issue_grants=issuable,max_cert_ttl_seconds=600, - max_child_ca_depth=1,max_delegation_depth=0) - l2_result,l2,_=await publish(l1,l1_key,l1_uid,l2_key,l2_uid,'ca',ca_grants,l2_policy) - require(l2_result.status=='ok',l2_result.error.code if l2_result.error else 'selftest_l2_failed') - l3_key,l3_uid=await register('test-l3') - l3_policy=IssuancePolicy(issue_grants=issuable,max_cert_ttl_seconds=600, - max_child_ca_depth=0,max_delegation_depth=0) - l3_result,l3,_=await publish(l2,l2_key,l2_uid,l3_key,l3_uid,'ca',ca_grants,l3_policy) - require(l3_result.status=='ok',l3_result.error.code if l3_result.error else 'selftest_l3_failed') - leaf_key,leaf_uid=await register('test-leaf') - leaf_result,leaf,_=await publish(l3,l3_key,l3_uid,leaf_key,leaf_uid,'capability',(read,)) - require(leaf_result.status=='ok',leaf_result.error.code if leaf_result.error else 'selftest_leaf_failed') + delegation_depth=0, + issuance=policy, + signature=Signature(key_id=parent_key.key_id, algorithm='ed25519', value=b''), + ) + cert = sign_certificate(cert, parent_key) + result = await call( + 'cert.publish', + {'csr_id': requested.data['csr_id'], 'certificate': wire(cert)}, + parent_key, + parent_uid, + (parent.resource_id,), + ) + return result, cert, requested + + l1_key, l1_uid = await register('test-l1') + l1_policy = IssuancePolicy( + issue_grants=issuable, + max_cert_ttl_seconds=600, + max_child_ca_depth=2, + max_delegation_depth=0, + ) + l1_request = await request_certificate(l1_key, l1_uid, ROOT_SUBJECT, 'ca', ca_grants, l1_policy) + l1 = await _approve_csr( + app, + l1_request.data['csr_id'], + root, + expected_digest=l1_request.data['request_digest'], + operator='isolated-selftest', + ) + l2_key, l2_uid = await register('test-l2') + l2_policy = IssuancePolicy( + issue_grants=issuable, + max_cert_ttl_seconds=600, + max_child_ca_depth=1, + max_delegation_depth=0, + ) + l2_result, l2, _ = await publish(l1, l1_key, l1_uid, l2_key, l2_uid, 'ca', ca_grants, l2_policy) + require( + l2_result.status == 'ok', l2_result.error.code if l2_result.error else 'selftest_l2_failed' + ) + l3_key, l3_uid = await register('test-l3') + l3_policy = IssuancePolicy( + issue_grants=issuable, + max_cert_ttl_seconds=600, + max_child_ca_depth=0, + max_delegation_depth=0, + ) + l3_result, l3, _ = await publish(l2, l2_key, l2_uid, l3_key, l3_uid, 'ca', ca_grants, l3_policy) + require( + l3_result.status == 'ok', l3_result.error.code if l3_result.error else 'selftest_l3_failed' + ) + leaf_key, leaf_uid = await register('test-leaf') + leaf_result, leaf, _ = await publish( + l3, l3_key, l3_uid, leaf_key, leaf_uid, 'capability', (read,) + ) + require( + leaf_result.status == 'ok', + leaf_result.error.code if leaf_result.error else 'selftest_leaf_failed', + ) async with app.metadata.transaction(write=False) as tx: - for cert in (l1,l2,l3,leaf): - await app.certificates.validate(cert.resource_id,tx) - paths=[await tx.path(cert.resource_id) for cert in (l1,l2,l3,leaf)] - scope_ok=True - for cert in (l1,l2,l3,leaf): - for grant in (*cert.grants,*(cert.issuance.issue_grants if cert.issuance else ())): - if grant.scope.resource_id==app.namespace_root: + for cert in (l1, l2, l3, leaf): + await app.certificates.validate(cert.resource_id, tx) + paths = [await tx.path(cert.resource_id) for cert in (l1, l2, l3, leaf)] + scope_ok = True + for cert in (l1, l2, l3, leaf): + for grant in (*cert.grants, *(cert.issuance.issue_grants if cert.issuance else ())): + if grant.scope.resource_id == app.namespace_root: continue - if app.namespace_root not in {item.id for item in await tx.ancestors(grant.scope.resource_id)}: - scope_ok=False - test_path='/_test/'+app.selftest_run_id - valid=all(path.startswith(test_path+'/') for path in paths) and scope_ok + if app.namespace_root not in { + item.id for item in await tx.ancestors(grant.scope.resource_id) + }: + scope_ok = False + test_path = '/_test/' + app.selftest_run_id + valid = all(path.startswith(test_path + '/') for path in paths) and scope_ok - l4_key,l4_uid=await register('test-l4') - l4_result,l4,l4_request=await publish(l3,l3_key,l3_uid,l4_key,l4_uid,'ca',ca_grants,l3_policy) + l4_key, l4_uid = await register('test-l4') + l4_result, l4, l4_request = await publish( + l3, l3_key, l3_uid, l4_key, l4_uid, 'ca', ca_grants, l3_policy + ) async with app.metadata.transaction(write=False) as tx: - pending=(await tx.csr_state(l4_request.data['csr_id'])).status=='pending' - absent=tx.one('SELECT 1 FROM certificates WHERE id=?',(l4.resource_id,)) is None - denied=(l4_result.error is not None and l4_result.error.code=='ca_depth_exceeded' - and pending and absent) - - out_of_scope=grant_for(app.registry.capability('discovery.basic'), - scope=Scope(resource_id='r_root',descendants=True),operations=('discovery.get@1',)) - bad_scope_key,bad_scope_uid=await register('test-bad-scope') - bad_scope,bad_scope_cert,bad_scope_request=await publish( - l1,l1_key,l1_uid,bad_scope_key,bad_scope_uid,'capability',(out_of_scope,)) + pending = (await tx.csr_state(l4_request.data['csr_id'])).status == 'pending' + absent = tx.one('SELECT 1 FROM certificates WHERE id=?', (l4.resource_id,)) is None + denied = ( + l4_result.error is not None + and l4_result.error.code == 'ca_depth_exceeded' + and pending + and absent + ) + + out_of_scope = grant_for( + app.registry.capability('discovery.basic'), + scope=Scope(resource_id='r_root', descendants=True), + operations=('discovery.get@1',), + ) + bad_scope_key, bad_scope_uid = await register('test-bad-scope') + bad_scope, bad_scope_cert, bad_scope_request = await publish( + l1, l1_key, l1_uid, bad_scope_key, bad_scope_uid, 'capability', (out_of_scope,) + ) async with app.metadata.transaction(write=False) as tx: - scope_pending=(await tx.csr_state(bad_scope_request.data['csr_id'])).status=='pending' - scope_absent=tx.one('SELECT 1 FROM certificates WHERE id=?',(bad_scope_cert.resource_id,)) is None - denied_scope=(bad_scope.error is not None and bad_scope.error.code=='issuance_scope_exceeded' - and scope_pending and scope_absent) - - broad_read=grant_for(app.registry.capability('discovery.basic'),scope=scope, - operations=('discovery.get@1','discovery.list@1')) - bad_ops_key,bad_ops_uid=await register('test-bad-ops') - bad_ops,bad_ops_cert,bad_ops_request=await publish( - l1,l1_key,l1_uid,bad_ops_key,bad_ops_uid,'capability',(broad_read,)) + scope_pending = (await tx.csr_state(bad_scope_request.data['csr_id'])).status == 'pending' + scope_absent = ( + tx.one('SELECT 1 FROM certificates WHERE id=?', (bad_scope_cert.resource_id,)) is None + ) + denied_scope = ( + bad_scope.error is not None + and bad_scope.error.code == 'issuance_scope_exceeded' + and scope_pending + and scope_absent + ) + + broad_read = grant_for( + app.registry.capability('discovery.basic'), + scope=scope, + operations=('discovery.get@1', 'discovery.list@1'), + ) + bad_ops_key, bad_ops_uid = await register('test-bad-ops') + bad_ops, bad_ops_cert, bad_ops_request = await publish( + l1, l1_key, l1_uid, bad_ops_key, bad_ops_uid, 'capability', (broad_read,) + ) async with app.metadata.transaction(write=False) as tx: - ops_pending=(await tx.csr_state(bad_ops_request.data['csr_id'])).status=='pending' - ops_absent=tx.one('SELECT 1 FROM certificates WHERE id=?',(bad_ops_cert.resource_id,)) is None - denied_ops=(bad_ops.error is not None and bad_ops.error.code=='issuance_scope_exceeded' - and ops_pending and ops_absent) - - broad_tool=replace(bounded_tool,constraints={'hosts':['example.org','other.example']}) - bad_constraints_key,bad_constraints_uid=await register('test-bad-constraints') - bad_constraints,bad_constraints_cert,bad_constraints_request=await publish( - l1,l1_key,l1_uid,bad_constraints_key,bad_constraints_uid,'capability',(broad_tool,)) + ops_pending = (await tx.csr_state(bad_ops_request.data['csr_id'])).status == 'pending' + ops_absent = ( + tx.one('SELECT 1 FROM certificates WHERE id=?', (bad_ops_cert.resource_id,)) is None + ) + denied_ops = ( + bad_ops.error is not None + and bad_ops.error.code == 'issuance_scope_exceeded' + and ops_pending + and ops_absent + ) + + broad_tool = replace(bounded_tool, constraints={'hosts': ['example.org', 'other.example']}) + bad_constraints_key, bad_constraints_uid = await register('test-bad-constraints') + bad_constraints, bad_constraints_cert, bad_constraints_request = await publish( + l1, l1_key, l1_uid, bad_constraints_key, bad_constraints_uid, 'capability', (broad_tool,) + ) async with app.metadata.transaction(write=False) as tx: - constraints_pending=(await tx.csr_state(bad_constraints_request.data['csr_id'])).status=='pending' - constraints_absent=tx.one('SELECT 1 FROM certificates WHERE id=?', - (bad_constraints_cert.resource_id,)) is None - denied_constraints=(bad_constraints.error is not None and - bad_constraints.error.code=='issuance_scope_exceeded' and - constraints_pending and constraints_absent) - - bad_ttl_key,bad_ttl_uid=await register('test-bad-ttl') - bad_ttl,bad_ttl_cert,bad_ttl_request=await publish( - l1,l1_key,l1_uid,bad_ttl_key,bad_ttl_uid,'capability',(read,),ttl=601) + constraints_pending = ( + await tx.csr_state(bad_constraints_request.data['csr_id']) + ).status == 'pending' + constraints_absent = ( + tx.one('SELECT 1 FROM certificates WHERE id=?', (bad_constraints_cert.resource_id,)) + is None + ) + denied_constraints = ( + bad_constraints.error is not None + and bad_constraints.error.code == 'issuance_scope_exceeded' + and constraints_pending + and constraints_absent + ) + + bad_ttl_key, bad_ttl_uid = await register('test-bad-ttl') + bad_ttl, bad_ttl_cert, bad_ttl_request = await publish( + l1, l1_key, l1_uid, bad_ttl_key, bad_ttl_uid, 'capability', (read,), ttl=601 + ) async with app.metadata.transaction(write=False) as tx: - ttl_pending=(await tx.csr_state(bad_ttl_request.data['csr_id'])).status=='pending' - ttl_absent=tx.one('SELECT 1 FROM certificates WHERE id=?',(bad_ttl_cert.resource_id,)) is None - denied_ttl=(bad_ttl.error is not None and bad_ttl.error.code in - {'certificate_validity_escalation','certificate_ttl_escalation'} and - ttl_pending and ttl_absent) - - bad_service_key,bad_service_uid=await register('test-bad-service') - bad_service,bad_service_cert,bad_service_request=await publish( - l1,l1_key,l1_uid,bad_service_key,bad_service_uid,'capability',(read,), - target_service='https://wrong-service.invalid') + ttl_pending = (await tx.csr_state(bad_ttl_request.data['csr_id'])).status == 'pending' + ttl_absent = ( + tx.one('SELECT 1 FROM certificates WHERE id=?', (bad_ttl_cert.resource_id,)) is None + ) + denied_ttl = ( + bad_ttl.error is not None + and bad_ttl.error.code in {'certificate_validity_escalation', 'certificate_ttl_escalation'} + and ttl_pending + and ttl_absent + ) + + bad_service_key, bad_service_uid = await register('test-bad-service') + bad_service, bad_service_cert, bad_service_request = await publish( + l1, + l1_key, + l1_uid, + bad_service_key, + bad_service_uid, + 'capability', + (read,), + target_service='https://wrong-service.invalid', + ) async with app.metadata.transaction(write=False) as tx: - service_pending=(await tx.csr_state(bad_service_request.data['csr_id'])).status=='pending' - service_absent=tx.one('SELECT 1 FROM certificates WHERE id=?',(bad_service_cert.resource_id,)) is None - denied_service=(bad_service.error is not None and bad_service.error.code=='csr_certificate_mismatch' - and service_pending and service_absent) - - bad_policy=IssuancePolicy(issue_grants=(*issuable,broad_read),max_cert_ttl_seconds=600, - max_child_ca_depth=1,max_delegation_depth=0) - bad_policy_key,bad_policy_uid=await register('test-bad-issue-grants') - bad_issue_grants,bad_policy_cert,bad_policy_request=await publish( - l1,l1_key,l1_uid,bad_policy_key,bad_policy_uid,'ca',ca_grants,bad_policy) + service_pending = ( + await tx.csr_state(bad_service_request.data['csr_id']) + ).status == 'pending' + service_absent = ( + tx.one('SELECT 1 FROM certificates WHERE id=?', (bad_service_cert.resource_id,)) is None + ) + denied_service = ( + bad_service.error is not None + and bad_service.error.code == 'csr_certificate_mismatch' + and service_pending + and service_absent + ) + + bad_policy = IssuancePolicy( + issue_grants=(*issuable, broad_read), + max_cert_ttl_seconds=600, + max_child_ca_depth=1, + max_delegation_depth=0, + ) + bad_policy_key, bad_policy_uid = await register('test-bad-issue-grants') + bad_issue_grants, bad_policy_cert, bad_policy_request = await publish( + l1, l1_key, l1_uid, bad_policy_key, bad_policy_uid, 'ca', ca_grants, bad_policy + ) async with app.metadata.transaction(write=False) as tx: - policy_pending=(await tx.csr_state(bad_policy_request.data['csr_id'])).status=='pending' - policy_absent=tx.one('SELECT 1 FROM certificates WHERE id=?',(bad_policy_cert.resource_id,)) is None - denied_policy=(bad_issue_grants.error is not None and - bad_issue_grants.error.code=='issuance_scope_exceeded' and - policy_pending and policy_absent) - - child_key,child_uid=await register('test-leaf-child') - leaf_issue,leaf_child,leaf_request=await publish( - leaf,leaf_key,leaf_uid,child_key,child_uid,'capability',(read,)) + policy_pending = (await tx.csr_state(bad_policy_request.data['csr_id'])).status == 'pending' + policy_absent = ( + tx.one('SELECT 1 FROM certificates WHERE id=?', (bad_policy_cert.resource_id,)) is None + ) + denied_policy = ( + bad_issue_grants.error is not None + and bad_issue_grants.error.code == 'issuance_scope_exceeded' + and policy_pending + and policy_absent + ) + + child_key, child_uid = await register('test-leaf-child') + leaf_issue, leaf_child, leaf_request = await publish( + leaf, leaf_key, leaf_uid, child_key, child_uid, 'capability', (read,) + ) async with app.metadata.transaction(write=False) as tx: - leaf_pending=(await tx.csr_state(leaf_request.data['csr_id'])).status=='pending' - leaf_absent=tx.one('SELECT 1 FROM certificates WHERE id=?',(leaf_child.resource_id,)) is None - leaf_denied=(leaf_issue.error is not None and leaf_pending and leaf_absent) + leaf_pending = (await tx.csr_state(leaf_request.data['csr_id'])).status == 'pending' + leaf_absent = ( + tx.one('SELECT 1 FROM certificates WHERE id=?', (leaf_child.resource_id,)) is None + ) + leaf_denied = leaf_issue.error is not None and leaf_pending and leaf_absent - revoked=True - for ancestor in (l1,l2,l3): + revoked = True + for ancestor in (l1, l2, l3): async with app.metadata.transaction(write=True) as tx: - tx.execute('UPDATE certificates SET revoked=1 WHERE id=?',(ancestor.resource_id,),write=True) + tx.execute( + 'UPDATE certificates SET revoked=1 WHERE id=?', (ancestor.resource_id,), write=True + ) async with app.metadata.transaction(write=False) as tx: try: - await app.certificates.validate(leaf.resource_id,tx) + await app.certificates.validate(leaf.resource_id, tx) except Failure as exc: - revoked=revoked and exc.code=='certificate_revoked' + revoked = revoked and exc.code == 'certificate_revoked' else: - revoked=False + revoked = False async with app.metadata.transaction(write=True) as tx: - tx.execute('UPDATE certificates SET revoked=0 WHERE id=?',(ancestor.resource_id,),write=True) - return {'ca_l1_l2_l3_leaf':valid,'ca_l3_cannot_issue_ca':denied, - 'ca_ancestor_revocation':revoked,'ca_scope_expansion_denied':denied_scope, - 'ca_operations_expansion_denied':denied_ops, - 'ca_constraints_expansion_denied':denied_constraints, - 'ca_ttl_expansion_denied':denied_ttl, - 'ca_target_service_change_denied':denied_service, - 'ca_issue_grants_expansion_denied':denied_policy, - 'ca_leaf_cannot_issue':leaf_denied} + tx.execute( + 'UPDATE certificates SET revoked=0 WHERE id=?', (ancestor.resource_id,), write=True + ) + return { + 'ca_l1_l2_l3_leaf': valid, + 'ca_l3_cannot_issue_ca': denied, + 'ca_ancestor_revocation': revoked, + 'ca_scope_expansion_denied': denied_scope, + 'ca_operations_expansion_denied': denied_ops, + 'ca_constraints_expansion_denied': denied_constraints, + 'ca_ttl_expansion_denied': denied_ttl, + 'ca_target_service_change_denied': denied_service, + 'ca_issue_grants_expansion_denied': denied_policy, + 'ca_leaf_cannot_issue': leaf_denied, + } async def selftest(): """No production database, listener, root PIN or live trust state is touched.""" - from dataclasses import replace + from msg.admin import recovery_replay + from msg.admin.config_check import configuration_selftest + from msg.admin.root import _approve_csr, _provision from msg.application import Application + from msg.bootstrap import feature_manifest from msg.config import write_example - from msg.admin.root import _provision,_approve_csr - from msg.security.crypto import Ed25519Signer,subject_id - from msg.security.age_keys import generate_age_key + from msg.core.models import CertificateRequest, Scope, Signature from msg.core.requests import request_for - from msg.core.models import CertificateRequest,Signature,Scope - from msg.security.certificates import csr_body + from msg.security.age_keys import generate_age_key from msg.security.capabilities import grant_for - from msg.bootstrap import feature_manifest - from msg.admin import recovery_replay - from msg.admin.config_check import configuration_selftest - configuration_fields=configuration_selftest() - tool_sandbox_details={'code':'not_run'} - checks={'configuration_loading':all(row['status']=='pass' - for row in configuration_fields.values())} + from msg.security.certificates import csr_body + from msg.security.crypto import Ed25519Signer, subject_id + + configuration_fields = configuration_selftest() + tool_sandbox_details = {'code': 'not_run'} + checks = { + 'configuration_loading': all( + row['status'] == 'pass' for row in configuration_fields.values() + ) + } try: inspect_recovery_checkpoint() - checks['recovery_checkpoint_replay']=recovery_replay.selftest()['status']=='passed' - except (Failure,OSError,ValueError,KeyError,TypeError,AttributeError): - checks['recovery_checkpoint_replay']=False - with tempfile.TemporaryDirectory(prefix='msg-selftest-') as temporary, temporary_postgres() as dsn: - folder=Path(temporary) - now=datetime.now(UTC) - run_id=uuid.uuid4().hex - test_path='/_test/'+run_id - app=Application(write_example(folder/'etc',folder/'data','https://selftest.invalid', - postgres_dsn=dsn),clock=lambda:now,selftest_run_id=run_id) + checks['recovery_checkpoint_replay'] = recovery_replay.selftest()['status'] == 'passed' + except Failure, OSError, ValueError, KeyError, TypeError, AttributeError: + checks['recovery_checkpoint_replay'] = False + with ( + tempfile.TemporaryDirectory(prefix='msg-selftest-') as temporary, + temporary_postgres() as dsn, + ): + folder = Path(temporary) + now = datetime.now(UTC) + run_id = uuid.uuid4().hex + test_path = '/_test/' + run_id + app = Application( + write_example( + folder / 'etc', folder / 'data', 'https://selftest.invalid', postgres_dsn=dsn + ), + clock=lambda: now, + selftest_run_id=run_id, + ) try: - csr,root=await _provision(app,'selftest-'+os.urandom(24).hex()) - await _approve_csr(app,csr,root,expected_digest=None,operator='isolated-selftest') + csr, root = await _provision(app, 'selftest-' + os.urandom(24).hex()) + await _approve_csr(app, csr, root, expected_digest=None, operator='isolated-selftest') async with app.metadata.transaction(write=False) as tx: - namespace=app.namespace_root - require(await tx.path(namespace)==test_path,'selftest_namespace_missing') - root_scoped=True + namespace = app.namespace_root + require(await tx.path(namespace) == test_path, 'selftest_namespace_missing') + root_scoped = True for (resource_id,) in tx.execute('SELECT id FROM resources'): - if resource_id in {'r_root','t_selftest'}: + if resource_id in {'r_root', 't_selftest'}: continue - ancestors=await tx.ancestors(resource_id) - if resource_id!=namespace and namespace not in {item.id for item in ancestors}: - root_scoped=False + ancestors = await tx.ancestors(resource_id) + if resource_id != namespace and namespace not in { + item.id for item in ancestors + }: + root_scoped = False break - checks['isolated_namespace']=root_scoped - async def call(name,args,key=None,subject=None,certs=(),expected=(),request_id=None, - contract_version=1): - packet=request_for(name,args,app.settings.service_url,subject=subject,signer=key, - certificates=certs,expected=expected,expires_at=now+timedelta(seconds=120), - request_id=request_id,contract_version=contract_version) + checks['isolated_namespace'] = root_scoped + + async def call( + name, + args, + key=None, + subject=None, + certs=(), + expected=(), + request_id=None, + contract_version=1, + ): + packet = request_for( + name, + args, + app.settings.service_url, + subject=subject, + signer=key, + certificates=certs, + expected=expected, + expires_at=now + timedelta(seconds=120), + request_id=request_id, + contract_version=contract_version, + ) return await app.executor.execute(packet) + async def register(name): - key=Ed25519Signer.generate();uid=subject_id(key.public_key) - _,recipient=generate_age_key() - result=await call('identity.register',{'handle':name,'public_key':b64(key.public_key), - 'encryption_recipient':recipient},key,uid,contract_version=2) - require(result.status=='ok','selftest_registration_failed') - return key,uid - async def approve(key,uid,grants): - csr=CertificateRequest(resource_id='pending',applicant=uid,subject_id=uid,requested_issuer=ROOT_SUBJECT, - public_key=key.public_key,kind='capability',grants=tuple(grants),issuance=None, - requested_ttl_seconds=600,target_service=app.settings.service_url,delegation_depth=0,authority_sources=(), - request_digest='',possession_proof=Signature(key_id=key.key_id,algorithm='ed25519',value=b'')) - proof=key.sign(canonical(csr_body(csr)),purpose='csr') - result=await call('cert.request',{'requested_issuer':ROOT_SUBJECT,'public_key':b64(key.public_key), - 'kind':'capability','grants':wire(grants),'requested_ttl_seconds':600,'possession_proof':wire(proof)},key,uid) - require(result.status=='ok','selftest_csr_failed') - return await _approve_csr(app,result.data['csr_id'],root,expected_digest=result.data['request_digest'],operator='isolated-selftest') - alice,ua=await register('alice');bob,ub=await register('bob') + key = Ed25519Signer.generate() + uid = subject_id(key.public_key) + _, recipient = generate_age_key() + result = await call( + 'identity.register', + { + 'handle': name, + 'public_key': b64(key.public_key), + 'encryption_recipient': recipient, + }, + key, + uid, + contract_version=2, + ) + require(result.status == 'ok', 'selftest_registration_failed') + return key, uid + + async def approve(key, uid, grants): + csr = CertificateRequest( + resource_id='pending', + applicant=uid, + subject_id=uid, + requested_issuer=ROOT_SUBJECT, + public_key=key.public_key, + kind='capability', + grants=tuple(grants), + issuance=None, + requested_ttl_seconds=600, + target_service=app.settings.service_url, + delegation_depth=0, + authority_sources=(), + request_digest='', + possession_proof=Signature(key_id=key.key_id, algorithm='ed25519', value=b''), + ) + proof = key.sign(canonical(csr_body(csr)), purpose='csr') + result = await call( + 'cert.request', + { + 'requested_issuer': ROOT_SUBJECT, + 'public_key': b64(key.public_key), + 'kind': 'capability', + 'grants': wire(grants), + 'requested_ttl_seconds': 600, + 'possession_proof': wire(proof), + }, + key, + uid, + ) + require(result.status == 'ok', 'selftest_csr_failed') + return await _approve_csr( + app, + result.data['csr_id'], + root, + expected_digest=result.data['request_digest'], + operator='isolated-selftest', + ) + + alice, ua = await register('alice') + bob, ub = await register('bob') from msg.admin.tool_check import check_tool_sandbox + try: - tool_sandbox_details=await check_tool_sandbox(app,call,approve,alice,ua) - checks['tool_sandbox']=True - except (Failure,OSError) as exc: - tool_sandbox_details={'code':getattr(exc,'code','tool_isolation_failed')} - checks['tool_sandbox']=False - post=await call('content.post_create',{'parent':test_path+'/tmp','body':'retained source bytes\r\n'},alice,ua,request_id='same-write') - require(post.status=='ok',post.error.code if post.error else 'selftest_post_failed') + tool_sandbox_details = await check_tool_sandbox(app, call, approve, alice, ua) + checks['tool_sandbox'] = True + except (Failure, OSError) as exc: + tool_sandbox_details = {'code': getattr(exc, 'code', 'tool_isolation_failed')} + checks['tool_sandbox'] = False + post = await call( + 'content.post_create', + {'parent': test_path + '/tmp', 'body': 'retained source bytes\r\n'}, + alice, + ua, + request_id='same-write', + ) + require(post.status == 'ok', post.error.code if post.error else 'selftest_post_failed') # Following reads only existing watches; use a separate fixture so # its explicit permission change cannot alter other lifecycle tests. - empty_following=await call('communication.following',{},bob,ub) - anonymous_following=await call('communication.following',{}) - followed=await call('content.post_create', - {'parent':test_path+'/tmp','body':'isolated Following fixture'},alice,ua) - require(followed.status=='ok','selftest_following_post_failed') - followed_id=followed.resources[0].id - watched=await call('communication.watch',{'id':followed_id},bob,ub) - shown=await call('communication.following',{},bob,ub) - hidden=await call('content.chmod',{'id':followed_id,'mode':'0600'},alice,ua, - expected=((followed_id,followed.data['generation']),)) - redacted=await call('communication.following',{},bob,ub) - checks['following']=(empty_following.status=='ok' and not empty_following.data['items'] and - anonymous_following.error is not None and - anonymous_following.error.code=='authentication_required' and - watched.status=='ok' and shown.status=='ok' and - [item['id'] for item in shown.data['items']]==[followed_id] and - hidden.status=='ok' and redacted.status=='ok' and not redacted.data['items']) - work = await call('communication.request_create', - {'title':'Isolated task','description':'Validate work records','requirements':'Explicit claim'},alice,ua) - offer = await call('communication.offer_create', - {'description':'Review','scope':'Selftest','availability':'Now'},alice,ua) - checkpoint = await call('communication.checkpoint_create', - {'summary':'Selftest progress','resource_refs':[]},alice,ua) - proposal_target = await call('content.post_create', - {'parent':test_path+'/tmp','body':'before proposal'},alice,ua) - proposal_source = await call('content.post_create', - {'parent':test_path+'/tmp','body':'after proposal'},alice,ua) - require(all(item.status=='ok' for item in - (work,offer,checkpoint,proposal_target,proposal_source)), 'selftest_collaboration_create_failed') - proposal = await call('communication.proposal_create', - {'target':proposal_target.resources[0].id, - 'base_revision':proposal_target.resources[0].revision, - 'content_ref':wire(proposal_source.resources[0]),'message':'Apply selftest text'},alice,ua) - require(proposal.status=='ok','selftest_proposal_create_failed') - accepted = await call('communication.proposal_accept', - {'id':proposal.data['id'],'proposal_revision':proposal.resources[0].revision, - 'base_revision':proposal_target.resources[0].revision},alice,ua, - expected=((proposal.data['id'],proposal.data['generation']), - (proposal_target.resources[0].id,proposal_target.data['generation']))) - restored = await call('discovery.get',{'id':proposal_target.resources[0].id},alice,ua) - checks['collaboration']=(accepted.status=='ok' and restored.status=='ok' and - restored.data['content']=='after proposal') + empty_following = await call('communication.following', {}, bob, ub) + anonymous_following = await call('communication.following', {}) + followed = await call( + 'content.post_create', + {'parent': test_path + '/tmp', 'body': 'isolated Following fixture'}, + alice, + ua, + ) + require(followed.status == 'ok', 'selftest_following_post_failed') + followed_id = followed.resources[0].id + watched = await call('communication.watch', {'id': followed_id}, bob, ub) + shown = await call('communication.following', {}, bob, ub) + hidden = await call( + 'content.chmod', + {'id': followed_id, 'mode': '0600'}, + alice, + ua, + expected=((followed_id, followed.data['generation']),), + ) + redacted = await call('communication.following', {}, bob, ub) + checks['following'] = ( + empty_following.status == 'ok' + and not empty_following.data['items'] + and anonymous_following.error is not None + and anonymous_following.error.code == 'authentication_required' + and watched.status == 'ok' + and shown.status == 'ok' + and [item['id'] for item in shown.data['items']] == [followed_id] + and hidden.status == 'ok' + and redacted.status == 'ok' + and not redacted.data['items'] + ) + work = await call( + 'communication.request_create', + { + 'title': 'Isolated task', + 'description': 'Validate work records', + 'requirements': 'Explicit claim', + }, + alice, + ua, + ) + offer = await call( + 'communication.offer_create', + {'description': 'Review', 'scope': 'Selftest', 'availability': 'Now'}, + alice, + ua, + ) + checkpoint = await call( + 'communication.checkpoint_create', + {'summary': 'Selftest progress', 'resource_refs': []}, + alice, + ua, + ) + proposal_target = await call( + 'content.post_create', + {'parent': test_path + '/tmp', 'body': 'before proposal'}, + alice, + ua, + ) + proposal_source = await call( + 'content.post_create', + {'parent': test_path + '/tmp', 'body': 'after proposal'}, + alice, + ua, + ) + require( + all( + item.status == 'ok' + for item in (work, offer, checkpoint, proposal_target, proposal_source) + ), + 'selftest_collaboration_create_failed', + ) + proposal = await call( + 'communication.proposal_create', + { + 'target': proposal_target.resources[0].id, + 'base_revision': proposal_target.resources[0].revision, + 'content_ref': wire(proposal_source.resources[0]), + 'message': 'Apply selftest text', + }, + alice, + ua, + ) + require(proposal.status == 'ok', 'selftest_proposal_create_failed') + accepted = await call( + 'communication.proposal_accept', + { + 'id': proposal.data['id'], + 'proposal_revision': proposal.resources[0].revision, + 'base_revision': proposal_target.resources[0].revision, + }, + alice, + ua, + expected=( + (proposal.data['id'], proposal.data['generation']), + (proposal_target.resources[0].id, proposal_target.data['generation']), + ), + ) + restored = await call( + 'discovery.get', {'id': proposal_target.resources[0].id}, alice, ua + ) + checks['collaboration'] = ( + accepted.status == 'ok' + and restored.status == 'ok' + and restored.data['content'] == 'after proposal' + ) # OnlineIssuer exercises only the independent temporary Test Root # created above. No production trust material or signer is opened. async with app.metadata.transaction(write=False) as tx: - identity_cert=next(decode(Certificate,loads(raw)) for (raw,) in - tx.execute('SELECT body FROM certificates WHERE subject=?',(ua,)) - if decode(Certificate,loads(raw)).kind=='identity') - issued_audits=[loads(raw) for (raw,) in tx.execute('SELECT body FROM audit ORDER BY seq') - if loads(raw)['event']['type']=='cert.auto.issue'] - registration_audit=next((entry for entry in issued_audits if - entry['event']['data']['certificate_id']==identity_cert.resource_id),None) - checks['online_registration']=bool(registration_audit and - registration_audit['event']['data']['automatic'] is True and - registration_audit['event']['data']['authority_source']['key_id']==alice.key_id) - ordinary=grant_for(app.registry.capability('discovery.basic'), - scope=Scope(resource_id=post.resources[0].id),operations=('discovery.get@1',)) - delegated=await call('identity.delegate',{'grantee':ub,'key_id':bob.key_id, - 'grants':wire((ordinary,)),'ttl':600},alice,ua,request_id='online-selftest-delegation') - require(delegated.status=='ok','selftest_online_delegation_failed') + identity_cert = next( + decode(Certificate, loads(raw)) + for (raw,) in tx.execute('SELECT body FROM certificates WHERE subject=?', (ua,)) + if decode(Certificate, loads(raw)).kind == 'identity' + ) + issued_audits = [ + loads(raw) + for (raw,) in tx.execute('SELECT body FROM audit ORDER BY seq') + if loads(raw)['event']['type'] == 'cert.auto.issue' + ] + registration_audit = next( + ( + entry + for entry in issued_audits + if entry['event']['data']['certificate_id'] == identity_cert.resource_id + ), + None, + ) + checks['online_registration'] = bool( + registration_audit + and registration_audit['event']['data']['automatic'] is True + and registration_audit['event']['data']['authority_source']['key_id'] + == alice.key_id + ) + ordinary = grant_for( + app.registry.capability('discovery.basic'), + scope=Scope(resource_id=post.resources[0].id), + operations=('discovery.get@1',), + ) + delegated = await call( + 'identity.delegate', + {'grantee': ub, 'key_id': bob.key_id, 'grants': wire((ordinary,)), 'ttl': 600}, + alice, + ua, + request_id='online-selftest-delegation', + ) + require(delegated.status == 'ok', 'selftest_online_delegation_failed') async with app.metadata.transaction(write=False) as tx: - audit_rows=[loads(raw) for (raw,) in tx.execute('SELECT body FROM audit ORDER BY seq')] - delegation_audit=next((entry for entry in audit_rows if - entry['event']['type']=='cert.auto.issue' and delegated.status=='ok' and - entry['event']['data']['certificate_id']==delegated.data['certificate_id']),None) - checks['online_delegation']=bool(delegation_audit and - delegation_audit['event']['data']['authority_source']['id']==delegated.resources[0].id and - delegation_audit['event']['data']['grant_digest']==digest((ordinary,))) - renewed=await call('identity.certificate_renew',{},alice,ua) - overlong=await call('identity.certificate_renew',{'ttl':31536000},alice,ua) - checks['online_renewal']=(renewed.status=='ok' and overlong.error is not None and - overlong.error.code=='renewal_ttl_exceeded') - revoked_source=await call('identity.delegation_revoke',{'id':delegated.resources[0].id},alice,ua) + audit_rows = [ + loads(raw) for (raw,) in tx.execute('SELECT body FROM audit ORDER BY seq') + ] + delegation_audit = next( + ( + entry + for entry in audit_rows + if entry['event']['type'] == 'cert.auto.issue' + and delegated.status == 'ok' + and entry['event']['data']['certificate_id'] == delegated.data['certificate_id'] + ), + None, + ) + checks['online_delegation'] = bool( + delegation_audit + and delegation_audit['event']['data']['authority_source']['id'] + == delegated.resources[0].id + and delegation_audit['event']['data']['grant_digest'] == digest((ordinary,)) + ) + renewed = await call('identity.certificate_renew', {}, alice, ua) + overlong = await call('identity.certificate_renew', {'ttl': 31536000}, alice, ua) + checks['online_renewal'] = ( + renewed.status == 'ok' + and overlong.error is not None + and overlong.error.code == 'renewal_ttl_exceeded' + ) + revoked_source = await call( + 'identity.delegation_revoke', {'id': delegated.resources[0].id}, alice, ua + ) async with app.metadata.transaction(write=False) as tx: - derived=await app.certificates.validate(identity_cert.resource_id,tx) - delegation_invalid=False - try:await app.certificates.validate(delegated.data['certificate_id'],tx) - except Failure as exc:delegation_invalid=exc.code=='authority_source_inactive' - checks['online_source_revocation']=revoked_source.status=='ok' and derived.kind=='identity' and delegation_invalid - repeat=await call('content.post_create',{'parent':test_path+'/tmp','body':'retained source bytes\r\n'},alice,ua,request_id='same-write') - checks['idempotency']=repeat.replayed and repeat.resources==post.resources - rejected=await call('content.archive',{'id':post.resources[0].id},bob,ub, - expected=((post.resources[0].id,post.data['generation']),)) - checks['sticky']=rejected.error is not None and rejected.error.code=='sticky_denied' - denied=await call('discovery.get',{'id':test_path+'/private'},bob,ub) - checks['deny']=denied.error is not None and denied.error.code=='permission_denied' - denied=await call('content.post_create',{'parent':test_path+'/certified','body':'gate'},bob,ub) - grant=grant_for(app.registry.capability('resource.certified_write'),scope=Scope(resource_id='t_certified',descendants=True), - operations=('content.post_create@1',)) - cert=await approve(bob,ub,(grant,)) - accepted=await call('content.post_create',{'parent':test_path+'/certified','body':'gate'},bob,ub,(cert.resource_id,)) - checks['certgate']=denied.error is not None and denied.error.code=='certificate_gate' and accepted.status=='ok' - wrong=grant_for(app.registry.capability('resource.certified_write'),scope=Scope(resource_id='t_main',descendants=True), - operations=('content.post_create@1',)) - wrong_cert=await approve(alice,ua,(wrong,)) - rejected=await call('content.post_create',{'parent':test_path+'/certified','body':'wrong scope'},alice,ua,(wrong_cert.resource_id,)) - checks['certificate_scope']=rejected.error is not None and rejected.error.code=='certificate_gate' - manage=grant_for(app.registry.capability('group.manage_override'),scope=Scope(resource_id='g_admins'), - operations=('group.member.add@1','group.member.remove@1')) - manager=await approve(alice,ua,(manage,)) - await call('group.member.add',{'group':'g_admins','subject':ub},alice,ua,(manager.resource_id,)) - child=await call('content.topic_create',{'parent':test_path+'/admins','name':'selftest'},bob,ub) - require(child.status=='ok','selftest_group_failed') + derived = await app.certificates.validate(identity_cert.resource_id, tx) + delegation_invalid = False + try: + await app.certificates.validate(delegated.data['certificate_id'], tx) + except Failure as exc: + delegation_invalid = exc.code == 'authority_source_inactive' + checks['online_source_revocation'] = ( + revoked_source.status == 'ok' and derived.kind == 'identity' and delegation_invalid + ) + repeat = await call( + 'content.post_create', + {'parent': test_path + '/tmp', 'body': 'retained source bytes\r\n'}, + alice, + ua, + request_id='same-write', + ) + checks['idempotency'] = repeat.replayed and repeat.resources == post.resources + rejected = await call( + 'content.archive', + {'id': post.resources[0].id}, + bob, + ub, + expected=((post.resources[0].id, post.data['generation']),), + ) + checks['sticky'] = rejected.error is not None and rejected.error.code == 'sticky_denied' + denied = await call('discovery.get', {'id': test_path + '/private'}, bob, ub) + checks['deny'] = denied.error is not None and denied.error.code == 'permission_denied' + denied = await call( + 'content.post_create', {'parent': test_path + '/certified', 'body': 'gate'}, bob, ub + ) + grant = grant_for( + app.registry.capability('resource.certified_write'), + scope=Scope(resource_id='t_certified', descendants=True), + operations=('content.post_create@1',), + ) + cert = await approve(bob, ub, (grant,)) + accepted = await call( + 'content.post_create', + {'parent': test_path + '/certified', 'body': 'gate'}, + bob, + ub, + (cert.resource_id,), + ) + checks['certgate'] = ( + denied.error is not None + and denied.error.code == 'certificate_gate' + and accepted.status == 'ok' + ) + wrong = grant_for( + app.registry.capability('resource.certified_write'), + scope=Scope(resource_id='t_main', descendants=True), + operations=('content.post_create@1',), + ) + wrong_cert = await approve(alice, ua, (wrong,)) + rejected = await call( + 'content.post_create', + {'parent': test_path + '/certified', 'body': 'wrong scope'}, + alice, + ua, + (wrong_cert.resource_id,), + ) + checks['certificate_scope'] = ( + rejected.error is not None and rejected.error.code == 'certificate_gate' + ) + manage = grant_for( + app.registry.capability('group.manage_override'), + scope=Scope(resource_id='g_admins'), + operations=('group.member.add@1', 'group.member.remove@1'), + ) + manager = await approve(alice, ua, (manage,)) + await call( + 'group.member.add', + {'group': 'g_admins', 'subject': ub}, + alice, + ua, + (manager.resource_id,), + ) + child = await call( + 'content.topic_create', + {'parent': test_path + '/admins', 'name': 'selftest'}, + bob, + ub, + ) + require(child.status == 'ok', 'selftest_group_failed') async with app.metadata.transaction(write=False) as tx: - resource=await tx.resource(child.resources[0].id) - checks['setgid']=resource.group=='g_admins' and bool(resource.mode&0o2000) - await call('group.member.remove',{'group':'g_admins','subject':ub},alice,ua,(manager.resource_id,)) - rejected=await call('discovery.get',{'id':child.resources[0].id},bob,ub) - checks['membership_revocation']=rejected.error is not None + resource = await tx.resource(child.resources[0].id) + checks['setgid'] = resource.group == 'g_admins' and bool(resource.mode & 0o2000) + await call( + 'group.member.remove', + {'group': 'g_admins', 'subject': ub}, + alice, + ua, + (manager.resource_id,), + ) + rejected = await call('discovery.get', {'id': child.resources[0].id}, bob, ub) + checks['membership_revocation'] = rejected.error is not None async with app.metadata.transaction(write=True) as tx: - tx.execute('UPDATE certificates SET revoked=1 WHERE id=?',(cert.resource_id,),write=True) - rejected=await call('content.post_create',{'parent':test_path+'/certified','body':'revoked'},bob,ub,(cert.resource_id,)) - checks['certificate_revocation']=rejected.error is not None and rejected.error.code=='certificate_revoked' - rejected=await call('discovery.get',{'id':test_path+'/private'},root,ROOT_SUBJECT) - checks['root_network_rejected']=rejected.error is not None and rejected.error.code=='local_only' - all_versions=await call('discovery.get',{'id':post.resources[0].id,'view':'meta'}) - checks['stable_id_read']=all_versions.status=='ok' + tx.execute( + 'UPDATE certificates SET revoked=1 WHERE id=?', (cert.resource_id,), write=True + ) + rejected = await call( + 'content.post_create', + {'parent': test_path + '/certified', 'body': 'revoked'}, + bob, + ub, + (cert.resource_id,), + ) + checks['certificate_revocation'] = ( + rejected.error is not None and rejected.error.code == 'certificate_revoked' + ) + rejected = await call( + 'discovery.get', {'id': test_path + '/private'}, root, ROOT_SUBJECT + ) + checks['root_network_rejected'] = ( + rejected.error is not None and rejected.error.code == 'local_only' + ) + all_versions = await call('discovery.get', {'id': post.resources[0].id, 'view': 'meta'}) + checks['stable_id_read'] = all_versions.status == 'ok' from msg.admin.honor_check import check_honors - checks['honor_ceremony_display']=await check_honors(app,call,register) - checks.update(await _selftest_ca_chain(app,root,call,register,now)) + + checks['honor_ceremony_display'] = await check_honors(app, call, register) + checks.update(await _selftest_ca_chain(app, root, call, register, now)) from msg.admin.upgrade_check import check_upgrade_recovery - checks['identity_upgrade_recovery']=await check_upgrade_recovery(app,now) + + checks['identity_upgrade_recovery'] = await check_upgrade_recovery(app, now) from msg.admin.token_delivery_check import check_token_delivery - checks['credential_delivery_recovery']=await check_token_delivery(app,now) + + checks['credential_delivery_recovery'] = await check_token_delivery(app, now) from msg.admin.custodial_check import check_custodial_history - checks['custodial_history_recovery']=await check_custodial_history(app,now) + + checks['custodial_history_recovery'] = await check_custodial_history(app, now) from msg.admin.market_check import check_market, check_market_e2e - checks['market_e2e']=await check_market_e2e(app,root,call,register) - checks['market_lifecycle']=await check_market(app,root,call,register,now) + + checks['market_e2e'] = await check_market_e2e(app, root, call, register) + checks['market_lifecycle'] = await check_market(app, root, call, register, now) from msg.admin.git_check import check_git - checks['git_push_read_cas']=await check_git(app,call,register) - checks.update(await _selftest_search_hosting(app,call,register,test_path)) + + checks['git_push_read_cas'] = await check_git(app, call, register) + checks.update(await _selftest_search_hosting(app, call, register, test_path)) from msg.admin.dm_check import check_private_dm - checks['private_dm']=await check_private_dm(app,now) + + checks['private_dm'] = await check_private_dm(app, now) from msg.admin.content_edit_check import check_content_editing - checks['content_editing']=await check_content_editing(app,call,register,test_path) + + checks['content_editing'] = await check_content_editing(app, call, register, test_path) from msg.admin.transfer_check import check_transfer - checks['transfer_state_machine']=await check_transfer(app,call,register) + + checks['transfer_state_machine'] = await check_transfer(app, call, register) except Failure as exc: - checks['failure']={'code':exc.code} + checks['failure'] = {'code': exc.code} finally: await app.close() - features=feature_results(feature_manifest(),checks,'selftest_case') - return {'ok':bool(checks) and all(value is True for value in checks.values()) and - all(row['status']!='fail' for row in features.values()), - 'checks':checks,'features':features,'cleaned_up':not folder.exists(), - 'configuration_fields':configuration_fields,'tool_sandbox':tool_sandbox_details} + features = feature_results(feature_manifest(), checks, 'selftest_case') + return { + 'ok': bool(checks) + and all(value is True for value in checks.values()) + and all(row['status'] != 'fail' for row in features.values()), + 'checks': checks, + 'features': features, + 'cleaned_up': not folder.exists(), + 'configuration_fields': configuration_fields, + 'tool_sandbox': tool_sandbox_details, + } diff --git a/src/msg/admin/dm_check.py b/src/msg/admin/dm_check.py index 63b9ba37..e37de964 100644 --- a/src/msg/admin/dm_check.py +++ b/src/msg/admin/dm_check.py @@ -1,4 +1,5 @@ """Exercise private conversations using fresh clients in the isolated Test Root.""" + from pathlib import Path from tempfile import TemporaryDirectory from uuid import uuid4 @@ -19,29 +20,39 @@ async def check_private_dm(app, now): async with httpx.AsyncClient(transport=httpx.ASGITransport(app=create_app(app))) as http: clients = [] for name in ('sender', 'recipient', 'outsider'): - client = MsgClient(ClientState(Path(directory) / name, - server=app.settings.service_url), + client = MsgClient( + ClientState(Path(directory) / name, server=app.settings.service_url), HTTPTransport(app.settings.service_url, http=http), - clock=lambda: now, retries=0) + clock=lambda: now, + retries=0, + ) client.checked(await client.register('dm-' + uuid4().hex[:16])) clients.append(client) sender, recipient, outsider = clients - requested = sender.checked(await sender.call('communication.dm_request', - {'recipient': recipient.state.subject})) + requested = sender.checked( + await sender.call( + 'communication.dm_request', {'recipient': recipient.state.subject} + ) + ) conversation = requested.data['conversation_id'] if requested.data['state'] != 'pending': return False - early = await sender.call('communication.dm_send', - {'conversation_id': conversation, 'body': 'not accepted'}) + early = await sender.call( + 'communication.dm_send', {'conversation_id': conversation, 'body': 'not accepted'} + ) if early.status != 'error' or early.error.code != 'dm_controlled_resource': return False - accepted = recipient.checked(await recipient.call('communication.dm_accept', - {'conversation_id': conversation})) + accepted = recipient.checked( + await recipient.call('communication.dm_accept', {'conversation_id': conversation}) + ) if accepted.data['state'] != 'active': return False body = 'isolated private conversation ' + uuid4().hex - sent = sender.checked(await sender.call('communication.dm_send', - {'conversation_id': conversation, 'body': body})) + sent = sender.checked( + await sender.call( + 'communication.dm_send', {'conversation_id': conversation, 'body': body} + ) + ) post = sent.resources[0].id for participant in (sender, recipient): read = await participant.call('discovery.get', {'id': post}) @@ -54,12 +65,15 @@ async def check_private_dm(app, now): ('communication.dm_send', {'conversation_id': conversation, 'body': 'intrusion'}), ): denied = await outsider.call(operation, arguments) - expected = 'dm_not_found' if operation == 'communication.dm_send' else 'permission_denied' + expected = ( + 'dm_not_found' if operation == 'communication.dm_send' else 'permission_denied' + ) if denied.status != 'error' or denied.error.code != expected: return False hidden = await outsider.call('discovery.search', {'query': body}) if hidden.status != 'ok' or hidden.data.get('items'): return False anonymous = await HTTPTransport(app.settings.service_url, http=http).call( - request_for('discovery.get', {'id': post}, app.settings.service_url)) + request_for('discovery.get', {'id': post}, app.settings.service_url) + ) return anonymous.status == 'error' and anonymous.error.code == 'permission_denied' diff --git a/src/msg/admin/git_check.py b/src/msg/admin/git_check.py index 536b3906..f4b92f4f 100644 --- a/src/msg/admin/git_check.py +++ b/src/msg/admin/git_check.py @@ -1,10 +1,11 @@ """Real Git publication/fetch checks inside the disposable selftest installation.""" + import asyncio -from datetime import timedelta import hashlib -from pathlib import Path import subprocess import tempfile +from datetime import timedelta +from pathlib import Path from urllib.parse import urlsplit import httpx @@ -15,8 +16,8 @@ from msg.core.errors import Failure, require from msg.core.requests import request_for from msg.extensions.repositories import NativeGitStore -from msg.transports.http import create_app from msg.transports.git_http import GitHTTPAdapter +from msg.transports.http import create_app async def check_git(app, call, register): @@ -26,12 +27,22 @@ async def check_git(app, call, register): require(created.status == 'ok', 'selftest_git_create_failed') rid = created.resources[0].id store = NativeGitStore(app) - env = {**store.env, 'GIT_AUTHOR_NAME': 'Selftest', 'GIT_AUTHOR_EMAIL': 'selftest@example.invalid'} + env = { + **store.env, + 'GIT_AUTHOR_NAME': 'Selftest', + 'GIT_AUTHOR_EMAIL': 'selftest@example.invalid', + } def run(folder, *args, input=None): try: - return subprocess.run(['git', '-C', str(folder), *args], input=input, - capture_output=True, check=True, timeout=30, env=env).stdout + return subprocess.run( + ['git', '-C', str(folder), *args], + input=input, + capture_output=True, + check=True, + timeout=30, + env=env, + ).stdout except (OSError, subprocess.SubprocessError) as exc: raise Failure('selftest_git_command_failed') from exc @@ -40,9 +51,20 @@ def pkt(data): async def snapshot(): async with app.metadata.transaction(write=False) as tx: - return {table: tuple(tx.rows(f'SELECT * FROM {table} ORDER BY 1')) - for table in ('resources', 'events', 'audit', 'jobs', 'results', 'transfers', - 'credentials', 'reactions', 'watches')} + return { + table: tuple(tx.rows(f'SELECT * FROM {table} ORDER BY 1')) + for table in ( + 'resources', + 'events', + 'audit', + 'jobs', + 'results', + 'transfers', + 'credentials', + 'reactions', + 'watches', + ) + } with tempfile.TemporaryDirectory(prefix='msg-selftest-git-') as temporary: work = Path(temporary) @@ -52,62 +74,114 @@ async def snapshot(): await asyncio.to_thread(run, work, 'add', 'README.md') await asyncio.to_thread(run, work, 'commit', '-m', 'isolated selftest') commit = (await asyncio.to_thread(run, work, 'rev-parse', 'HEAD')).decode().strip() - pack = await asyncio.to_thread(run, work, 'pack-objects', '--stdout', '--revs', - input=(commit + '\n').encode()) - receive = pkt(f'{"0" * 40} {commit} refs/heads/main\0report-status\n'.encode()) + b'0000' + pack + pack = await asyncio.to_thread( + run, work, 'pack-objects', '--stdout', '--revs', input=(commit + '\n').encode() + ) + receive = ( + pkt(f'{"0" * 40} {commit} refs/heads/main\0report-status\n'.encode()) + b'0000' + pack + ) def headers(request_id): - packet = request_for('git.http_receive', {'id': rid, - 'pack_digest': 'sha256:' + hashlib.sha256(receive).hexdigest(), 'pack_size': len(receive)}, - app.settings.service_url, signer=key, subject=owner, request_id=request_id, - expires_at=app.clock() + timedelta(seconds=120)) - return {'X-Msg-Request': b64(canonical(packet)), - 'Content-Type': 'application/x-git-receive-pack-request'} + packet = request_for( + 'git.http_receive', + { + 'id': rid, + 'pack_digest': 'sha256:' + hashlib.sha256(receive).hexdigest(), + 'pack_size': len(receive), + }, + app.settings.service_url, + signer=key, + subject=owner, + request_id=request_id, + expires_at=app.clock() + timedelta(seconds=120), + ) + return { + 'X-Msg-Request': b64(canonical(packet)), + 'Content-Type': 'application/x-git-receive-pack-request', + } - async with httpx.AsyncClient(transport=httpx.ASGITransport(app=create_app(app)), - base_url=app.settings.service_url) as http: + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=create_app(app)), base_url=app.settings.service_url + ) as http: endpoint = f'/-/git/{rid}/git-receive-pack' - denied = await http.post(endpoint, content=receive, - headers={'Content-Type': 'application/x-git-receive-pack-request'}) + denied = await http.post( + endpoint, + content=receive, + headers={'Content-Type': 'application/x-git-receive-pack-request'}, + ) require(denied.status_code == 401, 'selftest_git_unsigned_push_allowed') - pushed = await http.post(endpoint, content=receive, headers=headers('selftest-git-push')) - require(pushed.status_code == 200 and b'ok refs/heads/main\n' in pushed.content, - 'selftest_git_push_failed') - require(await asyncio.to_thread(run, store.path(rid), 'show', 'refs/heads/main:README.md') == payload, - 'selftest_git_publication_mismatch') + pushed = await http.post( + endpoint, content=receive, headers=headers('selftest-git-push') + ) + require( + pushed.status_code == 200 and b'ok refs/heads/main\n' in pushed.content, + 'selftest_git_push_failed', + ) + require( + await asyncio.to_thread(run, store.path(rid), 'show', 'refs/heads/main:README.md') + == payload, + 'selftest_git_publication_mismatch', + ) before = await snapshot() - replay = await http.post(endpoint, content=receive, headers=headers('selftest-git-push')) - require(replay.status_code == 200 and replay.content == pushed.content and - await snapshot() == before, 'selftest_git_replay_mutated') + replay = await http.post( + endpoint, content=receive, headers=headers('selftest-git-push') + ) + require( + replay.status_code == 200 + and replay.content == pushed.content + and await snapshot() == before, + 'selftest_git_replay_mutated', + ) # A new request with an obsolete expected ref must not overwrite it. - stale = await http.post(endpoint, content=receive, headers=headers('selftest-git-stale')) - require(stale.status_code == 200 and b'ng refs/heads/main ' in stale.content, - 'selftest_git_stale_ref_allowed') - refs = await asyncio.to_thread(run, store.path(rid), 'for-each-ref', '--format=%(refname) %(objectname)') + stale = await http.post( + endpoint, content=receive, headers=headers('selftest-git-stale') + ) + require( + stale.status_code == 200 and b'ng refs/heads/main ' in stale.content, + 'selftest_git_stale_ref_allowed', + ) + refs = await asyncio.to_thread( + run, store.path(rid), 'for-each-ref', '--format=%(refname) %(objectname)' + ) require(refs == f'refs/heads/main {commit}\n'.encode(), 'selftest_git_ref_changed') before = await snapshot() + # Production's /@user/*.git router does not expose /_test/. # Mount the real read adapter over its isolated resource path only; # it still executes git.refs authorization and git http-backend. async def read(request): return await GitHTTPAdapter(app).http( - request, urlsplit(created.data['read_url']).path, 'git-upload-pack') + request, urlsplit(created.data['read_url']).path, 'git-upload-pack' + ) + fetch_app = Starlette(routes=[Route('/fetch', read, methods=['POST'])]) - async with httpx.AsyncClient(transport=httpx.ASGITransport(app=fetch_app), - base_url=app.settings.service_url) as reader: - fetch = await reader.post('/fetch', + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=fetch_app), base_url=app.settings.service_url + ) as reader: + fetch = await reader.post( + '/fetch', content=pkt(f'want {commit}\n'.encode()) + b'00000009done\n', - headers={'Content-Type': 'application/x-git-upload-pack-request'}) - require(fetch.status_code == 200 and fetch.content.startswith(b'0008NAK\nPACK'), - 'selftest_git_fetch_failed') + headers={'Content-Type': 'application/x-git-upload-pack-request'}, + ) + require( + fetch.status_code == 200 and fetch.content.startswith(b'0008NAK\nPACK'), + 'selftest_git_fetch_failed', + ) # Import the actual response into a fresh repository, not the source fixture. fetched = work / 'fetched' fetched.mkdir() await asyncio.to_thread(run, fetched, 'init', '--bare', '--object-format=sha1') await asyncio.to_thread(run, fetched, 'index-pack', '--stdin', input=fetch.content[8:]) - require(await asyncio.to_thread(run, fetched, 'show', commit + ':README.md') == payload, - 'selftest_git_fetch_mismatch') - require(await snapshot() == before and refs == await asyncio.to_thread( - run, store.path(rid), 'for-each-ref', '--format=%(refname) %(objectname)'), - 'selftest_git_read_mutated') + require( + await asyncio.to_thread(run, fetched, 'show', commit + ':README.md') == payload, + 'selftest_git_fetch_mismatch', + ) + require( + await snapshot() == before + and refs + == await asyncio.to_thread( + run, store.path(rid), 'for-each-ref', '--format=%(refname) %(objectname)' + ), + 'selftest_git_read_mutated', + ) return True diff --git a/src/msg/admin/honor_check.py b/src/msg/admin/honor_check.py index 0da52930..fb9ab035 100644 --- a/src/msg/admin/honor_check.py +++ b/src/msg/admin/honor_check.py @@ -1,4 +1,5 @@ """Read-only honor diagnostics and the isolated selftest's real protocol client.""" + from msg.bootstrap import manifest from msg.core.codec import canonical, decode, digest from msg.core.errors import require @@ -21,42 +22,67 @@ async def inspect_honors(app, tx): 'achievement_grants': {'id', 'subject', 'achievement_id', 'spec_version', 'body'}, 'achievement_pins': {'subject', 'grant_id', 'position'}, }.items(): - columns = {row[0] for row in tx.rows( - "SELECT column_name FROM information_schema.columns " - "WHERE table_schema='public' AND table_name=?", (table,))} + columns = { + row[0] + for row in tx.rows( + 'SELECT column_name FROM information_schema.columns ' + "WHERE table_schema='public' AND table_name=?", + (table,), + ) + } require(required <= columns, 'honor_schema_missing', table) for name in ('start', 'answer', 'finish', 'pin', 'unpin', 'reorder', 'list'): spec = app.registry.operation('achievement.' + name) - require(spec.effect == ('read' if name == 'list' else 'transaction'), - 'honor_contract_invalid') + require( + spec.effect == ('read' if name == 'list' else 'transaction'), 'honor_contract_invalid' + ) if name in {'finish', 'pin', 'unpin', 'reorder'}: require(spec.require_signature, 'honor_contract_invalid') sample = manifest()['honor_sample'] - require(sample['policy'] == { - 'achievement_id': I_AM_NOT_HUMAN.id, 'spec_version': I_AM_NOT_HUMAN.version, - 'rounds': 5, 'round_ttl_seconds': ROUND_TTL_SECONDS, - 'ceremony_ttl_seconds': CEREMONY_TTL_SECONDS, 'strategy': STRATEGY, - 'strategy_version': STRATEGY_VERSION, 'privileges': False, - } and sample['profile']['max_pins'] == MAX_PINS and - sample['profile']['pinned_grant_ids'] == [] and sample['grants'] == [] and - sample['issuer']['claim'] == I_AM_NOT_HUMAN.claim and - sample['issuer']['purpose'] == 'achievement-grant' and - sample['issuer']['key_source'] == 'service_receipt_signer', 'honor_sample_drift') + require( + sample['policy'] + == { + 'achievement_id': I_AM_NOT_HUMAN.id, + 'spec_version': I_AM_NOT_HUMAN.version, + 'rounds': 5, + 'round_ttl_seconds': ROUND_TTL_SECONDS, + 'ceremony_ttl_seconds': CEREMONY_TTL_SECONDS, + 'strategy': STRATEGY, + 'strategy_version': STRATEGY_VERSION, + 'privileges': False, + } + and sample['profile']['max_pins'] == MAX_PINS + and sample['profile']['pinned_grant_ids'] == [] + and sample['grants'] == [] + and sample['issuer']['claim'] == I_AM_NOT_HUMAN.claim + and sample['issuer']['purpose'] == 'achievement-grant' + and sample['issuer']['key_source'] == 'service_receipt_signer', + 'honor_sample_drift', + ) resource = await tx.resource('r_honor_sample') revision = await tx.revision(ResourceRef(id=resource.id, revision=resource.revision)) - require(revision.content is not None and - revision.content.digest == digest(canonical(sample)), - 'honor_sample_drift') + require( + revision.content is not None and revision.content.digest == digest(canonical(sample)), + 'honor_sample_drift', + ) # Doctor does not load Application storage (which can create directories). # Read the installed issuer directly, as the online-CA doctor does. issuer = Ed25519Signer.from_bytes((app.settings.service_keys / 'receipt.key').read_bytes()) probe = canonical({'purpose': 'read-only-honor-doctor'}) - verify(issuer.public_key, probe, - issuer.sign(probe, purpose='achievement-grant'), - purpose='achievement-grant') - return {'read_only': True, 'issuer': issuer.key_id, - 'strategy': STRATEGY, 'spec_version': I_AM_NOT_HUMAN.version, - 'pregrant': False, 'privileges': False} + verify( + issuer.public_key, + probe, + issuer.sign(probe, purpose='achievement-grant'), + purpose='achievement-grant', + ) + return { + 'read_only': True, + 'issuer': issuer.key_id, + 'strategy': STRATEGY, + 'spec_version': I_AM_NOT_HUMAN.version, + 'pregrant': False, + 'privileges': False, + } async def check_honors(app, call, register): @@ -64,11 +90,17 @@ async def check_honors(app, call, register): key, subject = await register('honor-selftest') stranger, other = await register('honor-stranger') async with app.metadata.transaction(write=False) as tx: - security_before = (tx.rows('SELECT body FROM certificates ORDER BY id'), - tx.rows('SELECT body FROM credentials ORDER BY id')) + security_before = ( + tx.rows('SELECT body FROM certificates ORDER BY id'), + tx.rows('SELECT body FROM credentials ORDER BY id'), + ) empty = await call('achievement.list', {'subject_id': subject}) - require(empty.status == 'ok' and not empty.data['achievements'] and - not empty.data['pinned_grant_ids'], 'selftest_honor_pregranted') + require( + empty.status == 'ok' + and not empty.data['achievements'] + and not empty.data['pinned_grant_ids'], + 'selftest_honor_pregranted', + ) started = await call('achievement.start', {}, key, subject) require(started.status == 'ok', 'selftest_honor_start_failed') challenge = started.data @@ -76,29 +108,46 @@ async def check_honors(app, call, register): require(challenge.get('round') == number, 'selftest_honor_round_invalid') answer = 'y' if number == 4: - bits = ''.join('1' if char == '\u200c' else '0' for char in challenge['question'] - if char in {'\u200b', '\u200c'}) + bits = ''.join( + '1' if char == '\u200c' else '0' + for char in challenge['question'] + if char in {'\u200b', '\u200c'} + ) require(len(bits) == 16, 'selftest_honor_payload_invalid') - answer = ''.join(str(int(bits[i:i + 4], 2)) for i in range(0, 16, 4)) - result = await call('achievement.answer', { - 'challenge_id': challenge['challenge_id'], 'round': number, - 'question_digest': challenge['question_digest'], 'nonce': challenge['nonce'], - 'answer': answer, - }, key, subject) - require(result.status == 'ok' and result.data.get('round') == number + 1, - 'selftest_honor_answer_failed') + answer = ''.join(str(int(bits[i : i + 4], 2)) for i in range(0, 16, 4)) + result = await call( + 'achievement.answer', + { + 'challenge_id': challenge['challenge_id'], + 'round': number, + 'question_digest': challenge['question_digest'], + 'nonce': challenge['nonce'], + 'answer': answer, + }, + key, + subject, + ) + require( + result.status == 'ok' and result.data.get('round') == number + 1, + 'selftest_honor_answer_failed', + ) challenge = result.data - finished = await call('achievement.finish', { - **{field: challenge[field] for field in - ('challenge_id', 'question_digest', 'nonce', 'ceremony_digest')}, - 'statement': FINAL_STATEMENT, - }, key, subject) - require(finished.status == 'ok' and 'grant' in finished.data, - 'selftest_honor_finish_failed') + finished = await call( + 'achievement.finish', + { + **{ + field: challenge[field] + for field in ('challenge_id', 'question_digest', 'nonce', 'ceremony_digest') + }, + 'statement': FINAL_STATEMENT, + }, + key, + subject, + ) + require(finished.status == 'ok' and 'grant' in finished.data, 'selftest_honor_finish_failed') grant = dict(finished.data['grant']) signature = decode(Signature, grant.pop('signature')) - verify(app.receipt_signer.public_key, canonical(grant), signature, - purpose='achievement-grant') + verify(app.receipt_signer.public_key, canonical(grant), signature, purpose='achievement-grant') gid = grant['id'] denied = await call('achievement.pin', {'grant_id': gid}, stranger, other) pinned = await call('achievement.pin', {'grant_id': gid}, key, subject) @@ -108,13 +157,20 @@ async def check_honors(app, call, register): final = await call('achievement.list', {'subject_id': subject}) duplicate = await call('achievement.start', {}, key, subject) async with app.metadata.transaction(write=False) as tx: - security_after = (tx.rows('SELECT body FROM certificates ORDER BY id'), - tx.rows('SELECT body FROM credentials ORDER BY id')) + security_after = ( + tx.rows('SELECT body FROM certificates ORDER BY id'), + tx.rows('SELECT body FROM credentials ORDER BY id'), + ) count = tx.one('SELECT COUNT(*) FROM achievement_grants WHERE subject=?', (subject,))[0] - return (denied.error is not None and denied.error.code == 'achievement_grant_not_found' and - all(result.status == 'ok' for result in (pinned, reordered, public, unpinned, final)) and - list(public.data['pinned_grant_ids']) == [gid] and - not final.data['pinned_grant_ids'] and len(final.data['achievements']) == 1 and - count == 1 and duplicate.error is not None and - duplicate.error.code == 'achievement_already_granted' and - security_after == security_before) + return ( + denied.error is not None + and denied.error.code == 'achievement_grant_not_found' + and all(result.status == 'ok' for result in (pinned, reordered, public, unpinned, final)) + and list(public.data['pinned_grant_ids']) == [gid] + and not final.data['pinned_grant_ids'] + and len(final.data['achievements']) == 1 + and count == 1 + and duplicate.error is not None + and duplicate.error.code == 'achievement_already_granted' + and security_after == security_before + ) diff --git a/src/msg/admin/legacy_approval.py b/src/msg/admin/legacy_approval.py index 13379059..345495d8 100644 --- a/src/msg/admin/legacy_approval.py +++ b/src/msg/admin/legacy_approval.py @@ -19,18 +19,20 @@ from msg.security.crypto import Ed25519Signer, open_private_key from msg.security.quarantine import require_live_authority from msg.security.root_files import read_private, rotation_lock -from msg.storage import legacy_git_import as git -from msg.storage import legacy_identity_plan as identities -from msg.storage import legacy_resource_import as content +from msg.storage import ( + legacy_git_import as git, + legacy_identity_plan as identities, + legacy_resource_import as content, +) def write_new(path, value): path = Path(path) - require(not path.exists() and not path.is_symlink(), "approval_destination_exists") + require(not path.exists() and not path.is_symlink(), 'approval_destination_exists') fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600) - with os.fdopen(fd, "wb") as stream: + with os.fdopen(fd, 'wb') as stream: stream.write(canonical(value)) - stream.write(b"\n") + stream.write(b'\n') stream.flush() os.fsync(stream.fileno()) @@ -40,109 +42,109 @@ def read_json(path): def recognized_payload(value): - require(isinstance(value, dict), "legacy_approval_invalid") - kind = value.get("format") + require(isinstance(value, dict), 'legacy_approval_invalid') + kind = value.get('format') common = { - key: value.get(key) for key in ("parent", "parent_generation", "operator", "expires_at") + key: value.get(key) for key in ('parent', 'parent_generation', 'operator', 'expires_at') } if kind == content.PURPOSE: expected = content.approval_payload( - sha256=value.get("source_sha256"), - service=value.get("target_service"), - identities=value.get("identities"), + sha256=value.get('source_sha256'), + service=value.get('target_service'), + identities=value.get('identities'), **common, ) elif kind == git.PURPOSE: expected = git.git_approval( - bundle_sha256=value.get("bundle_sha256"), - refs_digest=value.get("refs_digest"), - default_ref=value.get("default_ref"), - name=value.get("name"), - service=value.get("target_service"), + bundle_sha256=value.get('bundle_sha256'), + refs_digest=value.get('refs_digest'), + default_ref=value.get('default_ref'), + name=value.get('name'), + service=value.get('target_service'), **common, ) elif kind == identities.PURPOSE: expected = identities.approval_payload( - plan_digest=value.get("plan_digest"), - mappings=value.get("mappings"), - service=value.get("target_service"), + plan_digest=value.get('plan_digest'), + mappings=value.get('mappings'), + service=value.get('target_service'), **common, ) else: - require(False, "unsupported_legacy_approval_purpose") - require(value == expected, "legacy_approval_invalid") + require(False, 'unsupported_legacy_approval_purpose') + require(value == expected, 'legacy_approval_invalid') return kind async def validate_and_review(app, approval): kind = recognized_payload(approval) - require(approval["target_service"] == app.settings.service_url, "legacy_service_mismatch") + require(approval['target_service'] == app.settings.service_url, 'legacy_service_mismatch') require( - app.clock() < parse_time(approval["expires_at"]) <= app.clock() + timedelta(hours=24), - "legacy_approval_expired", + app.clock() < parse_time(approval['expires_at']) <= app.clock() + timedelta(hours=24), + 'legacy_approval_expired', ) - mappings = approval.get("identities", approval.get("mappings", {})) - require(isinstance(mappings, dict), "legacy_mapping_invalid") + mappings = approval.get('identities', approval.get('mappings', {})) + require(isinstance(mappings, dict), 'legacy_mapping_invalid') async with app.metadata.transaction(write=False) as tx: require_live_authority(tx) - require(tx.setting("runtime_config", {}).get("accept_writes", True), "writes_paused") - holder = await tx.subject(approval["operator"]) - require(holder.kind == "registered", "legacy_operator_must_be_registered") + require(tx.setting('runtime_config', {}).get('accept_writes', True), 'writes_paused') + holder = await tx.subject(approval['operator']) + require(holder.kind == 'registered', 'legacy_operator_must_be_registered') holder_resource = await tx.resource(holder.resource_id) - parent = await tx.resource(approval["parent"]) + parent = await tx.resource(approval['parent']) require( parent.owner == holder.resource_id and parent.mode == 0o700 - and parent.state == "active" - and parent.generation == approval["parent_generation"], - "legacy_parent_not_private", + and parent.state == 'active' + and parent.generation == approval['parent_generation'], + 'legacy_parent_not_private', ) require( app.registry.resource_type(parent.type, parent.type_version).container, - "not_a_container", + 'not_a_container', ) if kind != git.PURPOSE: require( - not tx.one("SELECT id FROM resources WHERE parent=?", (parent.id,)), - "legacy_parent_not_empty", + not tx.one('SELECT id FROM resources WHERE parent=?', (parent.id,)), + 'legacy_parent_not_empty', ) target_counts = Counter() for target in mappings.values(): - require(target is None or isinstance(target, str), "legacy_mapping_invalid") + require(target is None or isinstance(target, str), 'legacy_mapping_invalid') if target is not None: require( - (await tx.subject(target)).kind == "registered", "legacy_mapping_target_invalid" + (await tx.subject(target)).kind == 'registered', 'legacy_mapping_target_invalid' ) target_counts[target] += 1 return { - "purpose": kind, - "approval_digest": digest(approval), - "artifact_binding": { + 'purpose': kind, + 'approval_digest': digest(approval), + 'artifact_binding': { key: approval[key] for key in ( - "source_sha256", - "bundle_sha256", - "refs_digest", - "default_ref", - "name", - "plan_digest", + 'source_sha256', + 'bundle_sha256', + 'refs_digest', + 'default_ref', + 'name', + 'plan_digest', ) if key in approval }, - "target_service": app.settings.service_url, - "root_public_key_digest": digest(app.certificates.root_public_key), - "real_holder": {"subject": holder.resource_id, "handle": holder_resource.name}, - "private_parent": { - "id": parent.id, - "path": await tx.path(parent.id), - "generation": parent.generation, + 'target_service': app.settings.service_url, + 'root_public_key_digest': digest(app.certificates.root_public_key), + 'real_holder': {'subject': holder.resource_id, 'handle': holder_resource.name}, + 'private_parent': { + 'id': parent.id, + 'path': await tx.path(parent.id), + 'generation': parent.generation, }, - "historical_identity_count": len(mappings), - "unmapped_count": sum(v is None for v in mappings.values()), - "explicit_new_subject_associations": dict(target_counts), - "legacy_authority_enabled": False, - "ownership_proof_claimed": False, - "expires_at": approval["expires_at"], + 'historical_identity_count': len(mappings), + 'unmapped_count': sum(v is None for v in mappings.values()), + 'explicit_new_subject_associations': dict(target_counts), + 'legacy_authority_enabled': False, + 'ownership_proof_claimed': False, + 'expires_at': approval['expires_at'], } @@ -160,30 +162,30 @@ def sign_approval(config_dir, source, destination): require_local_console(config_dir) require( not Path(destination).exists() and not Path(destination).is_symlink(), - "approval_destination_exists", + 'approval_destination_exists', ) approval = read_json(source) review, public = asyncio.run(inspect_approval(config_dir, approval)) print(canonical(review).decode()) - phrase = "APPROVE LEGACY IMPORT " + digest(approval) - require(input("Type " + phrase + ": ") == phrase, "approval_cancelled") + phrase = 'APPROVE LEGACY IMPORT ' + digest(approval) + require(input('Type ' + phrase + ': ') == phrase, 'approval_cancelled') settings = load_settings(config_dir) with rotation_lock(settings.root_private_dir): envelope = loads(read_private(root_envelope(config_dir))) - private = open_private_key(envelope, getpass.getpass("Root PIN/passphrase: ")) + private = open_private_key(envelope, getpass.getpass('Root PIN/passphrase: ')) signer = Ed25519Signer.from_bytes(private) - require(signer.public_key == public, "root_key_mismatch") + require(signer.public_key == public, 'root_key_mismatch') result = { - "approval": approval, - "signature": wire( + 'approval': approval, + 'signature': wire( signer.sign(canonical(approval), purpose=recognized_payload(approval)) ), } write_new(destination, result) return { - "status": "legacy_approval_signed", - "purpose": approval["format"], - "approval_digest": digest(approval), + 'status': 'legacy_approval_signed', + 'purpose': approval['format'], + 'approval_digest': digest(approval), } @@ -194,42 +196,42 @@ async def prepare(args): async with app.metadata.transaction(write=False) as tx: parent = await tx.resource(args.parent) common = { - "service": app.settings.service_url, - "parent": args.parent, - "parent_generation": parent.generation, - "operator": args.operator, - "expires_at": wire(app.clock() + timedelta(hours=2)), + 'service': app.settings.service_url, + 'parent': args.parent, + 'parent_generation': parent.generation, + 'operator': args.operator, + 'expires_at': wire(app.clock() + timedelta(hours=2)), } - if args.command == "prepare-content": + if args.command == 'prepare-content': needed = content.identity_requirements( args.snapshot, args.sha256, {}, summary_only=False - )["required"] - mapping = {old: None for old in needed} if args.unmapped else read_json(args.mappings) - require(set(mapping) == set(needed), "legacy_identity_mapping_incomplete") + )['required'] + mapping = dict.fromkeys(needed) if args.unmapped else read_json(args.mappings) + require(set(mapping) == set(needed), 'legacy_identity_mapping_incomplete') approval = content.approval_payload(sha256=args.sha256, identities=mapping, **common) - elif args.command == "prepare-git": + elif args.command == 'prepare-git': manifest = read_json(args.manifest) verified = git.verify_bundle( args.bundle, - manifest["bundle_sha256"], + manifest['bundle_sha256'], protected_work=app.settings.server.staging_dir, ) require( - verified["refs"] == manifest["refs"] and manifest.get("refs_stable") is True, - "legacy_git_refs_mismatch", + verified['refs'] == manifest['refs'] and manifest.get('refs_stable') is True, + 'legacy_git_refs_mismatch', ) approval = git.git_approval( - bundle_sha256=verified["bundle_sha256"], - refs_digest=verified["refs_digest"], - default_ref=manifest["default_ref"], + bundle_sha256=verified['bundle_sha256'], + refs_digest=verified['refs_digest'], + default_ref=manifest['default_ref'], name=args.name, **common, ) else: plan = read_json(args.plan) - needed = {row["old_id"] for row in plan["identities"]} - mapping = {old: None for old in needed} if args.unmapped else read_json(args.mappings) - require(set(mapping) == needed, "legacy_identity_mapping_incomplete") + needed = {row['old_id'] for row in plan['identities']} + mapping = dict.fromkeys(needed) if args.unmapped else read_json(args.mappings) + require(set(mapping) == needed, 'legacy_identity_mapping_incomplete') approval = identities.approval_payload( plan_digest=digest(plan), mappings=mapping, **common ) @@ -246,7 +248,7 @@ async def apply_identities(args): await app.load() envelope = read_json(args.signed_approval) return await identities.import_identity_records( - app, read_json(args.plan), envelope["approval"], envelope["signature"] + app, read_json(args.plan), envelope['approval'], envelope['signature'] ) finally: await app.close() @@ -254,45 +256,45 @@ async def apply_identities(args): def parser(): command = argparse.ArgumentParser(description=__doc__) - command.add_argument("--config-dir", type=Path, required=True) - sub = command.add_subparsers(dest="command", required=True) - for kind in ("content", "git", "identities"): - item = sub.add_parser("prepare-" + kind) - item.add_argument("--operator", required=True) - item.add_argument("--parent", required=True) - item.add_argument("--output", type=Path, required=True) - if kind == "content": - item.add_argument("--snapshot", type=Path, required=True) - item.add_argument("--sha256", required=True) - elif kind == "git": - item.add_argument("--bundle", type=Path, required=True) - item.add_argument("--manifest", type=Path, required=True) - item.add_argument("--name", required=True) + command.add_argument('--config-dir', type=Path, required=True) + sub = command.add_subparsers(dest='command', required=True) + for kind in ('content', 'git', 'identities'): + item = sub.add_parser('prepare-' + kind) + item.add_argument('--operator', required=True) + item.add_argument('--parent', required=True) + item.add_argument('--output', type=Path, required=True) + if kind == 'content': + item.add_argument('--snapshot', type=Path, required=True) + item.add_argument('--sha256', required=True) + elif kind == 'git': + item.add_argument('--bundle', type=Path, required=True) + item.add_argument('--manifest', type=Path, required=True) + item.add_argument('--name', required=True) else: - item.add_argument("--plan", type=Path, required=True) - if kind != "git": + item.add_argument('--plan', type=Path, required=True) + if kind != 'git': mapping = item.add_mutually_exclusive_group(required=True) - mapping.add_argument("--mappings", type=Path) - mapping.add_argument("--unmapped", action="store_true") - sign = sub.add_parser("sign") - sign.add_argument("--draft", type=Path, required=True) - sign.add_argument("--output", type=Path, required=True) - apply = sub.add_parser("apply-identities") - apply.add_argument("--plan", type=Path, required=True) - apply.add_argument("--signed-approval", type=Path, required=True) + mapping.add_argument('--mappings', type=Path) + mapping.add_argument('--unmapped', action='store_true') + sign = sub.add_parser('sign') + sign.add_argument('--draft', type=Path, required=True) + sign.add_argument('--output', type=Path, required=True) + apply = sub.add_parser('apply-identities') + apply.add_argument('--plan', type=Path, required=True) + apply.add_argument('--signed-approval', type=Path, required=True) return command def main(): args = parser().parse_args() - if args.command == "sign": + if args.command == 'sign': result = sign_approval(args.config_dir, args.draft, args.output) - elif args.command == "apply-identities": + elif args.command == 'apply-identities': result = asyncio.run(apply_identities(args)) else: result = asyncio.run(prepare(args)) print(json.dumps(result, sort_keys=True)) -if __name__ == "__main__": +if __name__ == '__main__': main() diff --git a/src/msg/admin/market.py b/src/msg/admin/market.py index 1bc971f0..a045ba78 100644 --- a/src/msg/admin/market.py +++ b/src/msg/admin/market.py @@ -1,4 +1,5 @@ """Root-signed, physical-console arbitration configuration; never a network API.""" + from __future__ import annotations import asyncio @@ -17,46 +18,86 @@ async def apply_market(app, signer, *, action, operator, subject=None, policy=None, expected=None): _root_signer(app, signer) - require(action in {'grant','revoke','publish'}, 'market_admin_action_invalid') + require(action in {'grant', 'revoke', 'publish'}, 'market_admin_action_invalid') if action == 'publish': policy = validate(policy) require(policy['id'] != DEFAULT_POLICY['id'], 'builtin_policy_immutable') async with app.metadata.transaction(write=True) as tx: if action == 'publish': - require(tx.one('SELECT 1 FROM arbitration_policies WHERE id=?', (policy['id'],)) is None, - 'arbitration_policy_exists') + require( + tx.one('SELECT 1 FROM arbitration_policies WHERE id=?', (policy['id'],)) is None, + 'arbitration_policy_exists', + ) for candidate in policy['candidates']: identity = await tx.subject(candidate) - require(identity.kind == 'registered' and not identity.local_only, 'invalid_arbitrator') - tx.execute('INSERT INTO arbitration_policies(id,body,digest) VALUES (?,?,?)', - (policy['id'],canonical(policy).decode(),digest(policy)), write=True) + require( + identity.kind == 'registered' and not identity.local_only, 'invalid_arbitrator' + ) + tx.execute( + 'INSERT INTO arbitration_policies(id,body,digest) VALUES (?,?,?)', + (policy['id'], canonical(policy).decode(), digest(policy)), + write=True, + ) before, after = None, policy else: identity = await tx.subject(subject) - require(identity.kind == 'registered' and not identity.local_only and subject != ROOT_SUBJECT, - 'invalid_arbitrator') + require( + identity.kind == 'registered' + and not identity.local_only + and subject != ROOT_SUBJECT, + 'invalid_arbitrator', + ) row = tx.one('SELECT epoch,active FROM arbitrator_roles WHERE subject=?', (subject,)) - before = {'epoch':row[0],'active':row[1]} if row else None + before = {'epoch': row[0], 'active': row[1]} if row else None if expected is not None: require(before == expected, 'market_preview_stale') - require((action=='grant' and (not row or not row[1])) or - (action=='revoke' and row and row[1]), 'arbitrator_role_unchanged') - after = {'epoch': new_id('role') if action=='grant' else row[0], 'active': action=='grant'} - tx.execute('''INSERT INTO arbitrator_roles(subject,epoch,active) VALUES (?,?,?) - ON CONFLICT(subject) DO UPDATE SET epoch=excluded.epoch,active=excluded.active''', - (subject,after['epoch'],after['active']), write=True) - statement = {'action':action,'subject':subject,'before':before,'after':after, - 'operator':operator,'at':wire(app.clock())} + require( + (action == 'grant' and (not row or not row[1])) + or (action == 'revoke' and row and row[1]), + 'arbitrator_role_unchanged', + ) + after = { + 'epoch': new_id('role') if action == 'grant' else row[0], + 'active': action == 'grant', + } + tx.execute( + """INSERT INTO arbitrator_roles(subject,epoch,active) VALUES (?,?,?) + ON CONFLICT(subject) DO UPDATE SET epoch=excluded.epoch,active=excluded.active""", + (subject, after['epoch'], after['active']), + write=True, + ) + statement = { + 'action': action, + 'subject': subject, + 'before': before, + 'after': after, + 'operator': operator, + 'at': wire(app.clock()), + } proof = signer.sign(canonical(statement), purpose='market-admin') - event = Event(id=new_id('audit'), type='root.market.'+action, time=app.clock(), - request_id=new_id('local-market'), actor=ROOT_SUBJECT, subject=ROOT_SUBJECT, - resources=(ResourceRef(id=ROOT_SUBJECT),), data={**statement,'root_signature':wire(proof)}) + event = Event( + id=new_id('audit'), + type='root.market.' + action, + time=app.clock(), + request_id=new_id('local-market'), + actor=ROOT_SUBJECT, + subject=ROOT_SUBJECT, + resources=(ResourceRef(id=ROOT_SUBJECT),), + data={**statement, 'root_signature': wire(proof)}, + ) await tx.append_event(event) - await tx.append_audit(AuditEvent(event=event, - authority=(ResourceRef(id=app.certificates.root_certificate.resource_id),), - before_digest=digest(before),after_digest=digest(after),previous_digest=None, - entry_digest='',result=action)) - return {'action':action,'audit_event_id':event.id,'result':after} + await tx.append_audit( + AuditEvent( + event=event, + authority=(ResourceRef(id=app.certificates.root_certificate.resource_id),), + before_digest=digest(before), + after_digest=digest(after), + previous_digest=None, + entry_digest='', + result=action, + ) + ) + return {'action': action, 'audit_event_id': event.id, 'result': after} class MarketAdmin: @@ -66,6 +107,7 @@ def __init__(self, config_dir): def execute(self, action, *, subject=None, policy_file=None): operator = require_local_console(self.config_dir) app = RootAdmin(self.config_dir)._app() + async def run(): await app.load() try: @@ -73,19 +115,37 @@ async def run(): resolved, before = subject, None async with app.metadata.transaction(write=False) as tx: if subject and subject.startswith('@'): - resolved = await tx.resolve('/'+subject) + resolved = await tx.resolve('/' + subject) if resolved: - row = tx.one('SELECT epoch,active FROM arbitrator_roles WHERE subject=?', (resolved,)) - before = {'epoch':row[0],'active':row[1]} if row else None - preview = {'action':action,'subject':resolved,'before':before,'policy':policy} + row = tx.one( + 'SELECT epoch,active FROM arbitrator_roles WHERE subject=?', (resolved,) + ) + before = {'epoch': row[0], 'active': row[1]} if row else None + preview = { + 'action': action, + 'subject': resolved, + 'before': before, + 'policy': policy, + } print(canonical(preview).decode()) - approval = 'CONFIRM MARKET '+digest(preview) - require(input('Type '+approval+' to continue: ') == approval, 'approval_cancelled') + approval = 'CONFIRM MARKET ' + digest(preview) + require( + input('Type ' + approval + ' to continue: ') == approval, 'approval_cancelled' + ) pin = getpass.getpass('Root PIN/passphrase: ') - signer = Ed25519Signer.from_bytes(open_private_key( - loads(root_envelope(self.config_dir).read_bytes()), pin)) - return await apply_market(app, signer, action=action, operator=operator, - subject=resolved,policy=policy,expected=before) + signer = Ed25519Signer.from_bytes( + open_private_key(loads(root_envelope(self.config_dir).read_bytes()), pin) + ) + return await apply_market( + app, + signer, + action=action, + operator=operator, + subject=resolved, + policy=policy, + expected=before, + ) finally: await app.close() + return asyncio.run(run()) diff --git a/src/msg/admin/market_check.py b/src/msg/admin/market_check.py index e40fdcc2..0a920077 100644 --- a/src/msg/admin/market_check.py +++ b/src/msg/admin/market_check.py @@ -1,4 +1,5 @@ """Read-only financial invariants and a deliberately isolated Test Root scenario.""" + from __future__ import annotations from copy import deepcopy @@ -16,161 +17,365 @@ async def inspect_market(tx): """No initialization, repair, timer, statement, event, job or network call.""" count = 0 - for oid,buyer,total,escrow,state in tx.rows( - 'SELECT id,buyer,total_price_minor,escrow_subject,state FROM store_orders ORDER BY id'): - expected = total if state in {'funded','delivered','accepted','disputed'} else 0 - require(_balance(tx,escrow)==expected, 'market_escrow_invariant') - require(tx.one('SELECT 1 FROM identities WHERE id=?',(escrow,)) is None and - tx.one('SELECT kind,subject_id FROM ledger_accounts WHERE id=?',(escrow,))==('order_escrow',None), - 'market_escrow_identity') - snapshot = tx.one('SELECT body FROM order_contracts WHERE order_id=?',(oid,)) + for oid, buyer, total, escrow, state in tx.rows( + 'SELECT id,buyer,total_price_minor,escrow_subject,state FROM store_orders ORDER BY id' + ): + expected = total if state in {'funded', 'delivered', 'accepted', 'disputed'} else 0 + require(_balance(tx, escrow) == expected, 'market_escrow_invariant') + require( + tx.one('SELECT 1 FROM identities WHERE id=?', (escrow,)) is None + and tx.one('SELECT kind,subject_id FROM ledger_accounts WHERE id=?', (escrow,)) + == ('order_escrow', None), + 'market_escrow_identity', + ) + snapshot = tx.one('SELECT body FROM order_contracts WHERE order_id=?', (oid,)) if snapshot: - locked = contract(tx,oid) + locked = contract(tx, oid) policy = locked['policy'] if locked['version'] != 5: validate(policy['policy']) - require(digest(policy['policy'])==policy['policy_digest'] and - locked['buyer']==buyer and locked['total_price_minor']==total, 'market_snapshot_invariant') - row = tx.one('SELECT body FROM order_settlements WHERE order_id=?',(oid,)) + require( + digest(policy['policy']) == policy['policy_digest'] + and locked['buyer'] == buyer + and locked['total_price_minor'] == total, + 'market_snapshot_invariant', + ) + row = tx.one('SELECT body FROM order_settlements WHERE order_id=?', (oid,)) if row: fact = loads(row[0]) - require(state in {'settled','refunded'} and fact['refund_minor']+fact['release_minor']==total, - 'market_settlement_invariant') - actual = tx.one("SELECT COALESCE(SUM(amount_minor),0) FROM money_ledger WHERE debit_account=?",(escrow,))[0] - require(actual==total,'market_settlement_invariant') + require( + state in {'settled', 'refunded'} + and fact['refund_minor'] + fact['release_minor'] == total, + 'market_settlement_invariant', + ) + actual = tx.one( + 'SELECT COALESCE(SUM(amount_minor),0) FROM money_ledger WHERE debit_account=?', + (escrow,), + )[0] + require(actual == total, 'market_settlement_invariant') elif snapshot: - require(state not in {'settled','refunded'},'market_settlement_missing') + require(state not in {'settled', 'refunded'}, 'market_settlement_missing') count += 1 - for raw,expected in tx.rows('SELECT body,digest FROM arbitration_policies'): - policy = loads(raw); validate(policy) - require(digest(policy)==expected,'arbitration_policy_corrupt') - for case_id,order_id,state in tx.rows('SELECT id,order_id,state FROM arbitration_cases'): - settlement = tx.one('SELECT decision_id FROM order_settlements WHERE order_id=?',(order_id,)) - if state=='executed': - require(settlement is not None and settlement[0] is not None,'market_decision_unfunded') + for raw, expected in tx.rows('SELECT body,digest FROM arbitration_policies'): + policy = loads(raw) + validate(policy) + require(digest(policy) == expected, 'arbitration_policy_corrupt') + for case_id, order_id, state in tx.rows('SELECT id,order_id,state FROM arbitration_cases'): + settlement = tx.one( + 'SELECT decision_id FROM order_settlements WHERE order_id=?', (order_id,) + ) + if state == 'executed': + require( + settlement is not None and settlement[0] is not None, 'market_decision_unfunded' + ) from msg.market.arbitration import _read from msg.market.rationale import record - row=tx.one('SELECT body FROM arbitration_decisions WHERE id=? AND case_id=?', - (settlement[0],case_id)) - require(row is not None,'market_decision_missing') - decision=loads(row[0]) - require(all(decision.get(k)==decision['proposal'].get(k) - for k in ('rationale_ref','rationale_digest')),'decision_rationale_mismatch') - record(tx,_read(tx,case_id),decision['proposal']) - return {'orders':count,'held_cases':tx.one("SELECT COUNT(*) FROM arbitration_cases WHERE state='held'")[0], - 'escrow_invariants':True,'private_reads_only':True} - - -async def check_market(app,root,call,register,now): + + row = tx.one( + 'SELECT body FROM arbitration_decisions WHERE id=? AND case_id=?', + (settlement[0], case_id), + ) + require(row is not None, 'market_decision_missing') + decision = loads(row[0]) + require( + all( + decision.get(k) == decision['proposal'].get(k) + for k in ('rationale_ref', 'rationale_digest') + ), + 'decision_rationale_mismatch', + ) + record(tx, _read(tx, case_id), decision['proposal']) + return { + 'orders': count, + 'held_cases': tx.one("SELECT COUNT(*) FROM arbitration_cases WHERE state='held'")[0], + 'escrow_invariants': True, + 'private_reads_only': True, + } + + +async def check_market(app, root, call, register, now): """20 MSG issue -> bank-funded 10 reward -> 5 bundle, entirely disposable.""" from msg.admin.market import apply_market from msg.admin.money import apply_money from msg.market.policy import DEFAULT_POLICY from msg.workers.effects import EffectWorker - require(app.selftest_run_id is not None,'selftest_namespace_required') + + require(app.selftest_run_id is not None, 'selftest_namespace_required') async with app.metadata.transaction(write=False) as tx: baseline_supply = _supply(tx) - bank,bank_id = await register('market-test-bank') - buyer,buyer_id = await register('market-test-buyer') - async def invoke(name,args,key=buyer,subject=buyer_id,**kw): - result = await call(name,args,key,subject,**kw) - require(result.status=='ok',result.error.code if result.error else 'market_selftest_failed') + bank, bank_id = await register('market-test-bank') + buyer, buyer_id = await register('market-test-buyer') + + async def invoke(name, args, key=buyer, subject=buyer_id, **kw): + result = await call(name, args, key, subject, **kw) + require( + result.status == 'ok', result.error.code if result.error else 'market_selftest_failed' + ) return result - await apply_money(app,root,action='bank_add',operator='isolated-selftest',subject_id=bank_id) - await apply_money(app,root,action='mint',operator='isolated-selftest',amount_minor=20_000_000) - await apply_money(app,root,action='transfer',operator='isolated-selftest',subject_id=bank_id,amount_minor=20_000_000) - bounty = await invoke('bounty.create',{'name':'identity-test','terms':'One-use signature test', - 'reward_minor':10_000_000,'budget_minor':10_000_000,'max_claims':1},bank,bank_id) - challenge = await invoke('bounty.challenge',{'listing_id':bounty.data['bounty']['listing_id']}) + + await apply_money( + app, root, action='bank_add', operator='isolated-selftest', subject_id=bank_id + ) + await apply_money( + app, root, action='mint', operator='isolated-selftest', amount_minor=20_000_000 + ) + await apply_money( + app, + root, + action='transfer', + operator='isolated-selftest', + subject_id=bank_id, + amount_minor=20_000_000, + ) + bounty = await invoke( + 'bounty.create', + { + 'name': 'identity-test', + 'terms': 'One-use signature test', + 'reward_minor': 10_000_000, + 'budget_minor': 10_000_000, + 'max_claims': 1, + }, + bank, + bank_id, + ) + challenge = await invoke( + 'bounty.challenge', {'listing_id': bounty.data['bounty']['listing_id']} + ) payload = challenge.data['challenge'] - await invoke('bounty.claim',{'challenge_id':payload['challenge_id'], - 'proof':wire(buyer.sign(canonical(payload),purpose='bounty-pop-v1'))}) - listing = await invoke('store.listing_create',{'name':'store-selftest','item_kind':'bundle', - 'price_minor':5_000_000,'currency_id':'primary','quantity':1,'delivery_mode':'managed_instant', - 'escrow_policy':'escrow-v1','dispute_policy':'dispute-v1','terms':'Immutable test bundle.'},bank,bank_id) + await invoke( + 'bounty.claim', + { + 'challenge_id': payload['challenge_id'], + 'proof': wire(buyer.sign(canonical(payload), purpose='bounty-pop-v1')), + }, + ) + listing = await invoke( + 'store.listing_create', + { + 'name': 'store-selftest', + 'item_kind': 'bundle', + 'price_minor': 5_000_000, + 'currency_id': 'primary', + 'quantity': 1, + 'delivery_mode': 'managed_instant', + 'escrow_policy': 'escrow-v1', + 'dispute_policy': 'dispute-v1', + 'terms': 'Immutable test bundle.', + }, + bank, + bank_id, + ) async with app.metadata.transaction(write=False) as tx: - parent = tx.one("SELECT id FROM resources WHERE parent=? AND name='files'",(bank_id,))[0] - file = await invoke('content.file_put',{'parent':parent,'name':'hello.txt','media_type':'text/plain', - 'data':b64(b'delivery-ok\n')},bank,bank_id) - package = await invoke('store.package_deposit',{'listing_id':listing.resources[0].id, - 'listing_revision':listing.resources[0].revision,'manifest':{'text':'msg.lmm.best store selftest'}, - 'payload_refs':[wire(file.resources[0])]},bank,bank_id) - active = await invoke('store.listing_update',{'id':listing.resources[0].id,'state':'active', - 'package_ref':package.data['package']['id']},bank,bank_id, - expected=((listing.resources[0].id,listing.data['generation']),)) + parent = tx.one("SELECT id FROM resources WHERE parent=? AND name='files'", (bank_id,))[0] + file = await invoke( + 'content.file_put', + { + 'parent': parent, + 'name': 'hello.txt', + 'media_type': 'text/plain', + 'data': b64(b'delivery-ok\n'), + }, + bank, + bank_id, + ) + package = await invoke( + 'store.package_deposit', + { + 'listing_id': listing.resources[0].id, + 'listing_revision': listing.resources[0].revision, + 'manifest': {'text': 'msg.lmm.best store selftest'}, + 'payload_refs': [wire(file.resources[0])], + }, + bank, + bank_id, + ) + active = await invoke( + 'store.listing_update', + { + 'id': listing.resources[0].id, + 'state': 'active', + 'package_ref': package.data['package']['id'], + }, + bank, + bank_id, + expected=((listing.resources[0].id, listing.data['generation']),), + ) old_settings = app.settings + class NoNetworkMail: - def __init__(self): self.messages=[] - async def send(self,job): - self.messages.append(dict(job.arguments)); return 'sent' - sender=NoNetworkMail() - app.settings=replace(app.settings,server=replace(app.settings.server,mail=MailConfig(enabled=True, - host='selftest.invalid',port=2525,tls='starttls',sender='test@selftest.invalid',credential_file=None))) + def __init__(self): + self.messages = [] + + async def send(self, job): + self.messages.append(dict(job.arguments)) + return 'sent' + + sender = NoNetworkMail() + app.settings = replace( + app.settings, + server=replace( + app.settings.server, + mail=MailConfig( + enabled=True, + host='selftest.invalid', + port=2525, + tls='starttls', + sender='test@selftest.invalid', + credential_file=None, + ), + ), + ) try: sale = active.data['listing'] - bought = await invoke('orders.buy',{'listing_id':sale['listing_id'], - 'listing_revision':sale['listing_revision'],'quantity':1,'total_price_minor':5_000_000, - 'currency_id':'primary','email':'buyer@selftest.invalid'},contract_version=3) + bought = await invoke( + 'orders.buy', + { + 'listing_id': sale['listing_id'], + 'listing_revision': sale['listing_revision'], + 'quantity': 1, + 'total_price_minor': 5_000_000, + 'currency_id': 'primary', + 'email': 'buyer@selftest.invalid', + }, + contract_version=3, + ) oid = bought.data['order']['id'] - worker=EffectWorker(app,mail_sender=sender) - while await worker.run_once(): pass - require(len(sender.messages)==1 and oid not in sender.messages[0]['text'],'market_email_leak') - proof=loads(sender.messages[0]['text']) - await invoke('orders.email_verify',{'order_id':oid,'challenge_id':proof['challenge_id'],'token':proof['token']}) - while await worker.run_once(): pass - require(len(sender.messages)==2 and set(loads(sender.messages[1]['text']))=={'order_id','handle','pickup'}, - 'market_email_leak') - delivery=await invoke('delivery.get',{'order_id':oid},contract_version=2) - require(delivery.data['delivery']['state']=='prepared' and - delivery.data['delivery']['payloads'][0]['data']==b64(b'delivery-ok\n'),'market_claim_invariant') + worker = EffectWorker(app, mail_sender=sender) + while await worker.run_once(): + pass + require( + len(sender.messages) == 1 and oid not in sender.messages[0]['text'], 'market_email_leak' + ) + proof = loads(sender.messages[0]['text']) + await invoke( + 'orders.email_verify', + {'order_id': oid, 'challenge_id': proof['challenge_id'], 'token': proof['token']}, + ) + while await worker.run_once(): + pass + require( + len(sender.messages) == 2 + and set(loads(sender.messages[1]['text'])) == {'order_id', 'handle', 'pickup'}, + 'market_email_leak', + ) + delivery = await invoke('delivery.get', {'order_id': oid}, contract_version=2) + require( + delivery.data['delivery']['state'] == 'prepared' + and delivery.data['delivery']['payloads'][0]['data'] == b64(b'delivery-ok\n'), + 'market_claim_invariant', + ) finally: - app.settings=old_settings + app.settings = old_settings # A separate 1 MSG service dispute refunds the test buyer through a real # fixed 2-of-3 panel. It leaves the official 5/15 final balances unchanged. - members={} + members = {} for i in range(3): - key,uid=await register('market-test-arbitrator-'+str(i)); members[uid]=key - await apply_market(app,root,action='grant',operator='isolated-selftest',subject=uid) - policy=deepcopy(DEFAULT_POLICY); policy.update(id='selftest-panel-v1',candidates=list(members)) - await apply_market(app,root,action='publish',operator='isolated-selftest',policy=policy) - service=await invoke('store.listing_create',{'name':'case-selftest','item_kind':'service', - 'price_minor':1_000_000,'currency_id':'primary','quantity':1,'delivery_mode':'service', - 'escrow_policy':'escrow-v1','dispute_policy':policy['id'],'terms':'Test service dispute.'},bank,bank_id) - service=await invoke('store.listing_update',{'id':service.resources[0].id,'state':'active'},bank,bank_id, - expected=((service.resources[0].id,service.data['generation']),)) - sale=service.data['listing'] - bought=await invoke('orders.buy',{'listing_id':sale['listing_id'],'listing_revision':sale['listing_revision'], - 'quantity':1,'total_price_minor':1_000_000,'currency_id':'primary'},contract_version=3) - opened=await invoke('orders.dispute_open',{'order_id':bought.data['order']['id'],'reason':'quality'}) - case=await invoke('orders.dispute_get',{'case_id':opened.data['case_id']}) - author=opened.data['panel'][0] + key, uid = await register('market-test-arbitrator-' + str(i)) + members[uid] = key + await apply_market(app, root, action='grant', operator='isolated-selftest', subject=uid) + policy = deepcopy(DEFAULT_POLICY) + policy.update(id='selftest-panel-v1', candidates=list(members)) + await apply_market(app, root, action='publish', operator='isolated-selftest', policy=policy) + service = await invoke( + 'store.listing_create', + { + 'name': 'case-selftest', + 'item_kind': 'service', + 'price_minor': 1_000_000, + 'currency_id': 'primary', + 'quantity': 1, + 'delivery_mode': 'service', + 'escrow_policy': 'escrow-v1', + 'dispute_policy': policy['id'], + 'terms': 'Test service dispute.', + }, + bank, + bank_id, + ) + service = await invoke( + 'store.listing_update', + {'id': service.resources[0].id, 'state': 'active'}, + bank, + bank_id, + expected=((service.resources[0].id, service.data['generation']),), + ) + sale = service.data['listing'] + bought = await invoke( + 'orders.buy', + { + 'listing_id': sale['listing_id'], + 'listing_revision': sale['listing_revision'], + 'quantity': 1, + 'total_price_minor': 1_000_000, + 'currency_id': 'primary', + }, + contract_version=3, + ) + opened = await invoke( + 'orders.dispute_open', {'order_id': bought.data['order']['id'], 'reason': 'quality'} + ) + case = await invoke('orders.dispute_get', {'case_id': opened.data['case_id']}) + author = opened.data['panel'][0] async with app.metadata.transaction(write=False) as tx: - parent=tx.one("SELECT id FROM resources WHERE parent=? AND name='files'",(author,))[0] - reason=await invoke('content.file_put',{'parent':parent,'name':'case-reason.txt', - 'media_type':'text/plain','data':b64(b'Selftest evidence warrants a full refund.')},members[author],author) - bound=await invoke('orders.dispute_rationale',{'case_id':opened.data['case_id'], - 'ref':wire(reason.resources[0])},members[author],author) - proposal={**case.data['proposal_base'],'outcome':'refund','refund_minor':1_000_000, - **{name:bound.data[name] for name in ('rationale_ref','rationale_digest')}} + parent = tx.one("SELECT id FROM resources WHERE parent=? AND name='files'", (author,))[0] + reason = await invoke( + 'content.file_put', + { + 'parent': parent, + 'name': 'case-reason.txt', + 'media_type': 'text/plain', + 'data': b64(b'Selftest evidence warrants a full refund.'), + }, + members[author], + author, + ) + bound = await invoke( + 'orders.dispute_rationale', + {'case_id': opened.data['case_id'], 'ref': wire(reason.resources[0])}, + members[author], + author, + ) + proposal = { + **case.data['proposal_base'], + 'outcome': 'refund', + 'refund_minor': 1_000_000, + **{name: bound.data[name] for name in ('rationale_ref', 'rationale_digest')}, + } for uid in opened.data['panel'][:2]: - voted=await invoke('orders.dispute_vote',{'proposal':proposal, - 'signature':wire(members[uid].sign(canonical(proposal),purpose='arbitration-decision'))},members[uid],uid) - args={'case_id':opened.data['case_id'],'decision_id':voted.data['decision']['id']} - await invoke('orders.dispute_execute',args) - await invoke('orders.dispute_execute',args) + voted = await invoke( + 'orders.dispute_vote', + { + 'proposal': proposal, + 'signature': wire( + members[uid].sign(canonical(proposal), purpose='arbitration-decision') + ), + }, + members[uid], + uid, + ) + args = {'case_id': opened.data['case_id'], 'decision_id': voted.data['decision']['id']} + await invoke('orders.dispute_execute', args) + await invoke('orders.dispute_execute', args) async with app.metadata.transaction(write=False) as tx: await inspect_market(tx) - require(_balance(tx,buyer_id)==5_000_000 and _balance(tx,bank_id)==15_000_000 and - _supply(tx)==baseline_supply+20_000_000, - 'market_conservation_failed') - for (escrow,) in tx.rows("SELECT id FROM ledger_accounts WHERE kind IN ('order_escrow','bounty_escrow')"): - require(_balance(tx,escrow)==0,'market_escrow_invariant') + require( + _balance(tx, buyer_id) == 5_000_000 + and _balance(tx, bank_id) == 15_000_000 + and _supply(tx) == baseline_supply + 20_000_000, + 'market_conservation_failed', + ) + for (escrow,) in tx.rows( + "SELECT id FROM ledger_accounts WHERE kind IN ('order_escrow','bounty_escrow')" + ): + require(_balance(tx, escrow) == 0, 'market_escrow_invariant') return True async def inspect_bounty_contracts(app, tx): """Check restored accounting projections against immutable published terms.""" from msg.plugins.bounty import _row, validate_contract + rows = tx.rows('SELECT listing_id FROM bounty_listings ORDER BY listing_id') if not rows: return @@ -178,62 +383,133 @@ async def inspect_bounty_contracts(app, tx): if app.contents is None: # doctor intentionally never opens the application writer stores. from types import SimpleNamespace + from msg.storage.read_only import GitContentReader - reader = SimpleNamespace(contents=GitContentReader( - app.settings.server.content_dir, binary_dir=app.settings.server.blob_dir)) + + reader = SimpleNamespace( + contents=GitContentReader( + app.settings.server.content_dir, binary_dir=app.settings.server.blob_dir + ) + ) for (listing_id,) in rows: await validate_contract(reader, tx, _row(tx, listing_id)) def inspect_clearing(app, tx): """Never migrate, mint, seed a bank, repair a row or unlock the Root key.""" - tables = ('ledger_accounts','money_ledger','money_bank_roles','server_offers', - 'money_purchases','resource_entitlements','bounty_listings','bounty_claims', - 'store_orders','store_deliveries','store_order_events') + tables = ( + 'ledger_accounts', + 'money_ledger', + 'money_bank_roles', + 'server_offers', + 'money_purchases', + 'resource_entitlements', + 'bounty_listings', + 'bounty_claims', + 'store_orders', + 'store_deliveries', + 'store_order_events', + ) for table in tables: - require(tx.one('SELECT to_regclass(?)',(table,))[0] is not None, 'market_schema_missing') - require(tx.one("SELECT 1 FROM information_schema.columns WHERE table_name='money_ledger' " - "AND column_name='entry_key'") is not None, 'market_schema_missing') - balances = {account:int(amount) for account,amount in tx.rows(''' + require(tx.one('SELECT to_regclass(?)', (table,))[0] is not None, 'market_schema_missing') + require( + tx.one( + "SELECT 1 FROM information_schema.columns WHERE table_name='money_ledger' " + "AND column_name='entry_key'" + ) + is not None, + 'market_schema_missing', + ) + balances = { + account: int(amount) + for account, amount in tx.rows(""" SELECT account,SUM(amount) FROM ( SELECT credit_account account,amount_minor amount FROM money_ledger WHERE credit_account IS NOT NULL UNION ALL SELECT debit_account account,-amount_minor amount FROM money_ledger WHERE debit_account IS NOT NULL - ) entries GROUP BY account''')} + ) entries GROUP BY account""") + } supply = _supply(tx) - require(0 <= supply <= MAX_MINOR and all(0 <= n <= MAX_MINOR for n in balances.values()) - and sum(balances.values()) == supply, 'money_conservation_failure') - require(tx.one("SELECT 1 FROM ledger_accounts a JOIN identities i ON i.id=a.id " - "WHERE a.kind<>'subject' LIMIT 1") is None, 'escrow_has_identity') - public = Ed25519Signer.from_bytes((app.settings.service_keys/'receipt.key').read_bytes()).public_key + require( + 0 <= supply <= MAX_MINOR + and all(0 <= n <= MAX_MINOR for n in balances.values()) + and sum(balances.values()) == supply, + 'money_conservation_failure', + ) + require( + tx.one( + 'SELECT 1 FROM ledger_accounts a JOIN identities i ON i.id=a.id ' + "WHERE a.kind<>'subject' LIMIT 1" + ) + is None, + 'escrow_has_identity', + ) + public = Ed25519Signer.from_bytes( + (app.settings.service_keys / 'receipt.key').read_bytes() + ).public_key ledger = {} - for row in tx.rows('''SELECT id,kind,amount_minor,debit_account,credit_account,seq, - policy_version,policy_digest,receipt,actor,request_id,entry_key FROM money_ledger'''): - receipt = loads(row[8]); body = receipt['body'] - expected = dict(zip(('transaction_id','kind','amount_minor','from_subject','to_subject', - 'ledger_sequence','policy_version','policy_digest'),row[:8])) - require(all(body.get(k)==v for k,v in expected.items()), 'money_receipt_mismatch') + for row in tx.rows("""SELECT id,kind,amount_minor,debit_account,credit_account,seq, + policy_version,policy_digest,receipt,actor,request_id,entry_key FROM money_ledger"""): + receipt = loads(row[8]) + body = receipt['body'] + expected = dict( + zip( + ( + 'transaction_id', + 'kind', + 'amount_minor', + 'from_subject', + 'to_subject', + 'ledger_sequence', + 'policy_version', + 'policy_digest', + ), + row[:8], + strict=True, + ) + ) + require(all(body.get(k) == v for k, v in expected.items()), 'money_receipt_mismatch') if row[6] >= 2: - require((body.get('actor'),body.get('request_id'),body.get('entry_key')) == row[9:], - 'money_receipt_mismatch') - verify(public,canonical(body),decode(Signature,receipt['signature']),purpose='money-receipt') + require( + (body.get('actor'), body.get('request_id'), body.get('entry_key')) == row[9:], + 'money_receipt_mismatch', + ) + verify( + public, + canonical(body), + decode(Signature, receipt['signature']), + purpose='money-receipt', + ) ledger[row[0]] = body - for oid,escrow,total,state,quote,refs in tx.rows('''SELECT id,escrow_subject,total_price_minor, - state,quote_digest,receipt_refs FROM store_orders'''): + for ( + oid, + escrow, + total, + state, + quote, + refs, + ) in tx.rows("""SELECT id,escrow_subject,total_price_minor, + state,quote_digest,receipt_refs FROM store_orders"""): require(state != 'accepted', 'incomplete_order_commit') immutable = tx.one('SELECT body FROM order_contracts WHERE order_id=?', (oid,)) if immutable is not None and loads(immutable[0]).get('version') == 5: require(state == 'settled', 'incomplete_entitlement_commit') - expected = total if state in {'funded','delivered','disputed'} else 0 - require(balances.get(escrow,0)==expected, 'order_escrow_mismatch') + expected = total if state in {'funded', 'delivered', 'disputed'} else 0 + require(balances.get(escrow, 0) == expected, 'order_escrow_mismatch') require(all(ref in ledger for ref in loads(refs)), 'order_receipt_missing') if quote is not None: - last=tx.one('SELECT to_state FROM store_order_events WHERE order_id=? ORDER BY seq DESC LIMIT 1',(oid,)) - require(last is not None and last[0]==state, 'order_history_mismatch') - if state=='settled': - delivery=tx.one('SELECT d.state,o.escrow_policy FROM store_deliveries d ' - 'JOIN store_orders o ON o.id=d.order_id WHERE o.id=?',(oid,)) - current=tx.one('SELECT 1 FROM order_contracts WHERE order_id=?',(oid,)) + last = tx.one( + 'SELECT to_state FROM store_order_events WHERE order_id=? ORDER BY seq DESC LIMIT 1', + (oid,), + ) + require(last is not None and last[0] == state, 'order_history_mismatch') + if state == 'settled': + delivery = tx.one( + 'SELECT d.state,o.escrow_policy FROM store_deliveries d ' + 'JOIN store_orders o ON o.id=d.order_id WHERE o.id=?', + (oid,), + ) + current = tx.one('SELECT 1 FROM order_contracts WHERE order_id=?', (oid,)) if current is not None: # V3 managed-instant settlement is a fulfillment/payment fact, # not buyer acknowledgement: its delivery intentionally stays @@ -241,39 +517,73 @@ def inspect_clearing(app, tx): # immutable contract validator instead of weakening this check # for arbitrary prepared legacy orders. from msg.market.policy import contract - locked=contract(tx,oid) + + locked = contract(tx, oid) if locked['version'] == 5: # The contract validator requires the exact entitlement grant; # synchronous issuance must not invent a Delivery or acceptance. require(delivery is None, 'unexpected_entitlement_delivery') else: - prepared=(delivery is not None and delivery[0]=='prepared' and - locked['listing']['delivery_mode']=='managed_instant') - require(delivery is not None and (delivery[0]=='claimed' or prepared), - 'settled_delivery_missing') + prepared = ( + delivery is not None + and delivery[0] == 'prepared' + and locked['listing']['delivery_mode'] == 'managed_instant' + ) + require( + delivery is not None and (delivery[0] == 'claimed' or prepared), + 'settled_delivery_missing', + ) else: - require(delivery is not None and (delivery[0]=='claimed' or - delivery==('prepared','escrow-instant-v1')), 'settled_delivery_missing') - for escrow,total,state,eid in tx.rows('SELECT escrow_account,total_minor,state,entitlement_id FROM money_purchases'): - require(balances.get(escrow,0)==(total if state=='pending' else 0),'purchase_escrow_mismatch') - require((state=='settled') == (eid is not None), 'purchase_entitlement_mismatch') - for lid,escrow,budget,reward,maximum,state in tx.rows('''SELECT listing_id,escrow_subject, - budget_minor,reward_minor,max_claims,state FROM bounty_listings'''): - claims=tx.rows('SELECT claimant,reward_minor,transaction_id,challenge_id FROM bounty_claims WHERE listing_id=?',(lid,)) - require(len(claims)<=maximum, 'bounty_limit_mismatch') - require(balances.get(escrow,0)==(0 if state=='closed' else budget-sum(c[1] for c in claims)), - 'bounty_escrow_mismatch') - for claimant,amount,tid,cid in claims: - receipt=ledger.get(tid,{}) - require(amount==reward and (receipt.get('from_subject'),receipt.get('to_subject'),receipt.get('amount_minor')) - == (escrow,claimant,amount), 'bounty_receipt_mismatch') - challenge=tx.one('SELECT consumed_at FROM bounty_challenges WHERE id=?',(cid,)) + require( + delivery is not None + and (delivery[0] == 'claimed' or delivery == ('prepared', 'escrow-instant-v1')), + 'settled_delivery_missing', + ) + for escrow, total, state, eid in tx.rows( + 'SELECT escrow_account,total_minor,state,entitlement_id FROM money_purchases' + ): + require( + balances.get(escrow, 0) == (total if state == 'pending' else 0), + 'purchase_escrow_mismatch', + ) + require((state == 'settled') == (eid is not None), 'purchase_entitlement_mismatch') + for lid, escrow, budget, reward, maximum, state in tx.rows("""SELECT listing_id,escrow_subject, + budget_minor,reward_minor,max_claims,state FROM bounty_listings"""): + claims = tx.rows( + 'SELECT claimant,reward_minor,transaction_id,challenge_id FROM bounty_claims WHERE listing_id=?', + (lid,), + ) + require(len(claims) <= maximum, 'bounty_limit_mismatch') + require( + balances.get(escrow, 0) + == (0 if state == 'closed' else budget - sum(c[1] for c in claims)), + 'bounty_escrow_mismatch', + ) + for claimant, amount, tid, cid in claims: + receipt = ledger.get(tid, {}) + require( + amount == reward + and ( + receipt.get('from_subject'), + receipt.get('to_subject'), + receipt.get('amount_minor'), + ) + == (escrow, claimant, amount), + 'bounty_receipt_mismatch', + ) + challenge = tx.one('SELECT consumed_at FROM bounty_challenges WHERE id=?', (cid,)) require(challenge is not None and challenge[0] is not None, 'bounty_nonce_mismatch') - return {'currency_id':'primary','scale':6,'total_supply_minor':supply, - 'banks':tx.one("SELECT COUNT(*) FROM money_bank_roles WHERE status='active'")[0], - 'enabled_offers':tx.one('SELECT COUNT(*) FROM server_offers WHERE enabled=TRUE')[0], - 'ledger_entries':len(ledger),'conserved':True,'signed_receipts_verified':True, - 'zero_default':'No mint, bank or offer is created by initialization or inspection.'} + return { + 'currency_id': 'primary', + 'scale': 6, + 'total_supply_minor': supply, + 'banks': tx.one("SELECT COUNT(*) FROM money_bank_roles WHERE status='active'")[0], + 'enabled_offers': tx.one('SELECT COUNT(*) FROM server_offers WHERE enabled=TRUE')[0], + 'ledger_entries': len(ledger), + 'conserved': True, + 'signed_receipts_verified': True, + 'zero_default': 'No mint, bank or offer is created by initialization or inspection.', + } async def check_market_e2e(app, root, call, register): @@ -282,131 +592,283 @@ async def check_market_e2e(app, root, call, register): Production MoneyAdmin still requires an OS-root physical console and hidden PIN. This fixture cannot be selected by configuration or a network request. """ - require(app.selftest_run_id is not None and app.settings.service_url=='https://selftest.invalid', - 'isolated_market_selftest_required') + require( + app.selftest_run_id is not None and app.settings.service_url == 'https://selftest.invalid', + 'isolated_market_selftest_required', + ) from msg.admin.money import _confirmed_money from msg.daemon import parser + async with app.metadata.transaction(write=False) as tx: - baseline=inspect_clearing(app,tx) - require(baseline['total_supply_minor']==0,'selftest_requires_zero_supply') - bank_key,bank=await register('bank-test') - buyer_key,buyer=await register('market-buyer') + baseline = inspect_clearing(app, tx) + require(baseline['total_supply_minor'] == 0, 'selftest_requires_zero_supply') + bank_key, bank = await register('bank-test') + buyer_key, buyer = await register('market-buyer') async def command(argv, expected_actions): - args=parser().parse_args(argv) - action='bank_fund' if args.money_command=='bank' and args.bank_command=='fund' else args.money_command - previews=[]; approvals=[] - def emit(value): previews.append(loads(value)) + args = parser().parse_args(argv) + action = ( + 'bank_fund' + if args.money_command == 'bank' and args.bank_command == 'fund' + else args.money_command + ) + previews = [] + approvals = [] + + def emit(value): + previews.append(loads(value)) + def confirm(prompt): - approval='CONFIRM MONEY '+digest(previews[-1]) - require(prompt=='Type '+approval+' to continue: ', 'selftest_approval_unbound') + approval = 'CONFIRM MONEY ' + digest(previews[-1]) + require(prompt == 'Type ' + approval + ' to continue: ', 'selftest_approval_unbound') approvals.append(previews[-1]['approval_for']) return approval + def unlock(): - require(approvals==expected_actions, 'selftest_unlock_before_confirmation') + require(approvals == expected_actions, 'selftest_unlock_before_confirmation') return root # Fresh disposable Test Root; never opens production key material. - return await _confirmed_money(app,action,operator='isolated-TestConsole',amount=args.amount, - subject_id=getattr(args,'subject_id',None),emit=emit,confirm=confirm,unlock=unlock) - - await command(['money','mint','20'], ['mint']) - funding=await command(['money','bank','fund','@bank-test','20'], ['bank_add','transfer']) - require(len(funding['audit_event_ids'])==2 and funding['root_balance_minor']==0, 'selftest_bank_fund_failed') - async def checked(name,args,key=None,subject=None,**kwargs): - result=await call(name,args,key,subject,**kwargs) - require(result.status=='ok','market_selftest_'+name.replace('.','_') + - (':' + result.error.code if result.error else '')) + + return await _confirmed_money( + app, + action, + operator='isolated-TestConsole', + amount=args.amount, + subject_id=getattr(args, 'subject_id', None), + emit=emit, + confirm=confirm, + unlock=unlock, + ) + + await command(['money', 'mint', '20'], ['mint']) + funding = await command(['money', 'bank', 'fund', '@bank-test', '20'], ['bank_add', 'transfer']) + require( + len(funding['audit_event_ids']) == 2 and funding['root_balance_minor'] == 0, + 'selftest_bank_fund_failed', + ) + + async def checked(name, args, key=None, subject=None, **kwargs): + result = await call(name, args, key, subject, **kwargs) + require( + result.status == 'ok', + 'market_selftest_' + + name.replace('.', '_') + + (':' + result.error.code if result.error else ''), + ) return result - bounty=await checked('bounty.create',{'name':'signature-reward', - 'terms':'Control of the current IdentityKey only, not human or Sybil proof.', - 'reward_minor':10_000_000,'budget_minor':10_000_000,'max_claims':1, - 'claim_limit_per_subject':1},bank_key,bank) - lid=bounty.data['bounty']['listing_id'] - challenge=await checked('bounty.challenge',{'listing_id':lid},buyer_key,buyer) - proof=challenge.data['challenge'] - claimed=await checked('bounty.claim',{'challenge_id':proof['challenge_id'], - 'proof':wire(buyer_key.sign(canonical(proof),purpose='bounty-pop-v1'))},buyer_key,buyer) - require(claimed.data['reward']['body']['amount_minor']==10_000_000 and - claimed.data['escrow_balance_minor']==0,'selftest_bounty_not_paid') - listing=await checked('store.listing_create',{'name':'test-delivery','item_kind':'bundle', - 'price_minor':5_000_000,'currency_id':'primary','quantity':1, - 'delivery_mode':'managed_instant','escrow_policy':'escrow-v1', - 'dispute_policy':'dispute-v1','terms':'Fixed text and file; immediate in-site delivery.'},bank_key,bank) + + bounty = await checked( + 'bounty.create', + { + 'name': 'signature-reward', + 'terms': 'Control of the current IdentityKey only, not human or Sybil proof.', + 'reward_minor': 10_000_000, + 'budget_minor': 10_000_000, + 'max_claims': 1, + 'claim_limit_per_subject': 1, + }, + bank_key, + bank, + ) + lid = bounty.data['bounty']['listing_id'] + challenge = await checked('bounty.challenge', {'listing_id': lid}, buyer_key, buyer) + proof = challenge.data['challenge'] + claimed = await checked( + 'bounty.claim', + { + 'challenge_id': proof['challenge_id'], + 'proof': wire(buyer_key.sign(canonical(proof), purpose='bounty-pop-v1')), + }, + buyer_key, + buyer, + ) + require( + claimed.data['reward']['body']['amount_minor'] == 10_000_000 + and claimed.data['escrow_balance_minor'] == 0, + 'selftest_bounty_not_paid', + ) + listing = await checked( + 'store.listing_create', + { + 'name': 'test-delivery', + 'item_kind': 'bundle', + 'price_minor': 5_000_000, + 'currency_id': 'primary', + 'quantity': 1, + 'delivery_mode': 'managed_instant', + 'escrow_policy': 'escrow-v1', + 'dispute_policy': 'dispute-v1', + 'terms': 'Fixed text and file; immediate in-site delivery.', + }, + bank_key, + bank, + ) async with app.metadata.transaction(write=False) as tx: - files=await tx.resolve((await tx.path(bank))+'/files') - payload=await checked('content.file_put',{'parent':files,'name':'hello.txt','data':b64(b'delivery-ok\n'), - 'media_type':'text/plain'},bank_key,bank) - listing_id=listing.resources[0].id - package=await checked('store.package_deposit',{'listing_id':listing_id, - 'listing_revision':listing.resources[0].revision,'manifest':{'text':'msg.lmm.best store selftest'}, - 'payload_refs':[wire(payload.resources[0])]},bank_key,bank) - active=await checked('store.listing_update',{'id':listing_id,'state':'active','package_ref':package.data['package']['id']}, - bank_key,bank,expected=((listing_id,listing.data['generation']),)) - quote=active.data['listing'] - args={'listing_id':listing_id,'listing_revision':quote['listing_revision'],'quantity':1, - 'currency_id':'primary','total_price_minor':5_000_000} + files = await tx.resolve((await tx.path(bank)) + '/files') + payload = await checked( + 'content.file_put', + { + 'parent': files, + 'name': 'hello.txt', + 'data': b64(b'delivery-ok\n'), + 'media_type': 'text/plain', + }, + bank_key, + bank, + ) + listing_id = listing.resources[0].id + package = await checked( + 'store.package_deposit', + { + 'listing_id': listing_id, + 'listing_revision': listing.resources[0].revision, + 'manifest': {'text': 'msg.lmm.best store selftest'}, + 'payload_refs': [wire(payload.resources[0])], + }, + bank_key, + bank, + ) + active = await checked( + 'store.listing_update', + {'id': listing_id, 'state': 'active', 'package_ref': package.data['package']['id']}, + bank_key, + bank, + expected=((listing_id, listing.data['generation']),), + ) + quote = active.data['listing'] + args = { + 'listing_id': listing_id, + 'listing_revision': quote['listing_revision'], + 'quantity': 1, + 'currency_id': 'primary', + 'total_price_minor': 5_000_000, + } # No seller request occurs after publishing. The signed buyer request alone # validates the immutable deposit and delivers; payment remains in escrow. - bought=await checked('orders.buy',args,buyer_key,buyer,request_id='market-e2e-buy', - contract_version=4) - require(bought.data['order']['state']=='delivered','selftest_order_not_delivered') - oid=bought.data['order']['id'] + bought = await checked( + 'orders.buy', args, buyer_key, buyer, request_id='market-e2e-buy', contract_version=4 + ) + require(bought.data['order']['state'] == 'delivered', 'selftest_order_not_delivered') + oid = bought.data['order']['id'] async with app.metadata.transaction(write=False) as tx: - escrow=tx.one('SELECT escrow_subject FROM store_orders WHERE id=?',(oid,))[0] - require((_balance(tx,buyer),_balance(tx,bank),_balance(tx,escrow)) == - (5_000_000,10_000_000,5_000_000),'selftest_awaiting_acceptance_balance_mismatch') - delivery=await checked('delivery.get',{'order_id':oid},buyer_key,buyer,contract_version=2) - content=delivery.data['delivery'] - require(content['manifest']=={'text':'msg.lmm.best store selftest'} and - content['payloads'][0]['data']==b64(b'delivery-ok\n') and - content['package_digest']==package.data['package']['digest'],'selftest_delivery_mismatch') - repeated=await checked('orders.buy',args,buyer_key,buyer,request_id='market-e2e-buy', - contract_version=4) - require(repeated.replayed and repeated.data==bought.data,'selftest_market_replay_failed') - require(content['state']=='prepared','selftest_false_buyer_ack') - accepted=await checked('delivery.accept',{'order_id':oid, - 'delivery_digest':content['delivery_digest']},buyer_key,buyer, - contract_version=2,request_id='market-e2e-accept') - require(accepted.data['state']=='settled','selftest_order_not_settled') + escrow = tx.one('SELECT escrow_subject FROM store_orders WHERE id=?', (oid,))[0] + require( + (_balance(tx, buyer), _balance(tx, bank), _balance(tx, escrow)) + == (5_000_000, 10_000_000, 5_000_000), + 'selftest_awaiting_acceptance_balance_mismatch', + ) + delivery = await checked( + 'delivery.get', {'order_id': oid}, buyer_key, buyer, contract_version=2 + ) + content = delivery.data['delivery'] + require( + content['manifest'] == {'text': 'msg.lmm.best store selftest'} + and content['payloads'][0]['data'] == b64(b'delivery-ok\n') + and content['package_digest'] == package.data['package']['digest'], + 'selftest_delivery_mismatch', + ) + repeated = await checked( + 'orders.buy', args, buyer_key, buyer, request_id='market-e2e-buy', contract_version=4 + ) + require(repeated.replayed and repeated.data == bought.data, 'selftest_market_replay_failed') + require(content['state'] == 'prepared', 'selftest_false_buyer_ack') + accepted = await checked( + 'delivery.accept', + {'order_id': oid, 'delivery_digest': content['delivery_digest']}, + buyer_key, + buyer, + contract_version=2, + request_id='market-e2e-accept', + ) + require(accepted.data['state'] == 'settled', 'selftest_order_not_settled') async with app.metadata.transaction(write=False) as tx: - require(_balance(tx,escrow)==0,'selftest_escrow_not_empty') - buyer_balance=await checked('money.balance',{},buyer_key,buyer) - bank_balance=await checked('money.balance',{},bank_key,bank) - require(buyer_balance.data['balance_minor']==5_000_000 and bank_balance.data['balance_minor']==15_000_000, - 'selftest_market_balance_mismatch') + require(_balance(tx, escrow) == 0, 'selftest_escrow_not_empty') + buyer_balance = await checked('money.balance', {}, buyer_key, buyer) + bank_balance = await checked('money.balance', {}, bank_key, bank) + require( + buyer_balance.data['balance_minor'] == 5_000_000 + and bank_balance.data['balance_minor'] == 15_000_000, + 'selftest_market_balance_mismatch', + ) async with app.metadata.transaction(write=False) as tx: - checked_state=inspect_clearing(app,tx) - require(checked_state['total_supply_minor']==20_000_000 and - checked_state['banks']==baseline['banks']+1 and - checked_state['enabled_offers']==baseline['enabled_offers'], - 'selftest_market_supply_mismatch') + checked_state = inspect_clearing(app, tx) + require( + checked_state['total_supply_minor'] == 20_000_000 + and checked_state['banks'] == baseline['banks'] + 1 + and checked_state['enabled_offers'] == baseline['enabled_offers'], + 'selftest_market_supply_mismatch', + ) # The 20 -> 10 -> 5 buyer checkpoint above remains the official PoP/store # acceptance flow. Separately exercise an explicit synchronous grant intent. from msg.admin.money import apply_offer from msg.client_market import entitlement_intent from msg.plugins.hosting_capacity import ENTITLEMENT_KIND, extra_capacity - await apply_offer(app, root, action='set', operator='isolated-TestConsole', - offer_id='selftest-byte-grant', fields={'resource_kind': 'website', 'unit': 'byte', - 'price_minor': 1, 'min_quantity': 1, 'max_quantity': 1, - 'entitlement_kind': ENTITLEMENT_KIND, 'duration_seconds': None}) + + await apply_offer( + app, + root, + action='set', + operator='isolated-TestConsole', + offer_id='selftest-byte-grant', + fields={ + 'resource_kind': 'website', + 'unit': 'byte', + 'price_minor': 1, + 'min_quantity': 1, + 'max_quantity': 1, + 'entitlement_kind': ENTITLEMENT_KIND, + 'duration_seconds': None, + }, + ) + class BuyerClient: async def call(self, name, params, **options): return await checked(name, params, buyer_key, buyer, **options) - intent = await entitlement_intent(BuyerClient(), {'offer_id': 'selftest-byte-grant', 'quantity': 1}) - grant = await checked('money.redeem', intent, buyer_key, buyer, - contract_version=3, request_id='market-e2e-entitlement') - repeated = await checked('money.redeem', intent, buyer_key, buyer, - contract_version=3, request_id='market-e2e-entitlement') + + intent = await entitlement_intent( + BuyerClient(), {'offer_id': 'selftest-byte-grant', 'quantity': 1} + ) + grant = await checked( + 'money.redeem', + intent, + buyer_key, + buyer, + contract_version=3, + request_id='market-e2e-entitlement', + ) + repeated = await checked( + 'money.redeem', + intent, + buyer_key, + buyer, + contract_version=3, + request_id='market-e2e-entitlement', + ) order = grant.data['order'] - require(order['state'] == 'settled' and order['contract_version'] == 5 and - order['delivered_at'] is None and repeated.replayed and repeated.data == grant.data, - 'selftest_entitlement_order_failed') + require( + order['state'] == 'settled' + and order['contract_version'] == 5 + and order['delivered_at'] is None + and repeated.replayed + and repeated.data == grant.data, + 'selftest_entitlement_order_failed', + ) async with app.metadata.transaction(write=False) as tx: locked = contract(tx, order['id']) - require(extra_capacity(tx, buyer, app.clock()) == 1 and - (_balance(tx, buyer), _balance(tx, bank), _balance(tx, ROOT_SUBJECT), - _balance(tx, locked['escrow_subject'])) == (4_999_999, 15_000_000, 1, 0) and - tx.one('SELECT COUNT(*) FROM store_deliveries WHERE order_id=?', (order['id'],))[0] == 0, - 'selftest_entitlement_grant_failed') - require(inspect_clearing(app, tx)['total_supply_minor'] == 20_000_000, - 'selftest_entitlement_supply_mismatch') + require( + extra_capacity(tx, buyer, app.clock()) == 1 + and ( + _balance(tx, buyer), + _balance(tx, bank), + _balance(tx, ROOT_SUBJECT), + _balance(tx, locked['escrow_subject']), + ) + == (4_999_999, 15_000_000, 1, 0) + and tx.one('SELECT COUNT(*) FROM store_deliveries WHERE order_id=?', (order['id'],))[0] + == 0, + 'selftest_entitlement_grant_failed', + ) + require( + inspect_clearing(app, tx)['total_supply_minor'] == 20_000_000, + 'selftest_entitlement_supply_mismatch', + ) return True diff --git a/src/msg/admin/money.py b/src/msg/admin/money.py index 9ea4556d..53e9b3f2 100644 --- a/src/msg/admin/money.py +++ b/src/msg/admin/money.py @@ -1,4 +1,5 @@ """Physical-console central bank operations. Never register these as network ops.""" + from __future__ import annotations import asyncio @@ -12,55 +13,75 @@ from msg.core.codec import canonical, digest, loads, wire from msg.core.errors import require from msg.core.models import AuditEvent, Event, ResourceRef -from msg.plugins.money import ( - CURRENCY_ID, - MAX_MINOR, - SCALE, - _balance, - _post_entry, - _supply, -) +from msg.market.offer_resources import OFFER_COLUMNS as _OFFER_COLUMNS +from msg.plugins.money import CURRENCY_ID, MAX_MINOR, SCALE, _balance, _post_entry, _supply from msg.plugins.offers import validate_local_offer from msg.security.crypto import Ed25519Signer, open_private_key def parse_amount(value: str) -> int: """Parse a displayed MSG amount without binary floats or silent rounding.""" - require(isinstance(value, str) and re.fullmatch(r'(?:0|[1-9][0-9]*)(?:\.[0-9]{1,6})?', value), - 'invalid_money_amount') - minor = int(Decimal(value) * (10 ** SCALE)) + require( + isinstance(value, str) and re.fullmatch(r'(?:0|[1-9][0-9]*)(?:\.[0-9]{1,6})?', value), + 'invalid_money_amount', + ) + minor = int(Decimal(value) * (10**SCALE)) require(0 < minor <= MAX_MINOR, 'invalid_money_amount') return minor def _root_signer(app, signer): - require(isinstance(signer, Ed25519Signer) and - signer.public_key == app.certificates.root_public_key, 'root_key_mismatch') + require( + isinstance(signer, Ed25519Signer) and signer.public_key == app.certificates.root_public_key, + 'root_key_mismatch', + ) -from msg.market.offer_resources import OFFER_COLUMNS as _OFFER_COLUMNS - _UNSET = object() def _offer_snapshot(tx, offer_id): - row = tx.one('''SELECT offer_id,resource_kind,unit,price_minor,min_quantity,max_quantity, + row = tx.one( + """SELECT offer_id,resource_kind,unit,price_minor,min_quantity,max_quantity, entitlement_kind,duration_seconds,enabled,price_revision - FROM server_offers WHERE offer_id=?''', (offer_id,)) - return dict(zip(_OFFER_COLUMNS, row)) if row else None + FROM server_offers WHERE offer_id=?""", + (offer_id,), + ) + return dict(zip(_OFFER_COLUMNS, row, strict=True)) if row else None -async def apply_offer(app, signer, *, action: str, operator: str, offer_id: str, - fields: dict | None = None, expected_offer=_UNSET): +async def apply_offer( + app, + signer, + *, + action: str, + operator: str, + offer_id: str, + fields: dict | None = None, + expected_offer=_UNSET, +): """Root-signed local offer mutation; set remains Registry/fulfiller gated.""" _root_signer(app, signer) require(action in {'set', 'disable'}, 'invalid_offer_action') - require(isinstance(offer_id, str) and - re.fullmatch(r'[a-zA-Z][a-zA-Z0-9_-]{0,79}', offer_id), 'invalid_offer_id') + require( + isinstance(offer_id, str) and re.fullmatch(r'[a-zA-Z][a-zA-Z0-9_-]{0,79}', offer_id), + 'invalid_offer_id', + ) if action == 'set': - require(isinstance(fields, dict) and set(fields) == { - 'resource_kind', 'unit', 'price_minor', 'min_quantity', - 'max_quantity', 'entitlement_kind', 'duration_seconds'}, 'invalid_offer_fields') + require( + isinstance(fields, dict) + and set(fields) + == { + 'resource_kind', + 'unit', + 'price_minor', + 'min_quantity', + 'max_quantity', + 'entitlement_kind', + 'duration_seconds', + }, + 'invalid_offer_fields', + ) validate_local_offer(app, **fields) else: require(fields is None, 'invalid_offer_fields') @@ -69,6 +90,7 @@ async def apply_offer(app, signer, *, action: str, operator: str, offer_id: str, async with app.metadata.transaction(write=True) as tx: previous = _offer_snapshot(tx, offer_id) from msg.market.offer_resources import authoritative_offer, write_offer_revision + source = await authoritative_offer(app, tx, offer_id) if source is not None: require(source == previous, 'offer_projection_mismatch') @@ -77,7 +99,8 @@ async def apply_offer(app, signer, *, action: str, operator: str, offer_id: str, if action == 'set': validate_local_offer(app, **fields) revision = 'price_' + uuid4().hex - tx.execute('''INSERT INTO server_offers + tx.execute( + """INSERT INTO server_offers (offer_id,resource_kind,unit,price_minor,min_quantity,max_quantity, entitlement_kind,duration_seconds,enabled,price_revision) VALUES (?,?,?,?,?,?,?,?,TRUE,?) @@ -86,51 +109,95 @@ async def apply_offer(app, signer, *, action: str, operator: str, offer_id: str, price_minor=excluded.price_minor,min_quantity=excluded.min_quantity, max_quantity=excluded.max_quantity,entitlement_kind=excluded.entitlement_kind, duration_seconds=excluded.duration_seconds,enabled=TRUE, - price_revision=excluded.price_revision''', - (offer_id, fields['resource_kind'], fields['unit'], fields['price_minor'], - fields['min_quantity'], fields['max_quantity'], fields['entitlement_kind'], - fields['duration_seconds'], revision), write=True) + price_revision=excluded.price_revision""", + ( + offer_id, + fields['resource_kind'], + fields['unit'], + fields['price_minor'], + fields['min_quantity'], + fields['max_quantity'], + fields['entitlement_kind'], + fields['duration_seconds'], + revision, + ), + write=True, + ) else: require(previous is not None, 'offer_not_found') require(previous['enabled'], 'offer_already_disabled') - tx.execute('UPDATE server_offers SET enabled=FALSE WHERE offer_id=?', - (offer_id,), write=True) + tx.execute( + 'UPDATE server_offers SET enabled=FALSE WHERE offer_id=?', (offer_id,), write=True + ) current = _offer_snapshot(tx, offer_id) if previous is None or source is not None: - await write_offer_revision(app, tx, signer, current, now=now, - request_id=request_id, new=previous is None) - statement = {'action': 'offer_' + action, 'offer_id': offer_id, - 'before': previous, 'after': current, 'operator': operator, - 'request_id': request_id, 'time': wire(now)} + await write_offer_revision( + app, tx, signer, current, now=now, request_id=request_id, new=previous is None + ) + statement = { + 'action': 'offer_' + action, + 'offer_id': offer_id, + 'before': previous, + 'after': current, + 'operator': operator, + 'request_id': request_id, + 'time': wire(now), + } signature = wire(signer.sign(canonical(statement), purpose='money-admin')) - event = Event(id='audit_' + uuid4().hex, type='root.money.offer.' + action, - time=now, request_id=request_id, actor=ROOT_SUBJECT, - subject=ROOT_SUBJECT, resources=(ResourceRef(id=ROOT_SUBJECT),), - data={**statement, 'root_signature': signature}) + event = Event( + id='audit_' + uuid4().hex, + type='root.money.offer.' + action, + time=now, + request_id=request_id, + actor=ROOT_SUBJECT, + subject=ROOT_SUBJECT, + resources=(ResourceRef(id=ROOT_SUBJECT),), + data={**statement, 'root_signature': signature}, + ) await tx.append_event(event) - await tx.append_audit(AuditEvent(event=event, - authority=(ResourceRef(id=app.certificates.root_certificate.resource_id),), - before_digest=digest(previous), after_digest=digest(current), - previous_digest=None, entry_digest='', result='offer_' + action)) + await tx.append_audit( + AuditEvent( + event=event, + authority=(ResourceRef(id=app.certificates.root_certificate.resource_id),), + before_digest=digest(previous), + after_digest=digest(current), + previous_digest=None, + entry_digest='', + result='offer_' + action, + ) + ) return {'action': action, 'offer': current, 'audit_event_id': event.id} -async def apply_money(app, signer, *, action: str, operator: str, amount_minor: int | None = None, - subject_id: str | None = None, expected_state: dict | None = None): +async def apply_money( + app, + signer, + *, + action: str, + operator: str, + amount_minor: int | None = None, + subject_id: str | None = None, + expected_state: dict | None = None, +): """Internal local use case; callers must establish the OS-console boundary. Each invocation creates a fresh local request id. PG serializes all writers, so balance and supply checks are re-evaluated within the committing transaction. """ _root_signer(app, signer) - require(action in {'mint', 'burn', 'transfer', 'bank_add', 'bank_remove', 'bank_fund'}, 'invalid_money_action') + require( + action in {'mint', 'burn', 'transfer', 'bank_add', 'bank_remove', 'bank_fund'}, + 'invalid_money_action', + ) if action in {'mint', 'burn', 'transfer', 'bank_fund'}: require(type(amount_minor) is int and 0 < amount_minor <= MAX_MINOR, 'invalid_money_amount') else: require(amount_minor is None, 'invalid_money_amount') if action in {'transfer', 'bank_add', 'bank_remove', 'bank_fund'}: - require(isinstance(subject_id, str) and subject_id not in {ROOT_SUBJECT, '@root'}, - 'invalid_money_recipient') + require( + isinstance(subject_id, str) and subject_id not in {ROOT_SUBJECT, '@root'}, + 'invalid_money_recipient', + ) else: require(subject_id is None, 'invalid_money_recipient') @@ -139,35 +206,56 @@ async def apply_money(app, signer, *, action: str, operator: str, amount_minor: async with app.metadata.transaction(write=True) as tx: if subject_id is not None: subject = await tx.subject(subject_id) - require(subject.kind in {'registered', 'custodial'} and not subject.local_only, - 'invalid_money_recipient') + require( + subject.kind in {'registered', 'custodial'} and not subject.local_only, + 'invalid_money_recipient', + ) before_supply = _supply(tx) before_root = _balance(tx, ROOT_SUBJECT) - row = (tx.one('SELECT status FROM money_bank_roles WHERE subject_id=?', (subject_id,)) - if action.startswith('bank_') else None) + row = ( + tx.one('SELECT status FROM money_bank_roles WHERE subject_id=?', (subject_id,)) + if action.startswith('bank_') + else None + ) before_role = row[0] if row else None if expected_state is not None: - current = {'supply': before_supply, 'root': before_root, - 'subject': _balance(tx, subject_id) if subject_id else None, - 'role': before_role} + current = { + 'supply': before_supply, + 'root': before_root, + 'subject': _balance(tx, subject_id) if subject_id else None, + 'role': before_role, + } require(current == expected_state, 'money_preview_stale') if action.startswith('bank_'): - require(action == 'bank_fund' or - (action == 'bank_add' and before_role != 'active') or - (action == 'bank_remove' and before_role == 'active'), 'bank_role_unchanged') + require( + action == 'bank_fund' + or (action == 'bank_add' and before_role != 'active') + or (action == 'bank_remove' and before_role == 'active'), + 'bank_role_unchanged', + ) if action in {'bank_add', 'bank_fund'}: - tx.execute('''INSERT INTO money_bank_roles(subject_id,status,granted_at,granted_by,revoked_at) + tx.execute( + """INSERT INTO money_bank_roles(subject_id,status,granted_at,granted_by,revoked_at) VALUES (?,?,?,?,NULL) ON CONFLICT(subject_id) DO UPDATE SET status='active',granted_at=excluded.granted_at,granted_by=excluded.granted_by, - revoked_at=NULL''', (subject_id, 'active', wire(now), ROOT_SUBJECT), write=True) + revoked_at=NULL""", + (subject_id, 'active', wire(now), ROOT_SUBJECT), + write=True, + ) else: - tx.execute("UPDATE money_bank_roles SET status='revoked',revoked_at=? WHERE subject_id=?", - (wire(now), subject_id), write=True) + tx.execute( + "UPDATE money_bank_roles SET status='revoked',revoked_at=? WHERE subject_id=?", + (wire(now), subject_id), + write=True, + ) receipt = None if action not in {'bank_add', 'bank_remove'}: if action == 'mint': - require(before_supply + amount_minor <= MAX_MINOR and - before_root + amount_minor <= MAX_MINOR, 'money_overflow') + require( + before_supply + amount_minor <= MAX_MINOR + and before_root + amount_minor <= MAX_MINOR, + 'money_overflow', + ) debit, credit, kind = None, ROOT_SUBJECT, 'mint' elif action == 'burn': require(before_root >= amount_minor, 'insufficient_funds') @@ -176,49 +264,87 @@ async def apply_money(app, signer, *, action: str, operator: str, amount_minor: require(before_root >= amount_minor, 'insufficient_funds') require(_balance(tx, subject_id) + amount_minor <= MAX_MINOR, 'money_overflow') debit, credit, kind = ROOT_SUBJECT, subject_id, 'transfer' - receipt = _post_entry(tx, sender=debit, recipient=credit, - amount=amount_minor, actor=ROOT_SUBJECT, request_id=request_id, - now=now, receipt_signer=app.receipt_signer, kind=kind, _local_issuer=True) - statement = {'action': action, 'subject_id': subject_id, 'amount_minor': amount_minor, - 'currency_id': CURRENCY_ID, 'operator': operator, 'request_id': request_id, - 'time': wire(now), 'receipt_digest': digest(receipt) if receipt else None, - 'before_supply_minor': before_supply, 'after_supply_minor': _supply(tx), - 'before_root_minor': before_root, 'after_root_minor': _balance(tx, ROOT_SUBJECT), - 'before_role': before_role} + receipt = _post_entry( + tx, + sender=debit, + recipient=credit, + amount=amount_minor, + actor=ROOT_SUBJECT, + request_id=request_id, + now=now, + receipt_signer=app.receipt_signer, + kind=kind, + _local_issuer=True, + ) + statement = { + 'action': action, + 'subject_id': subject_id, + 'amount_minor': amount_minor, + 'currency_id': CURRENCY_ID, + 'operator': operator, + 'request_id': request_id, + 'time': wire(now), + 'receipt_digest': digest(receipt) if receipt else None, + 'before_supply_minor': before_supply, + 'after_supply_minor': _supply(tx), + 'before_root_minor': before_root, + 'after_root_minor': _balance(tx, ROOT_SUBJECT), + 'before_role': before_role, + } audit_ids = [] actions = ('bank_add', 'transfer') if action == 'bank_fund' else (action,) for audit_action in actions: fact = {**statement, 'action': audit_action, 'combined_action': action} proof = wire(signer.sign(canonical(fact), purpose='money-admin')) - event = Event(id='audit_' + uuid4().hex, type='root.money.' + audit_action, - time=now, request_id=request_id, actor=ROOT_SUBJECT, - subject=ROOT_SUBJECT, - resources=(ResourceRef(id=subject_id or ROOT_SUBJECT),), - data={**fact, 'root_signature': proof}) + event = Event( + id='audit_' + uuid4().hex, + type='root.money.' + audit_action, + time=now, + request_id=request_id, + actor=ROOT_SUBJECT, + subject=ROOT_SUBJECT, + resources=(ResourceRef(id=subject_id or ROOT_SUBJECT),), + data={**fact, 'root_signature': proof}, + ) await tx.append_event(event) - await tx.append_audit(AuditEvent(event=event, - authority=(ResourceRef(id=app.certificates.root_certificate.resource_id),), - before_digest=digest({'supply': before_supply, 'root': before_root, 'role': before_role}), - after_digest=digest(fact), previous_digest=None, entry_digest='', result=audit_action)) + await tx.append_audit( + AuditEvent( + event=event, + authority=(ResourceRef(id=app.certificates.root_certificate.resource_id),), + before_digest=digest({ + 'supply': before_supply, + 'root': before_root, + 'role': before_role, + }), + after_digest=digest(fact), + previous_digest=None, + entry_digest='', + result=audit_action, + ) + ) audit_ids.append(event.id) - return {'action': action, 'receipt': receipt, 'audit_event_id': audit_ids[-1], - 'audit_event_ids': audit_ids, - 'root_balance_minor': statement['after_root_minor'], - 'total_supply_minor': statement['after_supply_minor'], - 'subject_id': subject_id} + return { + 'action': action, + 'receipt': receipt, + 'audit_event_id': audit_ids[-1], + 'audit_event_ids': audit_ids, + 'root_balance_minor': statement['after_root_minor'], + 'total_supply_minor': statement['after_supply_minor'], + 'subject_id': subject_id, + } - -async def _confirmed_money(app, action, *, operator, amount, subject_id, - emit, confirm, unlock): +async def _confirmed_money(app, action, *, operator, amount, subject_id, emit, confirm, unlock): """The command's approval pipeline, shared with an isolated TestConsole. The production caller must establish the physical-console boundary first. There is no network operation, config option or command flag selecting IO adapters. The diagnostic replaces only IO/unlock on its fresh Test Root. """ - require(action in {'mint','burn','transfer','bank_add','bank_remove','bank_fund'}, - 'invalid_money_action') + require( + action in {'mint', 'burn', 'transfer', 'bank_add', 'bank_remove', 'bank_fund'}, + 'invalid_money_action', + ) amount_minor = parse_amount(amount) if amount is not None else None resolved_subject = subject_id async with app.metadata.transaction(write=False) as tx: @@ -227,41 +353,71 @@ async def _confirmed_money(app, action, *, operator, amount, subject_id, resolved_subject = await tx.resolve(namespace + '/' + subject_id) if resolved_subject is not None: subject = await tx.subject(resolved_subject) - require(subject.kind in {'registered','custodial'} and not subject.local_only, - 'invalid_money_recipient') + require( + subject.kind in {'registered', 'custodial'} and not subject.local_only, + 'invalid_money_recipient', + ) root_before = _balance(tx, ROOT_SUBJECT) supply_before = _supply(tx) subject_before = _balance(tx, resolved_subject) if resolved_subject else None - role_row = (tx.one('SELECT status FROM money_bank_roles WHERE subject_id=?', - (resolved_subject,)) if action.startswith('bank_') else None) + role_row = ( + tx.one('SELECT status FROM money_bank_roles WHERE subject_id=?', (resolved_subject,)) + if action.startswith('bank_') + else None + ) role_before = role_row[0] if role_row else None - root_delta = (amount_minor if action == 'mint' else - -amount_minor if action in {'burn','transfer','bank_fund'} else 0) - supply_delta = (amount_minor if action == 'mint' else - -amount_minor if action == 'burn' else 0) - preview = {'action':action, 'amount_minor':amount_minor, 'currency_id':CURRENCY_ID, - 'subject_id':resolved_subject, - 'subject_handle':subject_id if subject_id != resolved_subject else None, - 'root_balance_before_minor':root_before, 'root_balance_after_minor':root_before+root_delta, - 'total_supply_before_minor':supply_before, 'total_supply_after_minor':supply_before+supply_delta, - 'recipient_balance_before_minor':subject_before, - 'recipient_balance_after_minor':subject_before+amount_minor - if action in {'transfer','bank_fund'} else subject_before, - 'bank_role_before':role_before, 'bank_role_after':'active' - if action in {'bank_add','bank_fund'} else 'revoked' if action=='bank_remove' else role_before} + root_delta = ( + amount_minor + if action == 'mint' + else -amount_minor + if action in {'burn', 'transfer', 'bank_fund'} + else 0 + ) + supply_delta = amount_minor if action == 'mint' else -amount_minor if action == 'burn' else 0 + preview = { + 'action': action, + 'amount_minor': amount_minor, + 'currency_id': CURRENCY_ID, + 'subject_id': resolved_subject, + 'subject_handle': subject_id if subject_id != resolved_subject else None, + 'root_balance_before_minor': root_before, + 'root_balance_after_minor': root_before + root_delta, + 'total_supply_before_minor': supply_before, + 'total_supply_after_minor': supply_before + supply_delta, + 'recipient_balance_before_minor': subject_before, + 'recipient_balance_after_minor': subject_before + amount_minor + if action in {'transfer', 'bank_fund'} + else subject_before, + 'bank_role_before': role_before, + 'bank_role_after': 'active' + if action in {'bank_add', 'bank_fund'} + else 'revoked' + if action == 'bank_remove' + else role_before, + } # Distinct confirmations, bound to one before/after snapshot. Unlock occurs # only after BOTH role and funding approvals, before the atomic use case. - approvals = ('bank_add','transfer') if action == 'bank_fund' else (action,) + approvals = ('bank_add', 'transfer') if action == 'bank_fund' else (action,) for approved_action in approvals: - part = {**preview, 'approval_for':approved_action} + part = {**preview, 'approval_for': approved_action} emit(canonical(part).decode()) # Escaped JSON, never terminal control text. approval = 'CONFIRM MONEY ' + digest(part) require(confirm('Type ' + approval + ' to continue: ') == approval, 'approval_cancelled') signer = unlock() - return await apply_money(app, signer, action=action, operator=operator, - amount_minor=amount_minor, subject_id=resolved_subject, - expected_state={'supply':supply_before, 'root':root_before, - 'subject':subject_before, 'role':role_before}) + return await apply_money( + app, + signer, + action=action, + operator=operator, + amount_minor=amount_minor, + subject_id=resolved_subject, + expected_state={ + 'supply': supply_before, + 'root': root_before, + 'subject': subject_before, + 'role': role_before, + }, + ) class MoneyAdmin: @@ -271,6 +427,7 @@ def __init__(self, config_dir): def execute(self, action, *, amount=None, subject_id=None): operator = require_local_console(self.config_dir) from msg.admin.root import RootAdmin + app = RootAdmin(self.config_dir)._app() def unlock(): @@ -281,43 +438,71 @@ def unlock(): async def run(): await app.load() try: - return await _confirmed_money(app, action, operator=operator, amount=amount, - subject_id=subject_id, emit=print, confirm=input, unlock=unlock) + return await _confirmed_money( + app, + action, + operator=operator, + amount=amount, + subject_id=subject_id, + emit=print, + confirm=input, + unlock=unlock, + ) finally: await app.close() + return asyncio.run(run()) def execute_offer(self, action, *, offer_id, fields=None, dry_run=False): operator = require_local_console(self.config_dir) require(action in {'set', 'disable', 'import'}, 'invalid_offer_action') require(not dry_run or action == 'import', 'invalid_offer_action') - require(isinstance(offer_id, str) and - re.fullmatch(r'[a-zA-Z][a-zA-Z0-9_-]{0,79}', offer_id), 'invalid_offer_id') + require( + isinstance(offer_id, str) and re.fullmatch(r'[a-zA-Z][a-zA-Z0-9_-]{0,79}', offer_id), + 'invalid_offer_id', + ) from msg.admin.root import RootAdmin + app = RootAdmin(self.config_dir)._app() async def run(): await app.load() try: if action == 'import': - from msg.admin.offer_import import preview_import, apply_import + from msg.admin.offer_import import apply_import, preview_import + async with app.metadata.transaction(write=False) as tx: plan = await preview_import(app, tx, offer_id) print(canonical(plan).decode()) if dry_run: return {'plan': plan, 'approval_digest': digest(plan)} approval = 'CONFIRM MONEY ' + digest(plan) - require(input('Type ' + approval + ' to continue: ') == approval, - 'approval_cancelled') + require( + input('Type ' + approval + ' to continue: ') == approval, + 'approval_cancelled', + ) pin = getpass.getpass('Root PIN/passphrase: ') - private = open_private_key(loads(root_envelope(self.config_dir).read_bytes()), pin) - return await apply_import(app, Ed25519Signer.from_bytes(private), plan, - operator=operator) + private = open_private_key( + loads(root_envelope(self.config_dir).read_bytes()), pin + ) + return await apply_import( + app, Ed25519Signer.from_bytes(private), plan, operator=operator + ) if action == 'set': - require(isinstance(fields, dict) and set(fields) == { - 'resource_kind', 'unit', 'price', 'min_quantity', - 'max_quantity', 'entitlement_kind', 'duration_seconds'}, - 'invalid_offer_fields') + require( + isinstance(fields, dict) + and set(fields) + == { + 'resource_kind', + 'unit', + 'price', + 'min_quantity', + 'max_quantity', + 'entitlement_kind', + 'duration_seconds', + }, + 'invalid_offer_fields', + ) fields['price_minor'] = parse_amount(fields.pop('price')) validate_local_offer(app, **fields) async with app.metadata.transaction(write=False) as tx: @@ -325,19 +510,30 @@ async def run(): if action == 'disable': require(previous is not None, 'offer_not_found') require(previous['enabled'], 'offer_already_disabled') - preview = {'action': 'offer_' + action, 'offer_id': offer_id, - 'before': previous, 'proposed': fields if action == 'set' else - {**previous, 'enabled': False}} + preview = { + 'action': 'offer_' + action, + 'offer_id': offer_id, + 'before': previous, + 'proposed': fields if action == 'set' else {**previous, 'enabled': False}, + } print(canonical(preview).decode()) approval = 'CONFIRM MONEY ' + digest(preview) - require(input('Type ' + approval + ' to continue: ') == approval, - 'approval_cancelled') + require( + input('Type ' + approval + ' to continue: ') == approval, 'approval_cancelled' + ) pin = getpass.getpass('Root PIN/passphrase: ') private = open_private_key(loads(root_envelope(self.config_dir).read_bytes()), pin) signer = Ed25519Signer.from_bytes(private) - return await apply_offer(app, signer, action=action, operator=operator, - offer_id=offer_id, fields=fields, - expected_offer=previous) + return await apply_offer( + app, + signer, + action=action, + operator=operator, + offer_id=offer_id, + fields=fields, + expected_offer=previous, + ) finally: await app.close() + return asyncio.run(run()) diff --git a/src/msg/admin/offer_import.py b/src/msg/admin/offer_import.py index 63d2f262..2377241b 100644 --- a/src/msg/admin/offer_import.py +++ b/src/msg/admin/offer_import.py @@ -1,4 +1,5 @@ """Explicit local adoption of a legacy offer; no financial rows are rewritten.""" + import re from uuid import uuid4 @@ -12,24 +13,53 @@ async def preview_import(app, tx, offer_id, *, revision_id=None): - require(isinstance(offer_id, str) and re.fullmatch(r'[a-zA-Z][a-zA-Z0-9_-]{0,79}', offer_id), - 'invalid_offer_id') + require( + isinstance(offer_id, str) and re.fullmatch(r'[a-zA-Z][a-zA-Z0-9_-]{0,79}', offer_id), + 'invalid_offer_id', + ) require(mapped_listing(tx, offer_id) is None, 'offer_already_resource') - row = tx.one('SELECT ' + ','.join(OFFER_COLUMNS) + ' FROM server_offers WHERE offer_id=?', (offer_id,)) + row = tx.one( + 'SELECT ' + ','.join(OFFER_COLUMNS) + ' FROM server_offers WHERE offer_id=?', (offer_id,) + ) require(row is not None, 'offer_not_found') - offer = dict(zip(OFFER_COLUMNS, row)) - validate_local_offer(app, **{k: offer[k] for k in ('resource_kind', 'unit', 'price_minor', - 'min_quantity', 'max_quantity', 'entitlement_kind', 'duration_seconds')}) - require(tx.one('SELECT 1 FROM resources WHERE id=? OR (parent=? AND name=?)', - (offer_id, 't_store', offer_id)) is None, 'offer_resource_conflict') + offer = dict(zip(OFFER_COLUMNS, row, strict=True)) + validate_local_offer( + app, + **{ + k: offer[k] + for k in ( + 'resource_kind', + 'unit', + 'price_minor', + 'min_quantity', + 'max_quantity', + 'entitlement_kind', + 'duration_seconds', + ) + }, + ) + require( + tx.one( + 'SELECT 1 FROM resources WHERE id=? OR (parent=? AND name=?)', + (offer_id, 't_store', offer_id), + ) + is None, + 'offer_resource_conflict', + ) revision_id = revision_id or 'v_' + uuid4().hex - require(isinstance(revision_id, str) and re.fullmatch(r'v_[a-f0-9]{32}', revision_id), - 'invalid_revision_id') - require(tx.one('SELECT 1 FROM revisions WHERE id=?', (revision_id,)) is None, - 'offer_revision_conflict') + require( + isinstance(revision_id, str) and re.fullmatch(r'v_[a-f0-9]{32}', revision_id), + 'invalid_revision_id', + ) + require( + tx.one('SELECT 1 FROM revisions WHERE id=?', (revision_id,)) is None, + 'offer_revision_conflict', + ) # The original serialized snapshots and receipt bytes are hashed unchanged. purchases, transactions, grants = [], {}, set() - for pid, owner, encoded in tx.rows('SELECT id,subject_id,offer_snapshot FROM money_purchases ORDER BY id'): + for pid, owner, encoded in tx.rows( + 'SELECT id,subject_id,offer_snapshot FROM money_purchases ORDER BY id' + ): snapshot = loads(encoded) require(isinstance(snapshot, dict), 'legacy_purchase_invalid') if snapshot.get('offer_id') != offer_id: @@ -39,24 +69,54 @@ async def preview_import(app, tx, offer_id, *, revision_id=None): purchases.append(tx.one('SELECT * FROM money_purchases WHERE id=?', (pid,))) for index, tid in enumerate(order['receipt_refs']): kind, reference = tx.one('SELECT kind,reference FROM money_ledger WHERE id=?', (tid,)) - expected_kind = 'transfer' if index == 0 else 'redeem' if purchase['state'] == 'settled' else 'refund' - expected_reference = ('purchase_fund:' if index == 0 else 'purchase_settle:' - if purchase['state'] == 'settled' else 'purchase_refund:') + pid - require((kind, reference) == (expected_kind, expected_reference), 'offer_import_inconsistent_ledger') + expected_kind = ( + 'transfer' + if index == 0 + else 'redeem' + if purchase['state'] == 'settled' + else 'refund' + ) + expected_reference = ( + 'purchase_fund:' + if index == 0 + else 'purchase_settle:' + if purchase['state'] == 'settled' + else 'purchase_refund:' + ) + pid + require( + (kind, reference) == (expected_kind, expected_reference), + 'offer_import_inconsistent_ledger', + ) transactions[tid] = tx.one('SELECT * FROM money_ledger WHERE id=?', (tid,)) if purchase['entitlement_id'] is not None: - grant = tx.one('SELECT offer_id,quantity,entitlement_kind FROM resource_entitlements WHERE id=?', - (purchase['entitlement_id'],)) - require(grant == (offer_id, purchase['quantity'], snapshot['entitlement_kind']), - 'offer_import_inconsistent_entitlements') + grant = tx.one( + 'SELECT offer_id,quantity,entitlement_kind FROM resource_entitlements WHERE id=?', + (purchase['entitlement_id'],), + ) + require( + grant == (offer_id, purchase['quantity'], snapshot['entitlement_kind']), + 'offer_import_inconsistent_entitlements', + ) grants.add(purchase['entitlement_id']) - entitlements = tx.rows('SELECT * FROM resource_entitlements WHERE offer_id=? ORDER BY id', (offer_id,)) + entitlements = tx.rows( + 'SELECT * FROM resource_entitlements WHERE offer_id=? ORDER BY id', (offer_id,) + ) require({row[0] for row in entitlements} == grants, 'offer_import_inconsistent_entitlements') - return {'action': 'offer_import', 'version': 1, 'offer_id': offer_id, - 'before': offer, 'listing_id': offer_id, 'revision_id': revision_id, - 'purchase_count': len(purchases), 'entitlement_count': len(entitlements), - 'dependencies_digest': digest({'purchases': purchases, - 'transactions': transactions, 'entitlements': entitlements})} + return { + 'action': 'offer_import', + 'version': 1, + 'offer_id': offer_id, + 'before': offer, + 'listing_id': offer_id, + 'revision_id': revision_id, + 'purchase_count': len(purchases), + 'entitlement_count': len(entitlements), + 'dependencies_digest': digest({ + 'purchases': purchases, + 'transactions': transactions, + 'entitlements': entitlements, + }), + } async def apply_import(app, signer, plan, *, operator): @@ -66,21 +126,53 @@ async def apply_import(app, signer, plan, *, operator): current = await preview_import(app, tx, plan['offer_id'], revision_id=plan['revision_id']) require(current == plan, 'offer_import_preview_stale') approved_digest = digest(plan) - await write_offer_revision(app, tx, signer, current['before'], now=now, - request_id=request_id, new=True, revision_id=plan['revision_id'], - source_digest=approved_digest) - statement = {'action': 'offer_import', 'plan': current, 'approved_digest': approved_digest, - 'operator': operator, 'request_id': request_id, 'time': wire(now)} + await write_offer_revision( + app, + tx, + signer, + current['before'], + now=now, + request_id=request_id, + new=True, + revision_id=plan['revision_id'], + source_digest=approved_digest, + ) + statement = { + 'action': 'offer_import', + 'plan': current, + 'approved_digest': approved_digest, + 'operator': operator, + 'request_id': request_id, + 'time': wire(now), + } signature = wire(signer.sign(canonical(statement), purpose='money-admin')) - event = Event(id='audit_' + uuid4().hex, type='root.money.offer.import', time=now, - request_id=request_id, actor=ROOT_SUBJECT, subject=ROOT_SUBJECT, + event = Event( + id='audit_' + uuid4().hex, + type='root.money.offer.import', + time=now, + request_id=request_id, + actor=ROOT_SUBJECT, + subject=ROOT_SUBJECT, resources=(ResourceRef(id=plan['listing_id'], revision=plan['revision_id']),), - data={**statement, 'root_signature': signature}) + data={**statement, 'root_signature': signature}, + ) await tx.append_event(event) - await tx.append_audit(AuditEvent(event=event, - authority=(ResourceRef(id=app.certificates.root_certificate.resource_id),), - before_digest=digest(plan['before']), after_digest=digest(plan['before']), - previous_digest=None, entry_digest='', result='offer_import')) - return {'action': 'offer_import', 'offer_id': plan['offer_id'], 'listing_id': plan['listing_id'], - 'revision_id': plan['revision_id'], 'approved_digest': approved_digest, - 'audit_event_id': event.id} + await tx.append_audit( + AuditEvent( + event=event, + authority=(ResourceRef(id=app.certificates.root_certificate.resource_id),), + before_digest=digest(plan['before']), + after_digest=digest(plan['before']), + previous_digest=None, + entry_digest='', + result='offer_import', + ) + ) + return { + 'action': 'offer_import', + 'offer_id': plan['offer_id'], + 'listing_id': plan['listing_id'], + 'revision_id': plan['revision_id'], + 'approved_digest': approved_digest, + 'audit_event_id': event.id, + } diff --git a/src/msg/admin/preflight.py b/src/msg/admin/preflight.py index eae60c37..db2335b3 100644 --- a/src/msg/admin/preflight.py +++ b/src/msg/admin/preflight.py @@ -4,6 +4,7 @@ no DDL, migration, filesystem repair, Root unlock, effect emission or sequence allocation. It never supplies missing operational evidence with fixture results. """ + from __future__ import annotations import argparse @@ -23,21 +24,43 @@ ('ingress_log_chain', '#64', 'CDN/proxy/APM/nginx/service redaction and listener evidence'), ('protected_snapshot', '#65/#84', 'independent provenance and digest of the field backup'), ('restore_conservation', '#65/#84', 'isolated old/new ledger and signed-history comparison'), - ('current_checkpoint', '#69', 'independently current signed revocation head outside the rollback set'), + ( + 'current_checkpoint', + '#69', + 'independently current signed revocation head outside the rollback set', + ), ('physical_console', '#70', 'real authorized local VT/serial console and rejection evidence'), ('ca_transition', '#70', 'finite old CA inventory and individually approved reissue/revoke'), - ('bank_fund_boundary', '#70/B', 'joint role versus injection gate evidence; no actual funding here'), - ('capacity_and_failure_drill', '#84', 'measured capacity and isolated install/crash/rollback drill'), + ( + 'bank_fund_boundary', + '#70/B', + 'joint role versus injection gate evidence; no actual funding here', + ), + ( + 'capacity_and_failure_drill', + '#84', + 'measured capacity and isolated install/crash/rollback drill', + ), ) def inspect_database(connection): - require(connection.execute('SHOW transaction_read_only').fetchone() == ('on',), - 'preflight_read_only_required') - tables = {row[0] for row in connection.execute( - 'SELECT tablename FROM pg_tables WHERE schemaname=current_schema()')} - result = {'read_only': True, 'state': 'empty' if not tables else 'unclassified', - 'table_count': len(tables), 'server_version': connection.info.server_version} + require( + connection.execute('SHOW transaction_read_only').fetchone() == ('on',), + 'preflight_read_only_required', + ) + tables = { + row[0] + for row in connection.execute( + 'SELECT tablename FROM pg_tables WHERE schemaname=current_schema()' + ) + } + result = { + 'read_only': True, + 'state': 'empty' if not tables else 'unclassified', + 'table_count': len(tables), + 'server_version': connection.info.server_version, + } if not tables: return result required = {'money_ledger', 'money_accounts', 'store_orders', 'bounty_listings', 'identities'} @@ -47,11 +70,19 @@ def inspect_database(connection): _validate_identity_references, _validate_ledger, ) + escrow = _escrow_sources(connection) - legacy = sum(connection.execute('SELECT 1 FROM identities WHERE id=%s', (key,)).fetchone() - is not None for key in escrow) - ledger = {'escrow_accounts': len(escrow), 'legacy_escrow_identities': legacy, - 'checks': 'blocked', 'migration_performed': False} + legacy = sum( + connection.execute('SELECT 1 FROM identities WHERE id=%s', (key,)).fetchone() + is not None + for key in escrow + ) + ledger = { + 'escrow_accounts': len(escrow), + 'legacy_escrow_identities': legacy, + 'checks': 'blocked', + 'migration_performed': False, + } result['state'] = 'typed_ledger' if 'ledger_accounts' in tables else 'legacy_ledger' try: _validate_identity_references(connection, escrow) @@ -77,54 +108,76 @@ def inspect_database(connection): ledger.update(rows=count, history_digest='sha256:' + hasher.hexdigest()) result['ledger'] = ledger if 'settings' in tables: - gate = connection.execute("SELECT value FROM settings WHERE key='recovery_quarantine'").fetchone() + gate = connection.execute( + "SELECT value FROM settings WHERE key='recovery_quarantine'" + ).fetchone() result['quarantine_present'] = gate is not None - row = connection.execute("SELECT value FROM settings WHERE key='active_root_certificate'").fetchone() + row = connection.execute( + "SELECT value FROM settings WHERE key='active_root_certificate'" + ).fetchone() result['active_root_certificate'] = loads(row[0]) if row else None if 'certificates' in tables: inventory = [] for identifier, subject, parent, revoked, body in connection.execute( - 'SELECT id,subject,parent,revoked,body FROM certificates ORDER BY id'): + 'SELECT id,subject,parent,revoked,body FROM certificates ORDER BY id' + ): certificate = decode(Certificate, loads(body)) - require((certificate.resource_id, certificate.subject_id, certificate.parent_certificate_id) == - (identifier, subject, parent), 'preflight_certificate_mismatch') + require( + (certificate.resource_id, certificate.subject_id, certificate.parent_certificate_id) + == (identifier, subject, parent), + 'preflight_certificate_mismatch', + ) if certificate.kind != 'ca': continue policy = certificate.issuance grants = (*certificate.grants, *(policy.issue_grants if policy is not None else ())) operations = sorted({operation for grant in grants for operation in grant.operations}) - inventory.append({'id': identifier, 'subject': subject, 'parent': parent, - 'revoked': bool(revoked), 'key_id': certificate.key_id, + inventory.append({ + 'id': identifier, + 'subject': subject, + 'parent': parent, + 'revoked': bool(revoked), + 'key_id': certificate.key_id, 'signed_grants_digest': digest(certificate.grants), - 'issuance_digest': digest(policy), 'operation_count': len(operations), + 'issuance_digest': digest(policy), + 'operation_count': len(operations), 'explicit_operations': all('*' not in operation for operation in operations), 'max_child_ca_depth': policy.max_child_ca_depth if policy is not None else None, 'max_cert_ttl_seconds': policy.max_cert_ttl_seconds if policy is not None else None, - 'next_step': 'explicit_operator_review_only'}) + 'next_step': 'explicit_operator_review_only', + }) result['ca_inventory'] = inventory return result def preflight(config_dir): """The config directory is explicit; no accidental default production probe.""" - result = {'format': 'msg-preflight-v1', 'captured_at': datetime.now(UTC).isoformat(), - 'decision': 'blocked', 'source_kind': 'unattested', 'mutation_performed': False, - 'root_private_material': 'not_opened', 'outbound_effects': 'not_run', - 'field_evidence': [{'id': key, 'issue': issue, 'required': description, 'status': 'missing'} - for key, issue, description in _FIELD_EVIDENCE]} + result = { + 'format': 'msg-preflight-v1', + 'captured_at': datetime.now(UTC).isoformat(), + 'decision': 'blocked', + 'source_kind': 'unattested', + 'mutation_performed': False, + 'root_private_material': 'not_opened', + 'outbound_effects': 'not_run', + 'field_evidence': [ + {'id': key, 'issue': issue, 'required': description, 'status': 'missing'} + for key, issue, description in _FIELD_EVIDENCE + ], + } try: settings = load_settings(Path(config_dir)) config = settings.config_dir / 'msgd.toml' if not config.is_file(): config = settings.config_dir / 'server.toml' result['config_digest'] = digest(config.read_bytes()) - except (OSError, Failure, ValueError, KeyError): + except OSError, Failure, ValueError, KeyError: return dict(result, error='preflight_configuration_unavailable') try: with psycopg.connect(settings.server.postgres_dsn, connect_timeout=5) as connection: connection.execute('SET TRANSACTION ISOLATION LEVEL REPEATABLE READ, READ ONLY') result['database'] = inspect_database(connection) - except (psycopg.Error, Failure, ValueError, KeyError): + except psycopg.Error, Failure, ValueError, KeyError: result['error'] = 'preflight_database_unavailable_or_inconsistent' return result diff --git a/src/msg/admin/recovery_policy.py b/src/msg/admin/recovery_policy.py index 8ec3bb36..1bd4a80f 100644 --- a/src/msg/admin/recovery_policy.py +++ b/src/msg/admin/recovery_policy.py @@ -1,16 +1,27 @@ """Typed policy restrictions for pinned recovery v2; never a promotion contract.""" -from dataclasses import replace + import re +from dataclasses import replace from msg.constants import ROOT_SUBJECT from msg.core.codec import decode, digest, loads, parse_time from msg.core.errors import require from msg.core.models import Certificate -FACTS = frozenset({'resource.acl.restrict', 'topic.policy.restrict', 'topic_ban.set', 'certificate.current', 'resource.authority.reconcile'}) -DOMAINS = {'resource.acl.restrict': 'resource_acl', 'topic.policy.restrict': 'topic_policy', - 'topic_ban.set': 'topic_ban', 'certificate.current': 'certificates', - 'resource.authority.reconcile': 'resource_authority'} +FACTS = frozenset({ + 'resource.acl.restrict', + 'topic.policy.restrict', + 'topic_ban.set', + 'certificate.current', + 'resource.authority.reconcile', +}) +DOMAINS = { + 'resource.acl.restrict': 'resource_acl', + 'topic.policy.restrict': 'topic_policy', + 'topic_ban.set': 'topic_ban', + 'certificate.current': 'certificates', + 'resource.authority.reconcile': 'resource_authority', +} def validate(fact): @@ -19,31 +30,55 @@ def validate(fact): if kind == 'resource.authority.reconcile': require(set(value) == {'previous', 'current'}, 'recovery_checkpoint_invalid') for state in value.values(): - require(type(state) is dict and set(state) == {'owner', 'group', 'parent'} and - all(type(state[k]) is str and 0 < len(state[k]) <= 256 for k in ('owner', 'group')) and - (state['parent'] is None or - (type(state['parent']) is str and 0 < len(state['parent']) <= 256)), - 'recovery_checkpoint_invalid') + require( + type(state) is dict + and set(state) == {'owner', 'group', 'parent'} + and all( + type(state[k]) is str and 0 < len(state[k]) <= 256 for k in ('owner', 'group') + ) + and ( + state['parent'] is None + or (type(state['parent']) is str and 0 < len(state['parent']) <= 256) + ), + 'recovery_checkpoint_invalid', + ) require(fact['subject'] == value['previous']['owner'], 'recovery_checkpoint_invalid') elif kind == 'certificate.current': - require(set(value) == {'body_digest'} and - (value['body_digest'] is None or - (type(value['body_digest']) is str and - re.fullmatch(r'sha256:[0-9a-f]{64}', value['body_digest']) is not None)), - 'recovery_checkpoint_invalid') + require( + set(value) == {'body_digest'} + and ( + value['body_digest'] is None + or ( + type(value['body_digest']) is str + and re.fullmatch(r'sha256:[0-9a-f]{64}', value['body_digest']) is not None + ) + ), + 'recovery_checkpoint_invalid', + ) elif kind == 'resource.acl.restrict': - require(set(value) == {'owner', 'group', 'mode'} and - all(type(value[k]) is str and 0 < len(value[k]) <= 256 for k in ('owner', 'group')) and - type(value['mode']) is int and 0 <= value['mode'] <= 0o777, - 'recovery_checkpoint_invalid') + require( + set(value) == {'owner', 'group', 'mode'} + and all(type(value[k]) is str and 0 < len(value[k]) <= 256 for k in ('owner', 'group')) + and type(value['mode']) is int + and 0 <= value['mode'] <= 0o777, + 'recovery_checkpoint_invalid', + ) elif kind == 'topic.policy.restrict': - require(set(value) == {'membership_policy'} and - value['membership_policy'] in ('open', 'approval', 'invite', 'closed'), - 'recovery_checkpoint_invalid') + require( + set(value) == {'membership_policy'} + and value['membership_policy'] in ('open', 'approval', 'invite', 'closed'), + 'recovery_checkpoint_invalid', + ) else: - require(kind == 'topic_ban.set' and set(value) == {'expires_at'} and - (value['expires_at'] is None or parse_time(value['expires_at']) > parse_time(fact['at'])), - 'recovery_checkpoint_invalid') + require( + kind == 'topic_ban.set' + and set(value) == {'expires_at'} + and ( + value['expires_at'] is None + or parse_time(value['expires_at']) > parse_time(fact['at']) + ), + 'recovery_checkpoint_invalid', + ) async def apply(tx, fact): @@ -53,8 +88,10 @@ async def apply(tx, fact): row = tx.one('SELECT subject,revoked,body FROM certificates WHERE id=?', (target,)) require(row is not None, 'recovery_fact_missing') original = decode(Certificate, loads(row[2])) - require(row[0] == original.subject_id == subject and original.resource_id == target, - 'recovery_fact_subject_mismatch') + require( + row[0] == original.subject_id == subject and original.resource_id == target, + 'recovery_fact_subject_mismatch', + ) # The exact signed body includes grants, constraints, issuance policy, # issuer/parent/source bindings and validity. Never rewrite it to invent # a constrained signature, or reactivate an already revoked certificate. @@ -65,6 +102,7 @@ async def apply(tx, fact): resource = await tx.resource(target) if kind == 'resource.authority.reconcile': from msg.security.quarantine import active + require(active(tx), 'recovery_quarantine_required') previous, current = value['previous'], value['current'] actual = {field: getattr(resource, field) for field in ('owner', 'group', 'parent')} @@ -74,27 +112,45 @@ async def apply(tx, fact): if current['parent'] is not None: require(current['parent'] != target, 'parent_cycle') await tx.resource(current['parent']) - require(all(item.id != target for item in await tx.ancestors(current['parent'])), 'parent_cycle') + require( + all(item.id != target for item in await tx.ancestors(current['parent'])), + 'parent_cycle', + ) if actual == current: return False # These are staged current facts, not authority to reopen the service. # Old and new scope-derived rights remain blocked globally until a later # complete, independently verified authority reconstruction is possible. - await tx.replace(replace(resource, **current, generation=resource.generation + 1, - modified_at=max(resource.modified_at, parse_time(fact['at'])), - modified_by=ROOT_SUBJECT), resource.generation) + await tx.replace( + replace( + resource, + **current, + generation=resource.generation + 1, + modified_at=max(resource.modified_at, parse_time(fact['at'])), + modified_by=ROOT_SUBJECT, + ), + resource.generation, + ) return True require(resource.owner == subject, 'recovery_fact_subject_mismatch') if kind == 'resource.acl.restrict': - require(resource.owner == value['owner'] and resource.group == value['group'], - 'recovery_policy_authority_change_unsupported') + require( + resource.owner == value['owner'] and resource.group == value['group'], + 'recovery_policy_authority_change_unsupported', + ) mode = (resource.mode & 0o7000) | (resource.mode & value['mode']) if resource.mode == mode: return False - await tx.replace(replace(resource, mode=mode, generation=resource.generation + 1, - modified_at=max(resource.modified_at, parse_time(fact['at'])), - modified_by=ROOT_SUBJECT), - resource.generation) + await tx.replace( + replace( + resource, + mode=mode, + generation=resource.generation + 1, + modified_at=max(resource.modified_at, parse_time(fact['at'])), + modified_by=ROOT_SUBJECT, + ), + resource.generation, + ) return True require(resource.type == 'topic', 'not_a_topic') if kind == 'topic.policy.restrict': @@ -110,8 +166,11 @@ async def apply(tx, fact): desired = 'closed' if previous == desired: return False - tx.execute('INSERT INTO topic_settings (topic,membership_policy) VALUES (?,?) ' - 'ON CONFLICT(topic) DO UPDATE SET membership_policy=excluded.membership_policy', - (target, desired), write=True) + tx.execute( + 'INSERT INTO topic_settings (topic,membership_policy) VALUES (?,?) ' + 'ON CONFLICT(topic) DO UPDATE SET membership_policy=excluded.membership_policy', + (target, desired), + write=True, + ) return True raise AssertionError('topic_ban.set uses the existing replay membership-removal path') diff --git a/src/msg/admin/recovery_proof.py b/src/msg/admin/recovery_proof.py index fe68809c..6447447a 100644 --- a/src/msg/admin/recovery_proof.py +++ b/src/msg/admin/recovery_proof.py @@ -1,14 +1,15 @@ """Strict complete-state recovery proof; physical local promotion is a separate ceremony.""" + from dataclasses import dataclass, replace from datetime import timedelta from uuid import uuid4 -from msg.admin.recovery_state import snapshot, runtime_controls, _runtime_row -from msg.constants import ROOT_SUBJECT, ONLINE_CA -from msg.core.codec import canonical, decode, digest, loads, parse_time, wire, unb64 +from msg.admin.recovery_state import _runtime_row, runtime_controls, snapshot +from msg.constants import ONLINE_CA, ROOT_SUBJECT +from msg.core.codec import canonical, decode, digest, loads, parse_time, unb64, wire from msg.core.errors import Failure, require from msg.core.models import AuditEvent, Certificate, Event, ResourceRef, Signature -from msg.security.crypto import verify, Ed25519Signer +from msg.security.crypto import Ed25519Signer, verify from msg.security.quarantine import SETTING, active FORMAT = 'msg-complete-recovery-state-v1' @@ -28,27 +29,64 @@ class IndependentRecoveryPin: def verify_proof(packet, pin, now): require(isinstance(pin, IndependentRecoveryPin), 'recovery_independent_pin_required') try: - require(type(packet) is dict and set(packet) == {'state', 'signature'}, 'recovery_proof_invalid') + require( + type(packet) is dict and set(packet) == {'state', 'signature'}, 'recovery_proof_invalid' + ) body = packet['state'] - require(type(body) is dict and set(body) == {'format', 'service', 'source_backup_sha256', - 'sequence', 'issued_at', 'expires_at', 'runtime_controls', 'metadata', 'files'}, 'recovery_proof_invalid') - require(body['format'] == FORMAT and body['service'] == pin.service and - body['source_backup_sha256'] == pin.source_backup_sha256 and - type(pin.public_key) is bytes and len(pin.public_key) == 32 and - type(pin.sequence) is int and pin.sequence >= 1 and type(body['sequence']) is int and - body['sequence'] == pin.sequence and digest(body) == pin.digest, - 'recovery_proof_pin_mismatch') + require( + type(body) is dict + and set(body) + == { + 'format', + 'service', + 'source_backup_sha256', + 'sequence', + 'issued_at', + 'expires_at', + 'runtime_controls', + 'metadata', + 'files', + }, + 'recovery_proof_invalid', + ) + require( + body['format'] == FORMAT + and body['service'] == pin.service + and body['source_backup_sha256'] == pin.source_backup_sha256 + and type(pin.public_key) is bytes + and len(pin.public_key) == 32 + and type(pin.sequence) is int + and pin.sequence >= 1 + and type(body['sequence']) is int + and body['sequence'] == pin.sequence + and digest(body) == pin.digest, + 'recovery_proof_pin_mismatch', + ) controls = body['runtime_controls'] - require(type(controls) is dict and set(controls) == {'present', 'fields'} and - type(controls['present']) is bool and type(controls['fields']) is dict and - set(controls['fields']) == {'accept_writes', 'cleanup_enabled'}, 'recovery_proof_invalid') + require( + type(controls) is dict + and set(controls) == {'present', 'fields'} + and type(controls['present']) is bool + and type(controls['fields']) is dict + and set(controls['fields']) == {'accept_writes', 'cleanup_enabled'}, + 'recovery_proof_invalid', + ) for field in controls['fields'].values(): - require(type(field) is dict and set(field) == {'present', 'value'} and - type(field['present']) is bool and type(field['value']) is bool, 'recovery_proof_invalid') - require(type(body['metadata']) is dict and type(body['files']) is dict, 'recovery_proof_invalid') + require( + type(field) is dict + and set(field) == {'present', 'value'} + and type(field['present']) is bool + and type(field['value']) is bool, + 'recovery_proof_invalid', + ) + require( + type(body['metadata']) is dict and type(body['files']) is dict, 'recovery_proof_invalid' + ) start, end = parse_time(body['issued_at']), parse_time(body['expires_at']) - require(start <= now < end and end-start <= timedelta(hours=24), 'recovery_proof_expired') - verify(pin.public_key, canonical(body), decode(Signature, packet['signature']), purpose=PURPOSE) + require(start <= now < end and end - start <= timedelta(hours=24), 'recovery_proof_expired') + verify( + pin.public_key, canonical(body), decode(Signature, packet['signature']), purpose=PURPOSE + ) return loads(canonical(body)) except (KeyError, TypeError, ValueError, OverflowError) as exc: raise Failure('recovery_proof_invalid') from exc @@ -56,10 +94,16 @@ def verify_proof(packet, pin, now): async def verify_current_authority(app, tx, public_key): from msg.security.backup_retirement import root_verifier - require(await root_verifier(tx, now=app.clock()) == public_key == app.certificates.root_public_key, - 'recovery_root_mismatch') + + require( + await root_verifier(tx, now=app.clock()) == public_key == app.certificates.root_public_key, + 'recovery_root_mismatch', + ) generation = tx.rows("SELECT value FROM settings WHERE key='recovery_runtime_generation'") - require(not generation or (type(loads(generation[0][0])) is str and bool(loads(generation[0][0]))), 'recovery_runtime_invalid') + require( + not generation or (type(loads(generation[0][0])) is str and bool(loads(generation[0][0]))), + 'recovery_runtime_invalid', + ) root = await tx.subject(ROOT_SUBJECT) require(root.local_only, 'root_policy_corrupt') for (identifier,) in tx.rows('SELECT id FROM certificates WHERE revoked=0 ORDER BY id'): @@ -71,17 +115,23 @@ async def verify_current_authority(app, tx, public_key): online = tx.setting('online_ca_certificate') require(type(online) is str, 'recovery_online_issuer_missing') certificate = await app.certificates.validate(online, tx) - require(certificate.subject_id == ONLINE_CA and certificate.key_id == app.online_signer.key_id, - 'recovery_online_key_mismatch') + require( + certificate.subject_id == ONLINE_CA and certificate.key_id == app.online_signer.key_id, + 'recovery_online_key_mismatch', + ) # The receipt signer has a registered credential, independent of online CA. receipt = tx.setting('receipt_public_key') - require(type(receipt) is dict and receipt.get('key_id') == app.receipt_signer.key_id and - unb64(receipt.get('public_key', ''), limit=32) == app.receipt_signer.public_key, - 'recovery_receipt_key_mismatch') + require( + type(receipt) is dict + and receipt.get('key_id') == app.receipt_signer.key_id + and unb64(receipt.get('public_key', ''), limit=32) == app.receipt_signer.public_key, + 'recovery_receipt_key_mismatch', + ) for (identifier,) in tx.rows('SELECT id FROM resources ORDER BY id'): await tx.ancestors(identifier) from msg.admin.market_check import inspect_market from msg.storage.ledger_migration import _validate_ledger + _validate_ledger(tx._connection) await inspect_market(tx) @@ -92,165 +142,348 @@ def configuration_state(settings): for key in ('listen', 'port'): value.pop(key) value['hosting_recovery_marker'] = settings.hosting_recovery_marker is not None - for key in ('config_dir', 'postgres_dsn', 'valkey_url', 'mail', 'content_dir', - 'repositories_dir', 'blob_dir', 'staging_dir', 'service_keys_dir'): + for key in ( + 'config_dir', + 'postgres_dsn', + 'valkey_url', + 'mail', + 'content_dir', + 'repositories_dir', + 'blob_dir', + 'staging_dir', + 'service_keys_dir', + ): value['server'].pop(key) return value def file_state(app, tx): - from msg.admin.backups import _db_refs, _verify_storage, _regular_tree, _hash + from msg.admin.backups import _db_refs, _hash, _regular_tree, _verify_storage + settings = app.settings from msg.config import load_settings + actual_settings = load_settings(settings.config_dir) if active(tx): - require(actual_settings.server.mail is None and actual_settings.server.valkey_url is None, 'recovery_target_not_isolated') + require( + actual_settings.server.mail is None and actual_settings.server.valkey_url is None, + 'recovery_target_not_isolated', + ) policy = configuration_state(actual_settings) require(actual_settings == settings, 'recovery_cached_configuration_mismatch') - require(settings.trust_file.is_file() and not settings.trust_file.is_symlink(), 'invalid_trust_anchor') + require( + settings.trust_file.is_file() and not settings.trust_file.is_symlink(), + 'invalid_trust_anchor', + ) trees = {} - for name, path in (('content', settings.server.content_dir), ('blobs', settings.server.blob_dir), - ('repositories', settings.server.repositories_dir), ('staging', settings.server.staging_dir), - ('service', settings.service_keys)): + for name, path in ( + ('content', settings.server.content_dir), + ('blobs', settings.server.blob_dir), + ('repositories', settings.server.repositories_dir), + ('staging', settings.server.staging_dir), + ('service', settings.service_keys), + ): if name in {'repositories', 'staging'} and not path.exists() and not path.is_symlink(): trees[name] = {} continue _regular_tree(path) - trees[name] = {str(file.relative_to(path)): {'size': file.stat().st_size, 'sha256': _hash(file)} - for file in sorted(path.rglob('*')) if file.is_file()} + trees[name] = { + str(file.relative_to(path)): {'size': file.stat().st_size, 'sha256': _hash(file)} + for file in sorted(path.rglob('*')) + if file.is_file() + } refs = _db_refs(tx._connection) for (identifier,) in tx.rows("SELECT id FROM resources WHERE type='repo' AND state<>'purged'"): - require((settings.server.repositories_dir/(identifier+'.git')).is_dir(), 'backup_repository_missing') - _verify_storage(settings.server.content_dir.parent, refs, settings=settings, repair_git_layout=False) - require(Ed25519Signer.from_bytes((settings.service_keys/'online.key').read_bytes()).public_key == - app.online_signer.public_key and - Ed25519Signer.from_bytes((settings.service_keys/'receipt.key').read_bytes()).public_key == - app.receipt_signer.public_key and - (settings.service_keys/'tokens.key').read_bytes() == app._token_secret, 'recovery_cached_keys_mismatch') - keys = {name: digest((settings.service_keys/name).read_bytes()) - for name in ('online.key', 'receipt.key', 'tokens.key')} - require(len((settings.service_keys/'tokens.key').read_bytes()) == 32, 'invalid_service_key') + require( + (settings.server.repositories_dir / (identifier + '.git')).is_dir(), + 'backup_repository_missing', + ) + _verify_storage( + settings.server.content_dir.parent, refs, settings=settings, repair_git_layout=False + ) + require( + Ed25519Signer.from_bytes((settings.service_keys / 'online.key').read_bytes()).public_key + == app.online_signer.public_key + and Ed25519Signer.from_bytes( + (settings.service_keys / 'receipt.key').read_bytes() + ).public_key + == app.receipt_signer.public_key + and (settings.service_keys / 'tokens.key').read_bytes() == app._token_secret, + 'recovery_cached_keys_mismatch', + ) + keys = { + name: digest((settings.service_keys / name).read_bytes()) + for name in ('online.key', 'receipt.key', 'tokens.key') + } + require(len((settings.service_keys / 'tokens.key').read_bytes()) == 32, 'invalid_service_key') # Digests commit random service key material without exporting it. - return {'configuration': policy, 'references': refs, 'trees': trees, 'service_key_digests': keys, - 'trust': loads(settings.trust_file.read_bytes())} + return { + 'configuration': policy, + 'references': refs, + 'trees': trees, + 'service_key_digests': keys, + 'trust': loads(settings.trust_file.read_bytes()), + } async def draft(app, *, public_key, source_backup_sha256, sequence): """Internal use case; public RootAdmin entry first enforces console and PIN.""" - require(type(sequence) is int and sequence > 0 and type(source_backup_sha256) is str and - len(source_backup_sha256) == 64 and all(c in '0123456789abcdef' for c in source_backup_sha256), - 'recovery_proof_invalid') + require( + type(sequence) is int + and sequence > 0 + and type(source_backup_sha256) is str + and len(source_backup_sha256) == 64 + and all(c in '0123456789abcdef' for c in source_backup_sha256), + 'recovery_proof_invalid', + ) async with app.metadata.transaction(write=True) as tx: - require(not active(tx) and not (app.settings.config_dir/'recovery-drill.json').exists() and not (app.settings.config_dir/'recovery-drill.json').is_symlink(), 'recovery_source_is_quarantined') + require( + not active(tx) + and not (app.settings.config_dir / 'recovery-drill.json').exists() + and not (app.settings.config_dir / 'recovery-drill.json').is_symlink(), + 'recovery_source_is_quarantined', + ) state = snapshot(tx) await verify_current_authority(app, tx, public_key) - body = {'format': FORMAT, 'service': app.settings.service_url, - 'source_backup_sha256': source_backup_sha256, 'sequence': sequence, - 'issued_at': wire(app.clock()), 'expires_at': wire(app.clock()+timedelta(hours=24)), - 'runtime_controls': runtime_controls(tx), 'metadata': state, 'files': file_state(app, tx)} + body = { + 'format': FORMAT, + 'service': app.settings.service_url, + 'source_backup_sha256': source_backup_sha256, + 'sequence': sequence, + 'issued_at': wire(app.clock()), + 'expires_at': wire(app.clock() + timedelta(hours=24)), + 'runtime_controls': runtime_controls(tx), + 'metadata': state, + 'files': file_state(app, tx), + } return body async def seal(app, body, signer): async with app.metadata.transaction(write=True) as tx: - require(not active(tx) and not (app.settings.config_dir/'recovery-drill.json').exists() and not (app.settings.config_dir/'recovery-drill.json').is_symlink(), 'recovery_source_is_quarantined') - require(body['service'] == app.settings.service_url and snapshot(tx) == body['metadata'] and - runtime_controls(tx) == body['runtime_controls'] and file_state(app, tx) == body['files'], - 'recovery_complete_state_mismatch') + require( + not active(tx) + and not (app.settings.config_dir / 'recovery-drill.json').exists() + and not (app.settings.config_dir / 'recovery-drill.json').is_symlink(), + 'recovery_source_is_quarantined', + ) + require( + body['service'] == app.settings.service_url + and snapshot(tx) == body['metadata'] + and runtime_controls(tx) == body['runtime_controls'] + and file_state(app, tx) == body['files'], + 'recovery_complete_state_mismatch', + ) await verify_current_authority(app, tx, signer.public_key) - require(parse_time(body['issued_at']) <= app.clock() < parse_time(body['expires_at']), 'recovery_proof_expired') + require( + parse_time(body['issued_at']) <= app.clock() < parse_time(body['expires_at']), + 'recovery_proof_expired', + ) return {'state': body, 'signature': wire(signer.sign(canonical(body), purpose=PURPOSE))} async def capture(app, signer, *, source_backup_sha256, sequence): - body = await draft(app, public_key=signer.public_key, source_backup_sha256=source_backup_sha256, sequence=sequence) + body = await draft( + app, + public_key=signer.public_key, + source_backup_sha256=source_backup_sha256, + sequence=sequence, + ) return await seal(app, body, signer) def _receipt(value, pin): - require(type(value) is dict and set(value) == {'receipt', 'signature'}, 'recovery_promotion_receipt_invalid') + require( + type(value) is dict and set(value) == {'receipt', 'signature'}, + 'recovery_promotion_receipt_invalid', + ) body = value['receipt'] - require(type(body) is dict and set(body) == {'manifest_digest', 'sequence', 'source_backup_sha256', - 'generation', 'prior_generation', 'audit_row', 'prior_audit_sequence'}, 'recovery_promotion_receipt_invalid') - require(type(body['generation']) is str and bool(body['generation']) and - (body['prior_generation'] is None or (type(body['prior_generation']) is str and bool(body['prior_generation']))) and - type(body['audit_row']) is list and len(body['audit_row']) == 4 and - type(body['audit_row'][0]) is int and body['audit_row'][0] > 0 and - type(body['prior_audit_sequence']) is list and len(body['prior_audit_sequence']) == 2 and - type(body['prior_audit_sequence'][0]) is int and type(body['prior_audit_sequence'][1]) is bool, - 'recovery_promotion_receipt_invalid') - verify(pin.public_key, canonical(body), decode(Signature, value['signature']), purpose=RECEIPT_PURPOSE) - require(body['manifest_digest'] == pin.digest and body['sequence'] == pin.sequence and - body['source_backup_sha256'] == pin.source_backup_sha256, 'recovery_promotion_receipt_mismatch') + require( + type(body) is dict + and set(body) + == { + 'manifest_digest', + 'sequence', + 'source_backup_sha256', + 'generation', + 'prior_generation', + 'audit_row', + 'prior_audit_sequence', + }, + 'recovery_promotion_receipt_invalid', + ) + require( + type(body['generation']) is str + and bool(body['generation']) + and ( + body['prior_generation'] is None + or (type(body['prior_generation']) is str and bool(body['prior_generation'])) + ) + and type(body['audit_row']) is list + and len(body['audit_row']) == 4 + and type(body['audit_row'][0]) is int + and body['audit_row'][0] > 0 + and type(body['prior_audit_sequence']) is list + and len(body['prior_audit_sequence']) == 2 + and type(body['prior_audit_sequence'][0]) is int + and type(body['prior_audit_sequence'][1]) is bool, + 'recovery_promotion_receipt_invalid', + ) + verify( + pin.public_key, + canonical(body), + decode(Signature, value['signature']), + purpose=RECEIPT_PURPOSE, + ) + require( + body['manifest_digest'] == pin.digest + and body['sequence'] == pin.sequence + and body['source_backup_sha256'] == pin.source_backup_sha256, + 'recovery_promotion_receipt_mismatch', + ) return body async def promote(app, packet, *, pin, signer, operator): """Internal fixture-capable use case. Caller must enforce physical console ceremony.""" body = verify_proof(packet, pin, app.clock()) - require(app.settings.server.mail is None and app.settings.server.valkey_url is None, - 'recovery_target_not_isolated') + require( + app.settings.server.mail is None and app.settings.server.valkey_url is None, + 'recovery_target_not_isolated', + ) require(signer.public_key == pin.public_key, 'recovery_root_mismatch') require(type(operator) is str and bool(operator), 'recovery_operator_required') - marker = app.settings.config_dir/'recovery-drill.json' + marker = app.settings.config_dir / 'recovery-drill.json' from msg.security.root_files import read_private + marker_present = marker.exists() or marker.is_symlink() if marker_present: marker_value = loads(read_private(marker)) - require(type(marker_value) is dict and marker_value.get('source_backup_sha256') == pin.source_backup_sha256, - 'recovery_checkpoint_backup_mismatch') + require( + type(marker_value) is dict + and marker_value.get('source_backup_sha256') == pin.source_backup_sha256, + 'recovery_checkpoint_backup_mismatch', + ) async with app.metadata.transaction(write=True) as tx: previous = tx.setting('recovery_promotion') # A backup of an already recovered source contains a historical receipt. # Only the exact current manifest can resume its pending delta; a new # recovery must independently match the entire initial source state. - pending = type(previous) is dict and type(previous.get('receipt')) is dict and previous['receipt'].get('manifest_digest') == pin.digest + pending = ( + type(previous) is dict + and type(previous.get('receipt')) is dict + and previous['receipt'].get('manifest_digest') == pin.digest + ) resume = _receipt(previous, pin) if pending else None gate = tx.setting(SETTING) - require(active(tx) and type(gate) is dict and gate.get('source_backup_sha256') == pin.source_backup_sha256, - 'recovery_quarantine_required') + require( + active(tx) + and type(gate) is dict + and gate.get('source_backup_sha256') == pin.source_backup_sha256, + 'recovery_quarantine_required', + ) if resume is None: require(marker_present, 'recovery_marker_required') - require(active(tx) and type(gate) is dict and gate.get('source_backup_sha256') == pin.source_backup_sha256 and - gate.get('format') == 'msg-recovery-quarantine-v1' and gate.get('authority') == 'health_only' and gate.get('outbound_enabled') is False, - 'recovery_quarantine_required') - actual = snapshot(tx, controls=body['runtime_controls'] if resume is None else None, resume=resume) + require( + active(tx) + and type(gate) is dict + and gate.get('source_backup_sha256') == pin.source_backup_sha256 + and gate.get('format') == 'msg-recovery-quarantine-v1' + and gate.get('authority') == 'health_only' + and gate.get('outbound_enabled') is False, + 'recovery_quarantine_required', + ) + actual = snapshot( + tx, controls=body['runtime_controls'] if resume is None else None, resume=resume + ) require(actual == body['metadata'], 'recovery_complete_state_mismatch') require(file_state(app, tx) == body['files'], 'recovery_complete_files_mismatch') await verify_current_authority(app, tx, pin.public_key) verify_proof(packet, pin, app.clock()) if resume is None: prior_generation = tx.setting('recovery_runtime_generation') - require(prior_generation is None or type(prior_generation) is str, 'recovery_runtime_invalid') + require( + prior_generation is None or type(prior_generation) is str, + 'recovery_runtime_invalid', + ) generation = uuid4().hex - event = Event(id='audit_'+uuid4().hex, type='recovery.complete_promotion', time=app.clock(), - request_id=pin.digest, actor=ROOT_SUBJECT, subject=ROOT_SUBJECT, resources=(), - data={'manifest_digest': pin.digest, 'sequence': pin.sequence, 'operator': operator, - 'runtime_generation': generation, 'requires_runtime_restart': True}) - audit_event = AuditEvent(event=event, + event = Event( + id='audit_' + uuid4().hex, + type='recovery.complete_promotion', + time=app.clock(), + request_id=pin.digest, + actor=ROOT_SUBJECT, + subject=ROOT_SUBJECT, + resources=(), + data={ + 'manifest_digest': pin.digest, + 'sequence': pin.sequence, + 'operator': operator, + 'runtime_generation': generation, + 'requires_runtime_restart': True, + }, + ) + audit_event = AuditEvent( + event=event, authority=(ResourceRef(id=tx.setting('active_root_certificate')),), - before_digest=pin.digest, after_digest=pin.digest, previous_digest=None, - entry_digest='', result='verified_complete_state') + before_digest=pin.digest, + after_digest=pin.digest, + previous_digest=None, + entry_digest='', + result='verified_complete_state', + ) previous = tx.one('SELECT seq,digest FROM audit ORDER BY seq DESC LIMIT 1') prior_sequence = actual['sequences']['audit_seq_seq'] - audit_seq = max(prior_sequence[0] + int(prior_sequence[1]), (previous[0]+1) if previous else 1) - audit_data = wire(replace(audit_event, previous_digest=previous[1] if previous else None, entry_digest='')) + audit_seq = max( + prior_sequence[0] + int(prior_sequence[1]), (previous[0] + 1) if previous else 1 + ) + audit_data = wire( + replace( + audit_event, previous_digest=previous[1] if previous else None, entry_digest='' + ) + ) audit_data.pop('entry_digest') - audit_event = replace(audit_event, previous_digest=previous[1] if previous else None, entry_digest=digest(audit_data)) + audit_event = replace( + audit_event, + previous_digest=previous[1] if previous else None, + entry_digest=digest(audit_data), + ) # Explicit allocation is transactional. PostgreSQL nextval is not: # using it here would invalidate the source commitment on rollback. - tx.execute('INSERT INTO audit (seq,digest,previous,body) VALUES (?,?,?,?)', - (audit_seq, audit_event.entry_digest, audit_event.previous_digest, canonical(audit_event).decode()), write=True) - audit_row = tx.one('SELECT seq,digest,previous,body FROM audit ORDER BY seq DESC LIMIT 1') - receipt = {'manifest_digest': pin.digest, 'sequence': pin.sequence, - 'source_backup_sha256': pin.source_backup_sha256, 'generation': generation, - 'prior_generation': prior_generation, 'audit_row': loads(canonical(audit_row)), - 'prior_audit_sequence': actual['sequences']['audit_seq_seq']} - tx.set_setting('recovery_promotion', {'receipt': receipt, - 'signature': wire(signer.sign(canonical(receipt), purpose=RECEIPT_PURPOSE))}) + tx.execute( + 'INSERT INTO audit (seq,digest,previous,body) VALUES (?,?,?,?)', + ( + audit_seq, + audit_event.entry_digest, + audit_event.previous_digest, + canonical(audit_event).decode(), + ), + write=True, + ) + audit_row = tx.one( + 'SELECT seq,digest,previous,body FROM audit ORDER BY seq DESC LIMIT 1' + ) + receipt = { + 'manifest_digest': pin.digest, + 'sequence': pin.sequence, + 'source_backup_sha256': pin.source_backup_sha256, + 'generation': generation, + 'prior_generation': prior_generation, + 'audit_row': loads(canonical(audit_row)), + 'prior_audit_sequence': actual['sequences']['audit_seq_seq'], + } + tx.set_setting( + 'recovery_promotion', + { + 'receipt': receipt, + 'signature': wire(signer.sign(canonical(receipt), purpose=RECEIPT_PURPOSE)), + }, + ) tx.set_setting('recovery_runtime_generation', generation) runtime = tx.setting('runtime_config') - normalized = _runtime_row(('runtime_config', canonical(runtime).decode()), body['runtime_controls']) + normalized = _runtime_row( + ('runtime_config', canonical(runtime).decode()), body['runtime_controls'] + ) if normalized is None: tx.execute("DELETE FROM settings WHERE key='runtime_config'", write=True) else: @@ -260,13 +493,19 @@ async def promote(app, packet, *, pin, signer, operator): # succeed. A crash between unlink and commit remains closed through the DB; # only an authenticated exact receipt can resume with the marker missing. import os + async with app.metadata.transaction(write=True) as tx: require(active(tx), 'recovery_quarantine_required') - require(_receipt(tx.setting('recovery_promotion'), pin) == resume, 'recovery_promotion_receipt_mismatch') + require( + _receipt(tx.setting('recovery_promotion'), pin) == resume, + 'recovery_promotion_receipt_mismatch', + ) require(snapshot(tx, resume=resume) == body['metadata'], 'recovery_complete_state_mismatch') require(file_state(app, tx) == body['files'], 'recovery_complete_files_mismatch') verify_proof(packet, pin, app.clock()) - tx.execute("SELECT setval('public.audit_seq_seq', ?, true)", (resume['audit_row'][0],), write=True) + tx.execute( + "SELECT setval('public.audit_seq_seq', ?, true)", (resume['audit_row'][0],), write=True + ) try: fd = os.open(marker.parent, os.O_RDONLY | os.O_DIRECTORY) try: @@ -278,27 +517,48 @@ async def promote(app, packet, *, pin, signer, operator): except OSError as exc: raise Failure('recovery_promotion_finish_required') from exc tx.execute('DELETE FROM settings WHERE key=?', (SETTING,), write=True) - return {'status': 'recovery_promoted', 'manifest_digest': pin.digest, - 'runtime_generation': resume['generation'], 'requires_runtime_restart': True} + return { + 'status': 'recovery_promoted', + 'manifest_digest': pin.digest, + 'runtime_generation': resume['generation'], + 'requires_runtime_restart': True, + } async def open_for_proof(config_dir): """Load only existing authority; no DDL, release-source sync, repair or worker.""" from msg.application import Application from msg.config import load_settings - from msg.storage.postgres import PostgresMetadataStore from msg.security.certificates import CertificateValidator + from msg.storage.postgres import PostgresMetadataStore + settings = load_settings(config_dir) app = Application(settings) app.metadata = PostgresMetadataStore(settings.server.postgres_dsn, initialize=False) - require(settings.trust_file.is_file() and not settings.trust_file.is_symlink(), 'invalid_trust_anchor') + require( + settings.trust_file.is_file() and not settings.trust_file.is_symlink(), + 'invalid_trust_anchor', + ) trust = loads(settings.trust_file.read_bytes()) - require(set(trust) == {'version', 'public_key', 'certificate'} and trust['version'] == 1, 'invalid_trust_anchor') - app.certificates = CertificateValidator(app.registry, decode(Certificate, trust['certificate']), - unb64(trust['public_key'], limit=32), settings.service_url, app.clock) + require( + set(trust) == {'version', 'public_key', 'certificate'} and trust['version'] == 1, + 'invalid_trust_anchor', + ) + app.certificates = CertificateValidator( + app.registry, + decode(Certificate, trust['certificate']), + unb64(trust['public_key'], limit=32), + settings.service_url, + app.clock, + ) from msg.admin.backups import _regular_tree + _regular_tree(settings.service_keys) - app.online_signer = Ed25519Signer.from_bytes((settings.service_keys/'online.key').read_bytes()) - app.receipt_signer = Ed25519Signer.from_bytes((settings.service_keys/'receipt.key').read_bytes()) - app._token_secret = (settings.service_keys/'tokens.key').read_bytes() + app.online_signer = Ed25519Signer.from_bytes( + (settings.service_keys / 'online.key').read_bytes() + ) + app.receipt_signer = Ed25519Signer.from_bytes( + (settings.service_keys / 'receipt.key').read_bytes() + ) + app._token_secret = (settings.service_keys / 'tokens.key').read_bytes() return app diff --git a/src/msg/admin/recovery_replay.py b/src/msg/admin/recovery_replay.py index 7ca9d14a..392b5e35 100644 --- a/src/msg/admin/recovery_replay.py +++ b/src/msg/admin/recovery_replay.py @@ -6,6 +6,7 @@ network operation, configuration override, or quarantine-clearing operation here. Unknown fact kinds require a newer reviewed replay implementation and stay blocked. """ + from __future__ import annotations import re @@ -23,17 +24,30 @@ FORMAT = 'msg-revocation-checkpoint-v1' POLICY_FORMAT = 'msg-revocation-checkpoint-v2' -SUPPORTED_FACTS = frozenset({'credential.revoke', 'certificate.revoke', 'share_grant.revoke', - 'share_grant_v2.revoke', 'share_link.revoke', 'membership.remove', - 'topic_membership.remove', 'topic_ban.apply', 'topic_ban.lift', 'identity_key.retire', - 'encryption_key.retire', 'vault.destroy'}) +SUPPORTED_FACTS = frozenset({ + 'credential.revoke', + 'certificate.revoke', + 'share_grant.revoke', + 'share_grant_v2.revoke', + 'share_link.revoke', + 'membership.remove', + 'topic_membership.remove', + 'topic_ban.apply', + 'topic_ban.lift', + 'identity_key.retire', + 'encryption_key.retire', + 'vault.destroy', +}) PROMOTION_BLOCKED_REASONS = ( 'quarantine_remains', 'supported_fact_inventory_incomplete', 'backup_retirement_not_attested', ) -_SHARE_TABLES = {'share_grant.revoke': 'share_grants', 'share_grant_v2.revoke': 'share_grants_v2', - 'share_link.revoke': 'share_links'} +_SHARE_TABLES = { + 'share_grant.revoke': 'share_grants', + 'share_grant_v2.revoke': 'share_grants_v2', + 'share_link.revoke': 'share_links', +} _MAX_ENTRIES = 10000 @@ -45,6 +59,7 @@ class TrustedCheckpointPin: It does not assert that an offline source was current or operator-approved. Production remains blocked without evidence for that separate trust ceremony. """ + service: str public_key: bytes digest: str @@ -54,26 +69,36 @@ class TrustedCheckpointPin: def verify_checkpoint(packet, *, pin): require(isinstance(pin, TrustedCheckpointPin), 'recovery_checkpoint_pin_required') try: - require(type(packet) is dict and set(packet) == {'checkpoint', 'signature'}, - 'recovery_checkpoint_invalid') + require( + type(packet) is dict and set(packet) == {'checkpoint', 'signature'}, + 'recovery_checkpoint_invalid', + ) body = packet['checkpoint'] policy = type(body) is dict and body.get('format') == POLICY_FORMAT fields = {'format', 'service', 'source_backup_sha256', 'sequence', 'entries'} if policy: fields.add('coverage') - require(type(body) is dict and set(body) == fields, - 'recovery_checkpoint_invalid') - require(type(pin.public_key) is bytes and len(pin.public_key) == 32 and - type(pin.sequence) is int and 0 <= pin.sequence <= _MAX_ENTRIES and - type(body['sequence']) is int and body['sequence'] == pin.sequence and - body['format'] in {FORMAT, POLICY_FORMAT} and body['service'] == pin.service and - isinstance(pin.service, str) and bool(pin.service) and - isinstance(body['source_backup_sha256'], str) and - re.fullmatch('[0-9a-f]{64}', body['source_backup_sha256']) is not None and - digest(body) == pin.digest, 'recovery_checkpoint_invalid') + require(type(body) is dict and set(body) == fields, 'recovery_checkpoint_invalid') + require( + type(pin.public_key) is bytes + and len(pin.public_key) == 32 + and type(pin.sequence) is int + and 0 <= pin.sequence <= _MAX_ENTRIES + and type(body['sequence']) is int + and body['sequence'] == pin.sequence + and body['format'] in {FORMAT, POLICY_FORMAT} + and body['service'] == pin.service + and isinstance(pin.service, str) + and bool(pin.service) + and isinstance(body['source_backup_sha256'], str) + and re.fullmatch('[0-9a-f]{64}', body['source_backup_sha256']) is not None + and digest(body) == pin.digest, + 'recovery_checkpoint_invalid', + ) entries = body['entries'] - require(type(entries) is list and len(entries) == pin.sequence, - 'recovery_checkpoint_invalid') + require( + type(entries) is list and len(entries) == pin.sequence, 'recovery_checkpoint_invalid' + ) domains = set() authority_chains = {} for sequence, fact in enumerate(entries, 1): @@ -81,25 +106,42 @@ def verify_checkpoint(packet, *, pin): fact_fields = {'sequence', 'kind', 'subject', 'target', 'at'} if typed: fact_fields.add('value') - require(type(fact) is dict and set(fact) == fact_fields and - type(fact['sequence']) is int and fact['sequence'] == sequence and - fact['kind'] in (SUPPORTED_FACTS | recovery_policy.FACTS if policy else SUPPORTED_FACTS) and - all(isinstance(fact[name], str) and 0 < len(fact[name]) <= 256 - for name in ('subject', 'target')), 'recovery_checkpoint_invalid') + require( + type(fact) is dict + and set(fact) == fact_fields + and type(fact['sequence']) is int + and fact['sequence'] == sequence + and fact['kind'] + in (SUPPORTED_FACTS | recovery_policy.FACTS if policy else SUPPORTED_FACTS) + and all( + isinstance(fact[name], str) and 0 < len(fact[name]) <= 256 + for name in ('subject', 'target') + ), + 'recovery_checkpoint_invalid', + ) require(parse_time(fact['at']) <= datetime.now(UTC), 'recovery_checkpoint_invalid') if typed: recovery_policy.validate(fact) if fact['kind'] == 'resource.authority.reconcile': chain = authority_chains.setdefault(fact['target'], []) - require(not chain or chain[-1]['value']['current'] == fact['value']['previous'], - 'recovery_authority_chain_gap') + require( + not chain or chain[-1]['value']['current'] == fact['value']['previous'], + 'recovery_authority_chain_gap', + ) chain.append(fact) domains.add(recovery_policy.DOMAINS[fact['kind']] if typed else 'revocations') if policy: - require(body['coverage'] == {'complete': False, 'domains': sorted(domains)} and - body['coverage'].get('complete') is False, 'recovery_checkpoint_incomplete_contract') - verify(pin.public_key, canonical(body), decode(Signature, packet['signature']), - purpose='recovery-checkpoint-v2' if policy else 'recovery-checkpoint-v1') + require( + body['coverage'] == {'complete': False, 'domains': sorted(domains)} + and body['coverage'].get('complete') is False, + 'recovery_checkpoint_incomplete_contract', + ) + verify( + pin.public_key, + canonical(body), + decode(Signature, packet['signature']), + purpose='recovery-checkpoint-v2' if policy else 'recovery-checkpoint-v1', + ) # Own the verified bytes across the subsequent await/transaction boundary. return loads(canonical(body)) except (Failure, KeyError, TypeError, ValueError, OverflowError) as exc: @@ -119,23 +161,32 @@ def _earlier(previous, at): def _revoke_credential(tx, subject, target, at): row = _owned(tx.one('SELECT subject,body FROM credentials WHERE id=?', (target,)), subject) credential = decode(Credential, loads(row[1])) - require(credential.id == target and credential.subject_id == subject, - 'recovery_fact_subject_mismatch') + require( + credential.id == target and credential.subject_id == subject, + 'recovery_fact_subject_mismatch', + ) when = parse_time(at) changed = credential.revoked_at is None or credential.revoked_at > when if changed: revoked = replace(credential, revoked_at=when) - tx.execute('UPDATE credentials SET body=? WHERE id=?', - (canonical(revoked).decode(), target), write=True) + tx.execute( + 'UPDATE credentials SET body=? WHERE id=?', + (canonical(revoked).decode(), target), + write=True, + ) if credential.kind == 'token': - delivery = tx.one('SELECT subject,consumed_at FROM token_deliveries WHERE credential_id=?', - (target,)) + delivery = tx.one( + 'SELECT subject,consumed_at FROM token_deliveries WHERE credential_id=?', (target,) + ) if delivery is not None: _owned(delivery, subject) earliest = _earlier(delivery[1], at) if earliest != delivery[1]: - tx.execute('UPDATE token_deliveries SET consumed_at=? WHERE credential_id=?', - (earliest, target), write=True) + tx.execute( + 'UPDATE token_deliveries SET consumed_at=? WHERE credential_id=?', + (earliest, target), + write=True, + ) changed = True return changed @@ -147,42 +198,60 @@ async def _apply(tx, fact): if kind == 'credential.revoke': return _revoke_credential(tx, subject, target, at) if kind == 'certificate.revoke': - row = _owned(tx.one('SELECT subject,revoked,body FROM certificates WHERE id=?', - (target,)), subject) + row = _owned( + tx.one('SELECT subject,revoked,body FROM certificates WHERE id=?', (target,)), subject + ) certificate = decode(Certificate, loads(row[2])) - require(certificate.resource_id == target and certificate.subject_id == subject, - 'recovery_fact_subject_mismatch') + require( + certificate.resource_id == target and certificate.subject_id == subject, + 'recovery_fact_subject_mismatch', + ) if row[1] != 1: tx.execute('UPDATE certificates SET revoked=1 WHERE id=?', (target,), write=True) return True return False if kind in _SHARE_TABLES: table = _SHARE_TABLES[kind] # Fixed closed vocabulary, never caller SQL. - row = _owned(tx.one(f'SELECT grantor,revoked_at FROM {table} WHERE id=?', (target,)), subject) + row = _owned( + tx.one(f'SELECT grantor,revoked_at FROM {table} WHERE id=?', (target,)), subject + ) earliest = _earlier(row[1], at) if row[1] != earliest: - tx.execute(f'UPDATE {table} SET revoked_at=? WHERE id=?', (earliest, target), write=True) + tx.execute( + f'UPDATE {table} SET revoked_at=? WHERE id=?', (earliest, target), write=True + ) return True return False if kind == 'membership.remove': require(target != 'g_public', 'recovery_virtual_membership_requires_identity_policy') - row = tx.one('SELECT generation,body FROM memberships WHERE org=? AND subject=?', - (target, subject)) + row = tx.one( + 'SELECT generation,body FROM memberships WHERE org=? AND subject=?', (target, subject) + ) require(row is not None, 'recovery_fact_missing') member = decode(Membership, loads(row[1])) - require(member.subject_id == subject and member.organization_id == target and - member.version == row[0], 'recovery_fact_subject_mismatch') + require( + member.subject_id == subject + and member.organization_id == target + and member.version == row[0], + 'recovery_fact_subject_mismatch', + ) if member.status != 'rejected': - await tx.update_identity(replace(member, status='rejected', version=member.version + 1), - member.version) + await tx.update_identity( + replace(member, status='rejected', version=member.version + 1), member.version + ) return True return False if kind == 'topic_membership.remove': - row = tx.one('SELECT status FROM topic_memberships WHERE topic=? AND subject=?', (target, subject)) + row = tx.one( + 'SELECT status FROM topic_memberships WHERE topic=? AND subject=?', (target, subject) + ) require(row is not None, 'recovery_fact_missing') if row[0] != 'removed': - tx.execute("UPDATE topic_memberships SET status='removed' WHERE topic=? AND subject=?", - (target, subject), write=True) + tx.execute( + "UPDATE topic_memberships SET status='removed' WHERE topic=? AND subject=?", + (target, subject), + write=True, + ) return True return False if kind in {'topic_ban.apply', 'topic_ban.set'}: @@ -190,23 +259,37 @@ async def _apply(tx, fact): if kind == 'topic_ban.set': await tx.subject(subject) require((await tx.resource(target)).type == 'topic', 'not_a_topic') - require(tx.one('SELECT id FROM resources WHERE id=?', (target,)) is not None, - 'recovery_fact_missing') + require( + tx.one('SELECT id FROM resources WHERE id=?', (target,)) is not None, + 'recovery_fact_missing', + ) # A live ban removes membership as well as denying topic participation. # Preserve that effect when replaying an old snapshot, including when a # later fact lifts the ban: lifting never restores an old admin role. - member = tx.one('SELECT role,status FROM topic_memberships WHERE topic=? AND subject=?', - (target, subject)) + member = tx.one( + 'SELECT role,status FROM topic_memberships WHERE topic=? AND subject=?', + (target, subject), + ) member_changed = member is not None and member != ('member', 'removed') if member_changed: - tx.execute("UPDATE topic_memberships SET role='member',status='removed' " - 'WHERE topic=? AND subject=?', (target, subject), write=True) - row = tx.one('SELECT status,expires_at FROM topic_bans WHERE topic=? AND subject=?', (target, subject)) + tx.execute( + "UPDATE topic_memberships SET role='member',status='removed' " + 'WHERE topic=? AND subject=?', + (target, subject), + write=True, + ) + row = tx.one( + 'SELECT status,expires_at FROM topic_bans WHERE topic=? AND subject=?', + (target, subject), + ) if row is None: - tx.execute('''INSERT INTO topic_bans + tx.execute( + """INSERT INTO topic_bans (topic,subject,actor,created_at,expires_at,reason,status) - VALUES (?,?,?,?,?,?,?)''', - (target, subject, ROOT_SUBJECT, at, expires, 'recovery-replay', 'active'), write=True) + VALUES (?,?,?,?,?,?,?)""", + (target, subject, ROOT_SUBJECT, at, expires, 'recovery-replay', 'active'), + write=True, + ) return True # Restrictions never shorten an existing active ban. A permanent ban # dominates finite expiries; only a separate explicit lift can remove it. @@ -217,46 +300,74 @@ async def _apply(tx, fact): expires = row[1] # V1 has no expiry and must not inherit a stale snapshot expiry. if row[0] != 'active' or row[1] != expires: - tx.execute("UPDATE topic_bans SET status='active',expires_at=? WHERE topic=? AND subject=?", - (expires, target, subject), write=True) + tx.execute( + "UPDATE topic_bans SET status='active',expires_at=? WHERE topic=? AND subject=?", + (expires, target, subject), + write=True, + ) return True return member_changed if kind == 'topic_ban.lift': row = tx.one('SELECT status FROM topic_bans WHERE topic=? AND subject=?', (target, subject)) require(row is not None, 'recovery_fact_missing') if row[0] != 'lifted': - tx.execute("UPDATE topic_bans SET status='lifted' WHERE topic=? AND subject=?", - (target, subject), write=True) + tx.execute( + "UPDATE topic_bans SET status='lifted' WHERE topic=? AND subject=?", + (target, subject), + write=True, + ) return True return False if kind in {'identity_key.retire', 'encryption_key.retire'}: table = 'identity_keys' if kind == 'identity_key.retire' else 'encryption_subkeys' - row = _owned(tx.one(f'SELECT subject,retired_at,is_primary FROM {table} WHERE key_id=?', - (target,)), subject) + row = _owned( + tx.one(f'SELECT subject,retired_at,is_primary FROM {table} WHERE key_id=?', (target,)), + subject, + ) earliest = _earlier(row[1], at) changed = row[1] != earliest or row[2] != 0 if changed: - tx.execute(f'UPDATE {table} SET retired_at=?,is_primary=0 WHERE key_id=?', - (earliest, target), write=True) + tx.execute( + f'UPDATE {table} SET retired_at=?,is_primary=0 WHERE key_id=?', + (earliest, target), + write=True, + ) if kind == 'identity_key.retire': changed = _revoke_credential(tx, subject, target, at) or changed - vault = tx.one('SELECT signing_key_id,signing_nonce,signing_ciphertext,age_ciphertext ' - 'FROM custodial_vault WHERE subject=?', (subject,)) - if vault is not None and vault[0] == target and (vault[1] is not None or vault[2] is not None): + vault = tx.one( + 'SELECT signing_key_id,signing_nonce,signing_ciphertext,age_ciphertext ' + 'FROM custodial_vault WHERE subject=?', + (subject,), + ) + if ( + vault is not None + and vault[0] == target + and (vault[1] is not None or vault[2] is not None) + ): status = 'decrypt_only' if vault[3] is not None else 'destroyed' - tx.execute('UPDATE custodial_vault SET signing_nonce=NULL,signing_ciphertext=NULL,' - 'status=? WHERE subject=?', (status, subject), write=True) + tx.execute( + 'UPDATE custodial_vault SET signing_nonce=NULL,signing_ciphertext=NULL,' + 'status=? WHERE subject=?', + (status, subject), + write=True, + ) changed = True return changed require(kind == 'vault.destroy', 'recovery_checkpoint_invalid') - row = tx.one('SELECT encryption_key_id,signing_nonce,signing_ciphertext,age_nonce,' - 'age_ciphertext,status FROM custodial_vault WHERE subject=?', (subject,)) + row = tx.one( + 'SELECT encryption_key_id,signing_nonce,signing_ciphertext,age_nonce,' + 'age_ciphertext,status FROM custodial_vault WHERE subject=?', + (subject,), + ) require(row is not None, 'recovery_fact_missing') require(row[0] == target, 'recovery_fact_subject_mismatch') if any(value is not None for value in row[1:5]) or row[5] != 'destroyed': - tx.execute("UPDATE custodial_vault SET signing_nonce=NULL,signing_ciphertext=NULL," - "age_nonce=NULL,age_ciphertext=NULL,status='destroyed',destroyed_at=? WHERE subject=?", - (at, subject), write=True) + tx.execute( + 'UPDATE custodial_vault SET signing_nonce=NULL,signing_ciphertext=NULL,' + "age_nonce=NULL,age_ciphertext=NULL,status='destroyed',destroyed_at=? WHERE subject=?", + (at, subject), + write=True, + ) return True return False @@ -264,6 +375,7 @@ async def _apply(tx, fact): async def replay(store, packet, *, pin, config_dir): """Physical-console entry; neither a TTY flag nor an HTTP caller can authorize it.""" from msg.admin.root import require_local_console + operator = require_local_console(config_dir) return await _replay(store, packet, pin=pin, operator=operator) @@ -283,17 +395,28 @@ async def _replay(store, packet, *, pin, operator): async with store.transaction(write=True) as tx: require(active(tx), 'recovery_quarantine_required') gate = tx.setting(SETTING) - require(isinstance(gate, dict) and gate.get('format') == 'msg-recovery-quarantine-v1' and - gate.get('outbound_enabled') is False and gate.get('authority') == 'health_only', - 'recovery_quarantine_invalid') - require(gate.get('source_backup_sha256') == body['source_backup_sha256'], - 'recovery_checkpoint_backup_mismatch') + require( + isinstance(gate, dict) + and gate.get('format') == 'msg-recovery-quarantine-v1' + and gate.get('outbound_enabled') is False + and gate.get('authority') == 'health_only', + 'recovery_quarantine_invalid', + ) + require( + gate.get('source_backup_sha256') == body['source_backup_sha256'], + 'recovery_checkpoint_backup_mismatch', + ) previous = tx.setting('recovery_replay') if previous is not None: - require(isinstance(previous, dict) and previous.get('format') in {FORMAT, body['format']} and - type(previous.get('sequence')) is int and 0 <= previous['sequence'] <= pin.sequence and - previous.get('prefix_digest') == digest(body['entries'][:previous['sequence']]), - 'recovery_checkpoint_regression') + require( + isinstance(previous, dict) + and previous.get('format') in {FORMAT, body['format']} + and type(previous.get('sequence')) is int + and 0 <= previous['sequence'] <= pin.sequence + and previous.get('prefix_digest') + == digest(body['entries'][: previous['sequence']]), + 'recovery_checkpoint_regression', + ) changed = 0 reconciled = set() authority_chains = {} @@ -306,70 +429,126 @@ async def _replay(store, packet, *, pin, operator): continue chain = authority_chains[fact['target']] actual_resource = await tx.resource(fact['target']) - actual = {field: getattr(actual_resource, field) for field in ('owner', 'group', 'parent')} - require(actual in [chain[0]['value']['previous']] + - [item['value']['current'] for item in chain], 'recovery_authority_state_mismatch') + actual = { + field: getattr(actual_resource, field) for field in ('owner', 'group', 'parent') + } + require( + actual + in [chain[0]['value']['previous']] + + [item['value']['current'] for item in chain], + 'recovery_authority_state_mismatch', + ) # Collapse only a verified contiguous chain; an old or intermediate # snapshot converges once without transiently restoring old scope. - fact = dict(chain[-1], subject=actual['owner'], - value={'previous': actual, 'current': chain[-1]['value']['current']}) + fact = dict( + chain[-1], + subject=actual['owner'], + value={'previous': actual, 'current': chain[-1]['value']['current']}, + ) reconciled.add(fact['target']) - if fact['kind'] in {'resource.acl.restrict', 'topic.policy.restrict'} and fact['target'] in authority_chains: + if ( + fact['kind'] in {'resource.acl.restrict', 'topic.policy.restrict'} + and fact['target'] in authority_chains + ): chain = authority_chains[fact['target']] - states = [chain[0]['value']['previous']] + [item['value']['current'] for item in chain] + states = [chain[0]['value']['previous']] + [ + item['value']['current'] for item in chain + ] actual_resource = await tx.resource(fact['target']) - actual = {field: getattr(actual_resource, field) for field in ('owner', 'group', 'parent')} + actual = { + field: getattr(actual_resource, field) for field in ('owner', 'group', 'parent') + } require(actual in states, 'recovery_authority_state_mismatch') bound = chain[0]['value']['previous'] for transition in chain: if transition['sequence'] < fact['sequence']: bound = transition['value']['current'] if fact['kind'] == 'resource.acl.restrict': - require(fact['subject'] == fact['value']['owner'] == bound['owner'] and - fact['value']['group'] == bound['group'], 'recovery_fact_subject_mismatch') - fact = dict(fact, subject=actual['owner'], - value=dict(fact['value'], owner=actual['owner'], group=actual['group'])) + require( + fact['subject'] == fact['value']['owner'] == bound['owner'] + and fact['value']['group'] == bound['group'], + 'recovery_fact_subject_mismatch', + ) + fact = dict( + fact, + subject=actual['owner'], + value=dict(fact['value'], owner=actual['owner'], group=actual['group']), + ) else: require(fact['subject'] == bound['owner'], 'recovery_fact_subject_mismatch') fact = dict(fact, subject=actual['owner']) changed += bool(await _apply(tx, fact)) - receipt = {'format': body['format'], 'checkpoint_digest': pin.digest, 'sequence': pin.sequence, - 'prefix_digest': digest(body['entries']), 'scope': 'supported_authority_reconciliation_facts', - 'promotion': 'blocked', - 'promotion_blocked_reasons': list(PROMOTION_BLOCKED_REASONS), - 'backup_retired': False} + receipt = { + 'format': body['format'], + 'checkpoint_digest': pin.digest, + 'sequence': pin.sequence, + 'prefix_digest': digest(body['entries']), + 'scope': 'supported_authority_reconciliation_facts', + 'promotion': 'blocked', + 'promotion_blocked_reasons': list(PROMOTION_BLOCKED_REASONS), + 'backup_retired': False, + } if body['format'] == POLICY_FORMAT: receipt['scope'] = 'supported_policy_reconciliation_only' receipt['coverage'] = body['coverage'] - staged = sorted({fact['target'] for fact in body['entries'] - if fact['kind'] == 'resource.authority.reconcile'}) + staged = sorted({ + fact['target'] + for fact in body['entries'] + if fact['kind'] == 'resource.authority.reconcile' + }) if staged: receipt['scope'] = 'supported_policy_reconciliation_only' receipt['authority_rebuild_required'] = staged - receipt['promotion_blocked_reasons'].append('derived_authority_inventory_incomplete') + receipt['promotion_blocked_reasons'].append( + 'derived_authority_inventory_incomplete' + ) if changed: tx.set_setting('authorization_epoch', tx.setting('authorization_epoch', 0) + 1) if changed or previous != receipt: tx.set_setting('recovery_replay', receipt) - event = Event(id='audit_' + uuid4().hex, type='recovery.revocation_replay', - time=datetime.now(UTC), request_id=pin.digest, actor=ROOT_SUBJECT, subject=ROOT_SUBJECT, - resources=(), data=dict(receipt, operator=operator, - checkpoint_key_id=key_id(pin.public_key))) - await tx.append_audit(AuditEvent(event=event, authority=(), before_digest=digest(previous), - after_digest=digest(receipt), previous_digest=None, entry_digest='', result='quarantined')) + event = Event( + id='audit_' + uuid4().hex, + type='recovery.revocation_replay', + time=datetime.now(UTC), + request_id=pin.digest, + actor=ROOT_SUBJECT, + subject=ROOT_SUBJECT, + resources=(), + data=dict(receipt, operator=operator, checkpoint_key_id=key_id(pin.public_key)), + ) + await tx.append_audit( + AuditEvent( + event=event, + authority=(), + before_digest=digest(previous), + after_digest=digest(receipt), + previous_digest=None, + entry_digest='', + result='quarantined', + ) + ) return dict(receipt, changed=changed) def selftest(): """Ephemeral signer test only; does not inspect or change any installation.""" from msg.security.crypto import Ed25519Signer + signer = Ed25519Signer.generate() - body = {'format': FORMAT, 'service': 'https://isolated.invalid', - 'source_backup_sha256': '0' * 64, 'sequence': 0, 'entries': []} - packet = {'checkpoint': body, 'signature': wire(signer.sign( - canonical(body), purpose='recovery-checkpoint-v1'))} - pin = TrustedCheckpointPin(service=body['service'], public_key=signer.public_key, - digest=digest(body), sequence=0) + body = { + 'format': FORMAT, + 'service': 'https://isolated.invalid', + 'source_backup_sha256': '0' * 64, + 'sequence': 0, + 'entries': [], + } + packet = { + 'checkpoint': body, + 'signature': wire(signer.sign(canonical(body), purpose='recovery-checkpoint-v1')), + } + pin = TrustedCheckpointPin( + service=body['service'], public_key=signer.public_key, digest=digest(body), sequence=0 + ) require(verify_checkpoint(packet, pin=pin) == body, 'recovery_selftest_failed') rejected = [] for invalid in (None, replace(pin, public_key=Ed25519Signer.generate().public_key)): @@ -379,15 +558,24 @@ def selftest(): rejected.append(exc.code) else: raise Failure('recovery_selftest_failed') - require(rejected == ['recovery_checkpoint_pin_required', 'recovery_checkpoint_invalid'], - 'recovery_selftest_failed') - return {'format': FORMAT, 'scope': 'isolated_signature_contract_only', - 'checks': 3, 'status': 'passed', 'database_used': False, - 'production_evidence': False, 'promotion': 'blocked'} + require( + rejected == ['recovery_checkpoint_pin_required', 'recovery_checkpoint_invalid'], + 'recovery_selftest_failed', + ) + return { + 'format': FORMAT, + 'scope': 'isolated_signature_contract_only', + 'checks': 3, + 'status': 'passed', + 'database_used': False, + 'production_evidence': False, + 'promotion': 'blocked', + } if __name__ == '__main__': import argparse + parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('--selftest', action='store_true', required=True) parser.parse_args() diff --git a/src/msg/admin/recovery_state.py b/src/msg/admin/recovery_state.py index 6c39552c..00f31017 100644 --- a/src/msg/admin/recovery_state.py +++ b/src/msg/admin/recovery_state.py @@ -1,7 +1,8 @@ """Complete, typed state commitments; no authority-bearing table is omitted.""" + +import hashlib from decimal import Decimal from importlib.resources import files -import hashlib from msg.core.codec import canonical, loads from msg.core.errors import require @@ -11,52 +12,106 @@ def catalogue(tx): - namespaces = tx.rows("SELECT nspname FROM pg_namespace WHERE nspname !~ '^pg_' AND nspname<>'information_schema' ORDER BY nspname") + namespaces = tx.rows( + "SELECT nspname FROM pg_namespace WHERE nspname !~ '^pg_' AND nspname<>'information_schema' ORDER BY nspname" + ) require(namespaces == [('public',)], 'recovery_schema_unknown_namespace') - relations = tx.rows("SELECT c.relname,c.relkind FROM pg_class c JOIN pg_namespace n ON n.oid=c.relnamespace WHERE n.nspname='public' ORDER BY c.relname") - require(all(kind in ('r', 'i', 'S') for _, kind in relations), 'recovery_schema_unknown_relation') - require(not tx.rows("SELECT 1 FROM pg_class c JOIN pg_namespace n ON n.oid=c.relnamespace WHERE n.nspname='public' AND (c.relrowsecurity OR c.relforcerowsecurity OR c.relpersistence<>'p')"), 'recovery_schema_unsupported') - require(not tx.rows("SELECT 1 FROM pg_policy p JOIN pg_class c ON c.oid=p.polrelid JOIN pg_namespace n ON n.oid=c.relnamespace WHERE n.nspname='public'"), 'recovery_schema_unsupported') - require(not tx.rows("SELECT 1 FROM pg_rewrite r JOIN pg_class c ON c.oid=r.ev_class JOIN pg_namespace n ON n.oid=c.relnamespace WHERE n.nspname='public'"), 'recovery_schema_unsupported') - require(not tx.rows("SELECT 1 FROM pg_collation c JOIN pg_namespace n ON n.oid=c.collnamespace WHERE n.nspname='public'"), 'recovery_schema_unsupported') - require(not tx.rows("SELECT 1 FROM pg_trigger t JOIN pg_class c ON c.oid=t.tgrelid JOIN pg_namespace n ON n.oid=c.relnamespace WHERE n.nspname='public' AND t.tgisinternal AND t.tgenabled<>'O'"), 'recovery_schema_unsupported') + relations = tx.rows( + "SELECT c.relname,c.relkind FROM pg_class c JOIN pg_namespace n ON n.oid=c.relnamespace WHERE n.nspname='public' ORDER BY c.relname" + ) + require( + all(kind in ('r', 'i', 'S') for _, kind in relations), 'recovery_schema_unknown_relation' + ) + require( + not tx.rows( + "SELECT 1 FROM pg_class c JOIN pg_namespace n ON n.oid=c.relnamespace WHERE n.nspname='public' AND (c.relrowsecurity OR c.relforcerowsecurity OR c.relpersistence<>'p')" + ), + 'recovery_schema_unsupported', + ) + require( + not tx.rows( + "SELECT 1 FROM pg_policy p JOIN pg_class c ON c.oid=p.polrelid JOIN pg_namespace n ON n.oid=c.relnamespace WHERE n.nspname='public'" + ), + 'recovery_schema_unsupported', + ) + require( + not tx.rows( + "SELECT 1 FROM pg_rewrite r JOIN pg_class c ON c.oid=r.ev_class JOIN pg_namespace n ON n.oid=c.relnamespace WHERE n.nspname='public'" + ), + 'recovery_schema_unsupported', + ) + require( + not tx.rows( + "SELECT 1 FROM pg_collation c JOIN pg_namespace n ON n.oid=c.collnamespace WHERE n.nspname='public'" + ), + 'recovery_schema_unsupported', + ) + require( + not tx.rows( + "SELECT 1 FROM pg_trigger t JOIN pg_class c ON c.oid=t.tgrelid JOIN pg_namespace n ON n.oid=c.relnamespace WHERE n.nspname='public' AND t.tgisinternal AND t.tgenabled<>'O'" + ), + 'recovery_schema_unsupported', + ) tables = [name for name, kind in relations if kind == 'r'] - result = {'tables': {}, 'constraints': [], 'indexes': [], 'triggers': [], 'functions': [], 'sequences': []} + result = { + 'tables': {}, + 'constraints': [], + 'indexes': [], + 'triggers': [], + 'functions': [], + 'sequences': [], + } for table in tables: - result['tables'][table] = tx.rows('''SELECT a.attname,format_type(a.atttypid,a.atttypmod), + result['tables'][table] = tx.rows( + """SELECT a.attname,format_type(a.atttypid,a.atttypmod), a.attnotnull,a.attidentity,pg_get_expr(d.adbin,d.adrelid), (SELECT n.nspname||'.'||co.collname FROM pg_collation co JOIN pg_namespace n ON n.oid=co.collnamespace WHERE co.oid=a.attcollation) FROM pg_attribute a JOIN pg_class c ON c.oid=a.attrelid JOIN pg_namespace n ON n.oid=c.relnamespace LEFT JOIN pg_attrdef d ON d.adrelid=c.oid AND d.adnum=a.attnum WHERE n.nspname='public' AND c.relname=? AND a.attnum>0 AND NOT a.attisdropped - ORDER BY a.attnum''', (table,)) - result['constraints'] = tx.rows('''SELECT c.relname,k.conname,k.contype,pg_get_constraintdef(k.oid),k.convalidated + ORDER BY a.attnum""", + (table,), + ) + result['constraints'] = ( + tx.rows("""SELECT c.relname,k.conname,k.contype,pg_get_constraintdef(k.oid),k.convalidated FROM pg_constraint k JOIN pg_class c ON c.oid=k.conrelid JOIN pg_namespace n ON n.oid=c.relnamespace - WHERE n.nspname='public' ORDER BY c.relname,k.conname''') - result['indexes'] = tx.rows("SELECT tablename,indexname,indexdef FROM pg_indexes WHERE schemaname='public' ORDER BY tablename,indexname") - result['triggers'] = tx.rows('''SELECT c.relname,t.tgname,t.tgenabled,pg_get_triggerdef(t.oid) + WHERE n.nspname='public' ORDER BY c.relname,k.conname""") + ) + result['indexes'] = tx.rows( + "SELECT tablename,indexname,indexdef FROM pg_indexes WHERE schemaname='public' ORDER BY tablename,indexname" + ) + result['triggers'] = tx.rows("""SELECT c.relname,t.tgname,t.tgenabled,pg_get_triggerdef(t.oid) FROM pg_trigger t JOIN pg_class c ON c.oid=t.tgrelid JOIN pg_namespace n ON n.oid=c.relnamespace - WHERE n.nspname='public' AND NOT t.tgisinternal ORDER BY c.relname,t.tgname''') - result['functions'] = tx.rows('''SELECT p.proname,pg_get_function_identity_arguments(p.oid), + WHERE n.nspname='public' AND NOT t.tgisinternal ORDER BY c.relname,t.tgname""") + result['functions'] = tx.rows("""SELECT p.proname,pg_get_function_identity_arguments(p.oid), pg_get_functiondef(p.oid) FROM pg_proc p JOIN pg_namespace n ON n.oid=p.pronamespace - WHERE n.nspname='public' ORDER BY p.proname,pg_get_function_identity_arguments(p.oid)''') - result['sequences'] = tx.rows("SELECT sequencename,data_type,start_value,min_value,max_value,increment_by,cycle FROM pg_sequences WHERE schemaname='public' ORDER BY sequencename") - result['database_collation'] = tx.one("SELECT encoding,datcollate,datctype FROM pg_database WHERE datname=current_database()") + WHERE n.nspname='public' ORDER BY p.proname,pg_get_function_identity_arguments(p.oid)""") + result['sequences'] = tx.rows( + "SELECT sequencename,data_type,start_value,min_value,max_value,increment_by,cycle FROM pg_sequences WHERE schemaname='public' ORDER BY sequencename" + ) + result['database_collation'] = tx.one( + 'SELECT encoding,datcollate,datctype FROM pg_database WHERE datname=current_database()' + ) return loads(canonical(result)) def shape(catalog): """Stable allowed object/column vocabulary, independent of PG deparser whitespace.""" - return {'tables': {table: [column[:4] for column in columns] for table, columns in catalog['tables'].items()}, - # PG18 additionally catalogs NOT NULL as contype=n. attnotnull above - # already pins each column; keep fixed vocabulary portable to PG16. - # Full signed catalogue still includes every constraint definition. - 'constraints': [row[:3] for row in catalog['constraints'] if row[2] != 'n'], - 'indexes': [row[:2] for row in catalog['indexes']], - 'triggers': [row[:2] for row in catalog['triggers']], - 'functions': [row[:2] for row in catalog['functions']], - 'sequences': [row[:2] for row in catalog['sequences']]} + return { + 'tables': { + table: [column[:4] for column in columns] + for table, columns in catalog['tables'].items() + }, + # PG18 additionally catalogs NOT NULL as contype=n. attnotnull above + # already pins each column; keep fixed vocabulary portable to PG16. + # Full signed catalogue still includes every constraint definition. + 'constraints': [row[:3] for row in catalog['constraints'] if row[2] != 'n'], + 'indexes': [row[:2] for row in catalog['indexes']], + 'triggers': [row[:2] for row in catalog['triggers']], + 'functions': [row[:2] for row in catalog['functions']], + 'sequences': [row[:2] for row in catalog['sequences']], + } def known_catalogue(tx): @@ -86,17 +141,30 @@ def runtime_controls(tx): current = tx.setting('runtime_config') require(current is None or type(current) is dict, 'recovery_runtime_invalid') values = current or {} - require(all(key not in values or type(values[key]) is bool for key in PAUSE_FIELDS), 'recovery_runtime_invalid') - return {'present': current is not None, - 'fields': {key: {'present': key in values, 'value': values.get(key, False)} for key in PAUSE_FIELDS}} + require( + all(key not in values or type(values[key]) is bool for key in PAUSE_FIELDS), + 'recovery_runtime_invalid', + ) + return { + 'present': current is not None, + 'fields': { + key: {'present': key in values, 'value': values.get(key, False)} for key in PAUSE_FIELDS + }, + } def _runtime_row(row, controls): value = loads(row[1]) - require(type(value) is dict and all(value.get(key) is False for key in PAUSE_FIELDS), 'recovery_runtime_not_paused') + require( + type(value) is dict and all(value.get(key) is False for key in PAUSE_FIELDS), + 'recovery_runtime_not_paused', + ) for key in PAUSE_FIELDS: wanted = controls['fields'][key] - require(type(wanted['present']) is bool and type(wanted['value']) is bool, 'recovery_runtime_invalid') + require( + type(wanted['present']) is bool and type(wanted['value']) is bool, + 'recovery_runtime_invalid', + ) value.pop(key) if wanted['present']: value[key] = wanted['value'] @@ -108,22 +176,36 @@ def _runtime_row(row, controls): def snapshot(tx, *, controls=None, resume=None): if resume is not None: - require(tx.setting('recovery_runtime_generation') == resume['generation'], - 'recovery_promotion_receipt_mismatch') + require( + tx.setting('recovery_runtime_generation') == resume['generation'], + 'recovery_promotion_receipt_mismatch', + ) catalog = known_catalogue(tx) result = {'catalogue': catalog, 'tables': {}, 'sequences': {}} # The application writer lock plus these relation locks serialize every DB # authority/effect change until capture or promotion commits. - tx.execute('LOCK TABLE ' + ','.join('public.' + quote(name) for name in catalog['tables']) + ' IN ACCESS EXCLUSIVE MODE', write=True) + tx.execute( + 'LOCK TABLE ' + + ','.join('public.' + quote(name) for name in catalog['tables']) + + ' IN ACCESS EXCLUSIVE MODE', + write=True, + ) for table, columns in catalog['tables'].items(): - rows = tx.rows('SELECT ' + ','.join(quote(column[0]) for column in columns) + ' FROM public.' + quote(table)) + rows = tx.rows( + 'SELECT ' + + ','.join(quote(column[0]) for column in columns) + + ' FROM public.' + + quote(table) + ) if table == 'settings': kept = [] for row in rows: if row[0] in CONTROL_SETTINGS: continue if resume is not None and row[0] == 'recovery_runtime_generation': - require(loads(row[1]) == resume['generation'], 'recovery_promotion_receipt_mismatch') + require( + loads(row[1]) == resume['generation'], 'recovery_promotion_receipt_mismatch' + ) if resume['prior_generation'] is None: continue row = (row[0], canonical(resume['prior_generation']).decode()) @@ -134,7 +216,10 @@ def snapshot(tx, *, controls=None, resume=None): rows = kept if resume is not None and table == 'audit': audit = resume['audit_row'] - require(sum(loads(canonical(row)) == audit for row in rows) == 1, 'recovery_promotion_receipt_mismatch') + require( + sum(loads(canonical(row)) == audit for row in rows) == 1, + 'recovery_promotion_receipt_mismatch', + ) require(max(row[0] for row in rows) == audit[0], 'recovery_promotion_state_changed') rows = [row for row in rows if row[0] != audit[0]] encoded = sorted(canonical([cell(value) for value in row]) for row in rows) @@ -146,7 +231,10 @@ def snapshot(tx, *, controls=None, resume=None): for name, *_ in catalog['sequences']: state = loads(canonical(tx.one('SELECT last_value,is_called FROM public.' + quote(name)))) if resume is not None and name == 'audit_seq_seq': - require(state in (resume['prior_audit_sequence'], [resume['audit_row'][0], True]), 'recovery_promotion_state_changed') + require( + state in (resume['prior_audit_sequence'], [resume['audit_row'][0], True]), + 'recovery_promotion_state_changed', + ) state = resume['prior_audit_sequence'] result['sequences'][name] = state return result diff --git a/src/msg/admin/restore_database.py b/src/msg/admin/restore_database.py index 7b7886c8..b7611ea2 100644 --- a/src/msg/admin/restore_database.py +++ b/src/msg/admin/restore_database.py @@ -1,7 +1,8 @@ """Import an offline dump and its quarantine gate in the same PostgreSQL commit.""" -from pathlib import Path + import os import subprocess +from pathlib import Path from msg.core.codec import canonical from msg.security.quarantine import SETTING @@ -17,21 +18,48 @@ def restore_dump(dump, *, safe_dsn, env, quarantine): script = Path(dump).parent / 'quarantined-restore.sql' script.touch(mode=0o600, exist_ok=False) try: - subprocess.run(['pg_restore', '--exit-on-error', '--no-owner', '--no-acl', - '--file', str(script), str(dump)], - env=env, check=True, capture_output=True) + subprocess.run( + [ + 'pg_restore', + '--exit-on-error', + '--no-owner', + '--no-acl', + '--file', + str(script), + str(dump), + ], + env=env, + check=True, + capture_output=True, + ) # Values are fixed recovery metadata, not secrets. Escape as a SQL # literal (including backslashes) independently of session string flags. value = canonical(quarantine).decode().replace('\\', '\\\\').replace("'", "''") - gate = ("\nSET search_path = public, pg_catalog;\n" - f"INSERT INTO settings(key,value) VALUES('{SETTING}',E'{value}') " - "ON CONFLICT(key) DO UPDATE SET value=EXCLUDED.value;\n") + gate = ( + '\nSET search_path = public, pg_catalog;\n' + f"INSERT INTO settings(key,value) VALUES('{SETTING}',E'{value}') " + 'ON CONFLICT(key) DO UPDATE SET value=EXCLUDED.value;\n' + ) with script.open('ab') as stream: stream.write(gate.encode()) stream.flush() os.fsync(stream.fileno()) - subprocess.run(['psql', '--no-psqlrc', '--quiet', '--single-transaction', - '--set', 'ON_ERROR_STOP=1', '--dbname', safe_dsn, '--file', str(script)], - env=env, check=True, capture_output=True) + subprocess.run( + [ + 'psql', + '--no-psqlrc', + '--quiet', + '--single-transaction', + '--set', + 'ON_ERROR_STOP=1', + '--dbname', + safe_dsn, + '--file', + str(script), + ], + env=env, + check=True, + capture_output=True, + ) finally: script.unlink(missing_ok=True) diff --git a/src/msg/admin/root.py b/src/msg/admin/root.py index 1945f7f4..6788269b 100644 --- a/src/msg/admin/root.py +++ b/src/msg/admin/root.py @@ -4,141 +4,278 @@ entrypoint must pass OS-console checks before calling them. Tests call these use cases only against disposable directories and never enable a network root route. """ + from __future__ import annotations import asyncio -from dataclasses import replace -from datetime import timedelta import getpass import os -from pathlib import Path import re import sys +from dataclasses import replace +from datetime import timedelta +from pathlib import Path -from msg.constants import * -from msg.core.codec import b64,unb64,canonical,decode,digest,loads,wire -from msg.core.errors import Failure,require -from msg.core.models import Certificate,CertificateRequest,Credential,IssuancePolicy,Signature,ResourceRef,Event,AuditEvent -from msg.security.crypto import Ed25519Signer,seal_private_key,open_private_key -from msg.security.certificates import sign_certificate,csr_body,verify_csr -from msg.security.capabilities import grant_for -from msg.storage.git import durable_write -from msg.security.trust_files import trust_file, write_trust, reserved_plugins_directory -from msg.bootstrap import bootstrap,seed_resource +from msg.bootstrap import bootstrap, seed_resource +from msg.constants import ( + ADMINS_GROUP as ADMINS_GROUP, + CA_SPACE as CA_SPACE, + CERT_SPACE as CERT_SPACE, + CSR_SPACE as CSR_SPACE, + ONLINE_CA as ONLINE_CA, + PROTOCOL_VERSION as PROTOCOL_VERSION, + PUBLIC_GROUP as PUBLIC_GROUP, + ROOT_SPACE as ROOT_SPACE, + ROOT_SUBJECT as ROOT_SUBJECT, + TOOLS_SPACE as TOOLS_SPACE, +) +from msg.core.codec import b64, canonical, digest, loads, unb64, wire +from msg.core.errors import require +from msg.core.models import ( + AuditEvent, + Certificate, + CertificateRequest, + Credential, + Event, + IssuancePolicy, + ResourceRef, + Signature, +) from msg.plugins.common import new_id from msg.plugins.identity import certificate_resource +from msg.security.capabilities import grant_for +from msg.security.certificates import csr_body, sign_certificate, verify_csr +from msg.security.crypto import Ed25519Signer, open_private_key, seal_private_key +from msg.security.trust_files import reserved_plugins_directory, trust_file, write_trust +from msg.storage.git import durable_write def root_envelope(config_dir): from msg.config import root_private_dir - current=root_private_dir(Path(config_dir))/'key.json' - legacy=Path(config_dir)/'root'/'key.json' + + current = root_private_dir(Path(config_dir)) / 'key.json' + legacy = Path(config_dir) / 'root' / 'key.json' return legacy if legacy.is_file() and not current.exists() else current -async def _provision(app,pin): +async def _provision(app, pin): """Initialize only an empty installation; partial state requires explicit recovery.""" - settings=app.settings - protected=settings.root_private_dir - marker=settings.config_dir/'initialization.pending' - require(not root_envelope(settings.config_dir).exists() and not settings.trust_file.exists() and - not marker.exists(),'initialization_requires_recovery') + settings = app.settings + protected = settings.root_private_dir + marker = settings.config_dir / 'initialization.pending' + require( + not root_envelope(settings.config_dir).exists() + and not settings.trust_file.exists() + and not marker.exists(), + 'initialization_requires_recovery', + ) await app.open_storage() async with app.metadata.transaction(write=False) as tx: - require(tx.one('SELECT COUNT(*) FROM resources')[0] == 0,'initialization_requires_recovery') - root=Ed25519Signer.generate() - envelope=seal_private_key(root.private_bytes(),pin) - settings.config_dir.mkdir(parents=True,exist_ok=True) - reserved_plugins_directory(settings.config_dir,create=True) - durable_write(marker,b'1\n',mode=0o600) - protected.mkdir(mode=0o700,parents=True,exist_ok=True) - os.chmod(protected,0o700) - settings.service_keys.mkdir(mode=0o700,parents=True,exist_ok=True) - os.chmod(settings.service_keys,0o700) - settings.trust_file.parent.mkdir(mode=0o755,parents=True,exist_ok=True) + require( + tx.one('SELECT COUNT(*) FROM resources')[0] == 0, 'initialization_requires_recovery' + ) + root = Ed25519Signer.generate() + envelope = seal_private_key(root.private_bytes(), pin) + settings.config_dir.mkdir(parents=True, exist_ok=True) + reserved_plugins_directory(settings.config_dir, create=True) + durable_write(marker, b'1\n', mode=0o600) + protected.mkdir(mode=0o700, parents=True, exist_ok=True) + os.chmod(protected, 0o700) + settings.service_keys.mkdir(mode=0o700, parents=True, exist_ok=True) + os.chmod(settings.service_keys, 0o700) + settings.trust_file.parent.mkdir(mode=0o755, parents=True, exist_ok=True) # Wrapped before touching the installation; a rejected PIN leaves no partial state. - durable_write(root_envelope(settings.config_dir),canonical(envelope),mode=0o600) - online=Ed25519Signer.generate() - receipt=Ed25519Signer.generate() - durable_write(settings.service_keys/'online.key',online.private_bytes(),mode=0o600) - durable_write(settings.service_keys/'receipt.key',receipt.private_bytes(),mode=0o600) - durable_write(settings.service_keys/'tokens.key',os.urandom(32),mode=0o600) - now=app.clock() - await bootstrap(app.metadata,app.contents,app.registry,now, - selftest_run_id=app.selftest_run_id) - root_grants=app.primary_ceiling() - root_certificate=Certificate(resource_id='cert_root',serial=new_id('serial'),subject_id=ROOT_SUBJECT, - key_id=root.key_id,issuer_id=ROOT_SUBJECT,parent_certificate_id=None,authority_sources=(),kind='ca', - grants=root_grants,not_before=now,expires_at=now+timedelta(days=3650),target_service=settings.service_url, - delegation_depth=8,issuance=IssuancePolicy(issue_grants=root_grants,max_cert_ttl_seconds=31536000, - max_child_ca_depth=3,max_delegation_depth=8),signature=Signature(key_id=root.key_id,algorithm='ed25519',value=b'')) - root_certificate=sign_certificate(root_certificate,root) - online_use=grant_for(app.registry.capability('cert.issue'),scope=app.default_scope()) - csr=CertificateRequest(resource_id=new_id('csr'),applicant=ONLINE_CA,subject_id=ONLINE_CA, - requested_issuer=ROOT_SUBJECT,public_key=online.public_key,kind='ca',grants=(online_use,), - issuance=IssuancePolicy(issue_grants=app.base_grants(),max_cert_ttl_seconds=settings.base_certificate_ttl, - max_child_ca_depth=0,max_delegation_depth=8),requested_ttl_seconds=31536000, - target_service=settings.service_url,delegation_depth=0,authority_sources=(),request_digest='', - possession_proof=Signature(key_id=online.key_id,algorithm='ed25519',value=b'')) - csr=replace(csr,request_digest=digest(csr_body(csr)), - possession_proof=online.sign(canonical(csr_body(csr)),purpose='csr')) + durable_write(root_envelope(settings.config_dir), canonical(envelope), mode=0o600) + online = Ed25519Signer.generate() + receipt = Ed25519Signer.generate() + durable_write(settings.service_keys / 'online.key', online.private_bytes(), mode=0o600) + durable_write(settings.service_keys / 'receipt.key', receipt.private_bytes(), mode=0o600) + durable_write(settings.service_keys / 'tokens.key', os.urandom(32), mode=0o600) + now = app.clock() + await bootstrap( + app.metadata, app.contents, app.registry, now, selftest_run_id=app.selftest_run_id + ) + root_grants = app.primary_ceiling() + root_certificate = Certificate( + resource_id='cert_root', + serial=new_id('serial'), + subject_id=ROOT_SUBJECT, + key_id=root.key_id, + issuer_id=ROOT_SUBJECT, + parent_certificate_id=None, + authority_sources=(), + kind='ca', + grants=root_grants, + not_before=now, + expires_at=now + timedelta(days=3650), + target_service=settings.service_url, + delegation_depth=8, + issuance=IssuancePolicy( + issue_grants=root_grants, + max_cert_ttl_seconds=31536000, + max_child_ca_depth=3, + max_delegation_depth=8, + ), + signature=Signature(key_id=root.key_id, algorithm='ed25519', value=b''), + ) + root_certificate = sign_certificate(root_certificate, root) + online_use = grant_for(app.registry.capability('cert.issue'), scope=app.default_scope()) + csr = CertificateRequest( + resource_id=new_id('csr'), + applicant=ONLINE_CA, + subject_id=ONLINE_CA, + requested_issuer=ROOT_SUBJECT, + public_key=online.public_key, + kind='ca', + grants=(online_use,), + issuance=IssuancePolicy( + issue_grants=app.base_grants(), + max_cert_ttl_seconds=settings.base_certificate_ttl, + max_child_ca_depth=0, + max_delegation_depth=8, + ), + requested_ttl_seconds=31536000, + target_service=settings.service_url, + delegation_depth=0, + authority_sources=(), + request_digest='', + possession_proof=Signature(key_id=online.key_id, algorithm='ed25519', value=b''), + ) + csr = replace( + csr, + request_digest=digest(csr_body(csr)), + possession_proof=online.sign(canonical(csr_body(csr)), purpose='csr'), + ) async with app.metadata.transaction(write=True) as tx: - for uid,signer,ceiling in ((ROOT_SUBJECT,root,()),(ONLINE_CA,online,(online_use,))): - await tx.save_credential(Credential(id=signer.key_id,subject_id=uid,kind='signing_key',verifier=signer.public_key, - ceiling=ceiling,not_before=now,expires_at=None,revoked_at=None),0) - for name in ('keys','certificates'): - await seed_resource(tx,app.contents,dict(id='r_'+digest((uid,name))[7:39],type='topic',name=name,parent=uid, - owner=uid,group=PUBLIC_GROUP,mode='0555'),now) - await certificate_resource(tx,root_certificate,now) - await tx.register_certificate(root_certificate,None,0) - await seed_resource(tx,app.contents,dict(id=csr.resource_id,type='csr',name=csr.resource_id,parent=CSR_SPACE, - owner=ONLINE_CA,group=ADMINS_GROUP,mode='0600'),now) + for uid, signer, ceiling in ((ROOT_SUBJECT, root, ()), (ONLINE_CA, online, (online_use,))): + await tx.save_credential( + Credential( + id=signer.key_id, + subject_id=uid, + kind='signing_key', + verifier=signer.public_key, + ceiling=ceiling, + not_before=now, + expires_at=None, + revoked_at=None, + ), + 0, + ) + for name in ('keys', 'certificates'): + await seed_resource( + tx, + app.contents, + { + 'id': 'r_' + digest((uid, name))[7:39], + 'type': 'topic', + 'name': name, + 'parent': uid, + 'owner': uid, + 'group': PUBLIC_GROUP, + 'mode': '0555', + }, + now, + ) + await certificate_resource(tx, root_certificate, now) + await tx.register_certificate(root_certificate, None, 0) + await seed_resource( + tx, + app.contents, + { + 'id': csr.resource_id, + 'type': 'csr', + 'name': csr.resource_id, + 'parent': CSR_SPACE, + 'owner': ONLINE_CA, + 'group': ADMINS_GROUP, + 'mode': '0600', + }, + now, + ) await tx.save_csr(csr) - tx.set_setting('active_root_certificate',root_certificate.resource_id) - tx.set_setting('online_ca_request',csr.resource_id) - tx.set_setting('receipt_public_key',{'key_id':receipt.key_id,'public_key':b64(receipt.public_key)}) - write_trust(settings.config_dir,{'version':1,'public_key':b64(root.public_key), - 'certificate':wire(root_certificate)},writer=durable_write) + tx.set_setting('active_root_certificate', root_certificate.resource_id) + tx.set_setting('online_ca_request', csr.resource_id) + tx.set_setting( + 'receipt_public_key', {'key_id': receipt.key_id, 'public_key': b64(receipt.public_key)} + ) + write_trust( + settings.config_dir, + {'version': 1, 'public_key': b64(root.public_key), 'certificate': wire(root_certificate)}, + writer=durable_write, + ) marker.unlink() await app.load() - return csr.resource_id,root + return csr.resource_id, root -async def _approve_csr(app,csr_id,signer, *, expected_digest,operator): - now=app.clock() +async def _approve_csr(app, csr_id, signer, *, expected_digest, operator): + now = app.clock() async with app.metadata.transaction(write=True) as tx: - csr=await tx.csr(csr_id) - state=await tx.csr_state(csr_id) - require(csr.requested_issuer==ROOT_SUBJECT,'not_root_request') + csr = await tx.csr(csr_id) + state = await tx.csr_state(csr_id) + require(csr.requested_issuer == ROOT_SUBJECT, 'not_root_request') if expected_digest is not None: - require(csr.request_digest==expected_digest,'csr_changed_after_confirmation') + require(csr.request_digest == expected_digest, 'csr_changed_after_confirmation') verify_csr(csr) - if state.status=='issued': + if state.status == 'issued': return await tx.certificate(state.certificate_id) - require(state.status=='pending','csr_not_pending') - require(signer.key_id==app.certificates.root_certificate.key_id,'root_key_mismatch') - parent=app.certificates.root_certificate - certificate=Certificate(resource_id=new_id('cert'),serial=new_id('serial'),subject_id=csr.subject_id, - key_id=csr.possession_proof.key_id,issuer_id=ROOT_SUBJECT,parent_certificate_id=parent.resource_id, - authority_sources=csr.authority_sources,kind=csr.kind,grants=csr.grants,not_before=now, - expires_at=min(now+timedelta(seconds=csr.requested_ttl_seconds),parent.expires_at), - target_service=csr.target_service,delegation_depth=csr.delegation_depth,issuance=csr.issuance, - signature=Signature(key_id=signer.key_id,algorithm='ed25519',value=b'')) - certificate=sign_certificate(certificate,signer) - await certificate_resource(tx,certificate,now) - await app.certificates.validate_publication(certificate,csr,tx) - await tx.register_certificate(certificate,csr_id,state.generation) - credential=await tx.credential(certificate.key_id) + require(state.status == 'pending', 'csr_not_pending') + require(signer.key_id == app.certificates.root_certificate.key_id, 'root_key_mismatch') + parent = app.certificates.root_certificate + certificate = Certificate( + resource_id=new_id('cert'), + serial=new_id('serial'), + subject_id=csr.subject_id, + key_id=csr.possession_proof.key_id, + issuer_id=ROOT_SUBJECT, + parent_certificate_id=parent.resource_id, + authority_sources=csr.authority_sources, + kind=csr.kind, + grants=csr.grants, + not_before=now, + expires_at=min(now + timedelta(seconds=csr.requested_ttl_seconds), parent.expires_at), + target_service=csr.target_service, + delegation_depth=csr.delegation_depth, + issuance=csr.issuance, + signature=Signature(key_id=signer.key_id, algorithm='ed25519', value=b''), + ) + certificate = sign_certificate(certificate, signer) + await certificate_resource(tx, certificate, now) + await app.certificates.validate_publication(certificate, csr, tx) + await tx.register_certificate(certificate, csr_id, state.generation) + credential = await tx.credential(certificate.key_id) if not credential.ceiling: - subject=await tx.subject(credential.subject_id) - await tx.save_credential(replace(credential,ceiling=certificate.grants),subject.auth_version) - if csr_id==tx.setting('online_ca_request'): - tx.set_setting('online_ca_certificate',certificate.resource_id) - event=Event(id=new_id('audit'),type='root.cert.issue',time=now,request_id=csr_id, - actor=ROOT_SUBJECT,subject=ROOT_SUBJECT,resources=(ResourceRef(id=certificate.resource_id),), - data={'operator':operator,'csr_digest':csr.request_digest,'grants':wire(certificate.grants)}) - await tx.append_audit(AuditEvent(event=event,authority=(ResourceRef(id=parent.resource_id),), - before_digest=csr.request_digest,after_digest=digest(certificate),previous_digest=None,entry_digest='',result='issued')) + subject = await tx.subject(credential.subject_id) + await tx.save_credential( + replace(credential, ceiling=certificate.grants), subject.auth_version + ) + if csr_id == tx.setting('online_ca_request'): + tx.set_setting('online_ca_certificate', certificate.resource_id) + event = Event( + id=new_id('audit'), + type='root.cert.issue', + time=now, + request_id=csr_id, + actor=ROOT_SUBJECT, + subject=ROOT_SUBJECT, + resources=(ResourceRef(id=certificate.resource_id),), + data={ + 'operator': operator, + 'csr_digest': csr.request_digest, + 'grants': wire(certificate.grants), + }, + ) + await tx.append_audit( + AuditEvent( + event=event, + authority=(ResourceRef(id=parent.resource_id),), + before_digest=csr.request_digest, + after_digest=digest(certificate), + previous_digest=None, + entry_digest='', + result='issued', + ) + ) return certificate @@ -150,249 +287,449 @@ def require_local_console(config_dir): Deployments that need another administrative channel must use a separately audited OS policy, not relax this check through the network configuration. """ - require(hasattr(os,'geteuid') and os.geteuid()==0,'local_os_administrator_required') - require(not any(name in os.environ for name in ('SSH_CONNECTION','SSH_CLIENT','SSH_TTY')),'remote_admin_forbidden') - require(sys.stdin.isatty() and sys.stdout.isatty(),'local_console_required') - tty=os.path.realpath(os.ttyname(sys.stdin.fileno())) - require(re.fullmatch(r'/dev/(console|tty[0-9]+|ttyS[0-9]+|hvc[0-9]+)',tty) is not None, - 'local_console_required') - path=Path(config_dir).resolve() - require(path.exists() and path.stat().st_uid==0 and not path.stat().st_mode&0o022,'unsafe_config_owner') + require(hasattr(os, 'geteuid') and os.geteuid() == 0, 'local_os_administrator_required') + require( + not any(name in os.environ for name in ('SSH_CONNECTION', 'SSH_CLIENT', 'SSH_TTY')), + 'remote_admin_forbidden', + ) + require(sys.stdin.isatty() and sys.stdout.isatty(), 'local_console_required') + tty = os.path.realpath(os.ttyname(sys.stdin.fileno())) + require( + re.fullmatch(r'/dev/(console|tty[0-9]+|ttyS[0-9]+|hvc[0-9]+)', tty) is not None, + 'local_console_required', + ) + path = Path(config_dir).resolve() + require( + path.exists() and path.stat().st_uid == 0 and not path.stat().st_mode & 0o022, + 'unsafe_config_owner', + ) return f'uid:{os.geteuid()}:{tty}' class RootAdmin: - def __init__(self,config_dir=Path('/etc/msgd')): - self.config_dir=Path(config_dir) + def __init__(self, config_dir=Path('/etc/msgd')): + self.config_dir = Path(config_dir) def _app(self): from msg.application import Application from msg.config import load_settings + return Application(load_settings(self.config_dir)) def initialize(self): require_local_console(self.config_dir) - app=self._app() + app = self._app() if root_envelope(self.config_dir).exists() and app.settings.trust_file.exists(): - status=self.doctor() - require(all(status['checks'].get(k,{}).get('ok') for k in ('storage','root_trust','bootstrap','root_private_boundary')), - 'initialization_requires_recovery') - private=open_private_key(loads(root_envelope(self.config_dir).read_bytes()),getpass.getpass('Current root PIN/passphrase: ')) - trust=loads(app.settings.trust_file.read_bytes()) - require(Ed25519Signer.from_bytes(private).public_key==unb64(trust['public_key']),'root_key_mismatch') + status = self.doctor() + require( + all( + status['checks'].get(k, {}).get('ok') + for k in ('storage', 'root_trust', 'bootstrap', 'root_private_boundary') + ), + 'initialization_requires_recovery', + ) + private = open_private_key( + loads(root_envelope(self.config_dir).read_bytes()), + getpass.getpass('Current root PIN/passphrase: '), + ) + trust = loads(app.settings.trust_file.read_bytes()) + require( + Ed25519Signer.from_bytes(private).public_key == unb64(trust['public_key']), + 'root_key_mismatch', + ) return status['root_id'] - first=getpass.getpass('New root PIN/passphrase: ') - second=getpass.getpass('Confirm root PIN/passphrase: ') - require(first==second,'pin_confirmation_mismatch') - csr,_=asyncio.run(_provision(app,first)) - print(canonical({'status':'initialized','root_id':ROOT_SUBJECT,'online_ca_request':csr}).decode()) + first = getpass.getpass('New root PIN/passphrase: ') + second = getpass.getpass('Confirm root PIN/passphrase: ') + require(first == second, 'pin_confirmation_mismatch') + csr, _ = asyncio.run(_provision(app, first)) + print( + canonical({ + 'status': 'initialized', + 'root_id': ROOT_SUBJECT, + 'online_ca_request': csr, + }).decode() + ) return ROOT_SUBJECT - def issue(self,csr_id): - operator=require_local_console(self.config_dir) - app=self._app() + def issue(self, csr_id): + operator = require_local_console(self.config_dir) + app = self._app() + async def read(): await app.load() async with app.metadata.transaction(write=False) as tx: - return await tx.csr(csr_id),await tx.csr_state(csr_id) - csr,state=asyncio.run(read()) + return await tx.csr(csr_id), await tx.csr_state(csr_id) + + csr, state = asyncio.run(read()) # json.dumps escapes terminal controls; no raw user-controlled text is printed. - print(canonical({'request':wire(csr),'state':wire(state),'public_key_fingerprint':digest(csr.public_key)}).decode()) - if state.status=='issued': + print( + canonical({ + 'request': wire(csr), + 'state': wire(state), + 'public_key_fingerprint': digest(csr.public_key), + }).decode() + ) + if state.status == 'issued': + async def previous(): async with app.metadata.transaction(write=False) as tx: return await tx.certificate(state.certificate_id) + return asyncio.run(previous()) - require(input('Type the exact CSR digest to approve: ').strip()==csr.request_digest,'approval_cancelled') - pin=getpass.getpass('Root PIN/passphrase: ') - private=open_private_key(loads(root_envelope(self.config_dir).read_bytes()),pin) - signer=Ed25519Signer.from_bytes(private) - return asyncio.run(_approve_csr(app,csr_id,signer,expected_digest=csr.request_digest,operator=operator)) + require( + input('Type the exact CSR digest to approve: ').strip() == csr.request_digest, + 'approval_cancelled', + ) + pin = getpass.getpass('Root PIN/passphrase: ') + private = open_private_key(loads(root_envelope(self.config_dir).read_bytes()), pin) + signer = Ed25519Signer.from_bytes(private) + return asyncio.run( + _approve_csr(app, csr_id, signer, expected_digest=csr.request_digest, operator=operator) + ) def change_pin(self): require_local_console(self.config_dir) - require(input('Type CHANGE PIN to continue: ')=='CHANGE PIN','approval_cancelled') - path=root_envelope(self.config_dir) - private=open_private_key(loads(path.read_bytes()),getpass.getpass('Current root PIN/passphrase: ')) - first=getpass.getpass('New root PIN/passphrase: ') - require(first==getpass.getpass('Confirm new root PIN/passphrase: '),'pin_confirmation_mismatch') - durable_write(path,canonical(seal_private_key(private,first)),mode=0o600) + require(input('Type CHANGE PIN to continue: ') == 'CHANGE PIN', 'approval_cancelled') + path = root_envelope(self.config_dir) + private = open_private_key( + loads(path.read_bytes()), getpass.getpass('Current root PIN/passphrase: ') + ) + first = getpass.getpass('New root PIN/passphrase: ') + require( + first == getpass.getpass('Confirm new root PIN/passphrase: '), + 'pin_confirmation_mismatch', + ) + durable_write(path, canonical(seal_private_key(private, first)), mode=0o600) def rotate(self, *, lost_key=False, resume=False): - operator=require_local_console(self.config_dir) - app=self._app() - from msg.admin.rotation import prepare,complete,journal_path + operator = require_local_console(self.config_dir) + app = self._app() + from msg.admin.rotation import complete, journal_path, prepare + if resume: - require(journal_path(app).is_file(),'rotation_journal_missing') - journal=loads(journal_path(app).read_bytes()) + require(journal_path(app).is_file(), 'rotation_journal_missing') + journal = loads(journal_path(app).read_bytes()) print(canonical(journal['statement']).decode()) - require(input('Type the new root fingerprint to resume: ')==journal['statement']['new_fingerprint'],'approval_cancelled') - pin=getpass.getpass('New root PIN/passphrase: ') + require( + input('Type the new root fingerprint to resume: ') + == journal['statement']['new_fingerprint'], + 'approval_cancelled', + ) + pin = getpass.getpass('New root PIN/passphrase: ') else: asyncio.run(app.load()) - print(canonical({'old_fingerprint':digest(app.certificates.root_public_key), - 'effect':'all existing certificate chains require reissuance','lost_key':lost_key}).decode()) - require(input('Type ROTATE ROOT '+digest(app.certificates.root_public_key)+': ')== - 'ROTATE ROOT '+digest(app.certificates.root_public_key),'approval_cancelled') - old=None if lost_key else Ed25519Signer.from_bytes(open_private_key( - loads(root_envelope(self.config_dir).read_bytes()),getpass.getpass('Current root PIN/passphrase: '))) - pin=getpass.getpass('New root PIN/passphrase: ') - require(pin==getpass.getpass('Confirm new root PIN/passphrase: '),'pin_confirmation_mismatch') - journal=prepare(app,Ed25519Signer.generate(),pin,old_signer=old,operator=operator) - return asyncio.run(complete(app,journal,pin=pin)) + print( + canonical({ + 'old_fingerprint': digest(app.certificates.root_public_key), + 'effect': 'all existing certificate chains require reissuance', + 'lost_key': lost_key, + }).decode() + ) + require( + input('Type ROTATE ROOT ' + digest(app.certificates.root_public_key) + ': ') + == 'ROTATE ROOT ' + digest(app.certificates.root_public_key), + 'approval_cancelled', + ) + old = ( + None + if lost_key + else Ed25519Signer.from_bytes( + open_private_key( + loads(root_envelope(self.config_dir).read_bytes()), + getpass.getpass('Current root PIN/passphrase: '), + ) + ) + ) + pin = getpass.getpass('New root PIN/passphrase: ') + require( + pin == getpass.getpass('Confirm new root PIN/passphrase: '), + 'pin_confirmation_mismatch', + ) + journal = prepare(app, Ed25519Signer.generate(), pin, old_signer=old, operator=operator) + return asyncio.run(complete(app, journal, pin=pin)) def doctor(self): # Imported lazily: this read-only diagnostic must not run schema migrations. from msg.admin.diagnostics import doctor + return doctor(self.config_dir) def selftest(self): from msg.admin.diagnostics import selftest - return asyncio.run(selftest()) + return asyncio.run(selftest()) - def revoke(self,certificate_id,reason): - operator=require_local_console(self.config_dir) - app=self._app() + def revoke(self, certificate_id, reason): + operator = require_local_console(self.config_dir) + app = self._app() asyncio.run(app.load()) - async def read(): - async with app.metadata.transaction(write=False) as tx:return await tx.certificate(certificate_id) - certificate=asyncio.run(read()) - print(canonical({'certificate':wire(certificate),'reason':reason}).decode()) - require(input('Type REVOKE '+certificate_id+' to continue: ')=='REVOKE '+certificate_id,'approval_cancelled') - private=open_private_key(loads(root_envelope(self.config_dir).read_bytes()),getpass.getpass('Root PIN/passphrase: ')) - return asyncio.run(_revoke(app,certificate_id,Ed25519Signer.from_bytes(private),reason=reason,operator=operator)) - - def sign_recovery_proof(self,source_backup_sha256,sequence,destination): + async def read(): + async with app.metadata.transaction(write=False) as tx: + return await tx.certificate(certificate_id) + + certificate = asyncio.run(read()) + print(canonical({'certificate': wire(certificate), 'reason': reason}).decode()) + require( + input('Type REVOKE ' + certificate_id + ' to continue: ') == 'REVOKE ' + certificate_id, + 'approval_cancelled', + ) + private = open_private_key( + loads(root_envelope(self.config_dir).read_bytes()), + getpass.getpass('Root PIN/passphrase: '), + ) + return asyncio.run( + _revoke( + app, + certificate_id, + Ed25519Signer.from_bytes(private), + reason=reason, + operator=operator, + ) + ) + + def sign_recovery_proof(self, source_backup_sha256, sequence, destination): require_local_console(self.config_dir) - from msg.admin.recovery_proof import open_for_proof, draft, seal from msg.admin.backup_retirement import write_record + from msg.admin.recovery_proof import draft, open_for_proof, seal from msg.security.root_files import read_private - app=asyncio.run(open_for_proof(self.config_dir)) + + app = asyncio.run(open_for_proof(self.config_dir)) try: - body=asyncio.run(draft(app,public_key=app.certificates.root_public_key, - source_backup_sha256=source_backup_sha256,sequence=sequence)) - fingerprint=digest(body) - print(canonical({'service':body['service'],'sequence':sequence, - 'source_backup_sha256':source_backup_sha256,'state_digest':fingerprint, - 'tables':len(body['metadata']['tables']),'expires_at':body['expires_at']}).decode()) - require(input('Type SIGN COMPLETE RECOVERY '+fingerprint+': ')== - 'SIGN COMPLETE RECOVERY '+fingerprint,'approval_cancelled') - envelope=loads(read_private(root_envelope(self.config_dir))) - signer=Ed25519Signer.from_bytes(open_private_key(envelope,getpass.getpass('Root PIN/passphrase: '))) - packet=asyncio.run(seal(app,body,signer)) - write_record(Path(destination),packet) - return {'status':'complete_recovery_state_signed','digest':fingerprint, - 'independent_pin_required':True,'path':str(destination)} + body = asyncio.run( + draft( + app, + public_key=app.certificates.root_public_key, + source_backup_sha256=source_backup_sha256, + sequence=sequence, + ) + ) + fingerprint = digest(body) + print( + canonical({ + 'service': body['service'], + 'sequence': sequence, + 'source_backup_sha256': source_backup_sha256, + 'state_digest': fingerprint, + 'tables': len(body['metadata']['tables']), + 'expires_at': body['expires_at'], + }).decode() + ) + require( + input('Type SIGN COMPLETE RECOVERY ' + fingerprint + ': ') + == 'SIGN COMPLETE RECOVERY ' + fingerprint, + 'approval_cancelled', + ) + envelope = loads(read_private(root_envelope(self.config_dir))) + signer = Ed25519Signer.from_bytes( + open_private_key(envelope, getpass.getpass('Root PIN/passphrase: ')) + ) + packet = asyncio.run(seal(app, body, signer)) + write_record(Path(destination), packet) + return { + 'status': 'complete_recovery_state_signed', + 'digest': fingerprint, + 'independent_pin_required': True, + 'path': str(destination), + } finally: asyncio.run(app.close()) - def promote_recovery(self,source,independent_trust): - operator=require_local_console(self.config_dir) - from msg.admin.recovery_proof import open_for_proof, verify_proof, promote, IndependentRecoveryPin + def promote_recovery(self, source, independent_trust): + operator = require_local_console(self.config_dir) + from msg.admin.recovery_proof import ( + IndependentRecoveryPin, + open_for_proof, + promote, + verify_proof, + ) from msg.security.root_files import read_private - packet=loads(read_private(source,limit=64*1024*1024)) - trust=loads(read_private(independent_trust,limit=65536)) - require(type(trust) is dict and set(trust)=={ - 'service','public_key','digest','sequence','source_backup_sha256'},'recovery_independent_pin_required') - pin=IndependentRecoveryPin(**dict(trust,public_key=unb64(trust['public_key'],limit=32))) - app=asyncio.run(open_for_proof(self.config_dir)) + + packet = loads(read_private(source, limit=64 * 1024 * 1024)) + trust = loads(read_private(independent_trust, limit=65536)) + require( + type(trust) is dict + and set(trust) + == {'service', 'public_key', 'digest', 'sequence', 'source_backup_sha256'}, + 'recovery_independent_pin_required', + ) + pin = IndependentRecoveryPin(**dict(trust, public_key=unb64(trust['public_key'], limit=32))) + app = asyncio.run(open_for_proof(self.config_dir)) try: - body=verify_proof(packet,pin,app.clock()) - print(canonical({'service':pin.service,'state_digest':pin.digest,'sequence':pin.sequence, - 'source_backup_sha256':pin.source_backup_sha256,'tables':len(body['metadata']['tables']), - 'effect':'release verified complete recovery; restart all runtimes'}).decode()) - require(input('Type PROMOTE COMPLETE RECOVERY '+pin.digest+': ')== - 'PROMOTE COMPLETE RECOVERY '+pin.digest,'approval_cancelled') - envelope=loads(read_private(root_envelope(self.config_dir))) - signer=Ed25519Signer.from_bytes(open_private_key(envelope,getpass.getpass('Root PIN/passphrase: '))) - return asyncio.run(promote(app,packet,pin=pin,signer=signer,operator=operator)) + body = verify_proof(packet, pin, app.clock()) + print( + canonical({ + 'service': pin.service, + 'state_digest': pin.digest, + 'sequence': pin.sequence, + 'source_backup_sha256': pin.source_backup_sha256, + 'tables': len(body['metadata']['tables']), + 'effect': 'release verified complete recovery; restart all runtimes', + }).decode() + ) + require( + input('Type PROMOTE COMPLETE RECOVERY ' + pin.digest + ': ') + == 'PROMOTE COMPLETE RECOVERY ' + pin.digest, + 'approval_cancelled', + ) + envelope = loads(read_private(root_envelope(self.config_dir))) + signer = Ed25519Signer.from_bytes( + open_private_key(envelope, getpass.getpass('Root PIN/passphrase: ')) + ) + return asyncio.run(promote(app, packet, pin=pin, signer=signer, operator=operator)) finally: asyncio.run(app.close()) - def sign_backup_retirement(self,source,destination): + def sign_backup_retirement(self, source, destination): """Root-sign an operator's statement that listed backup sets lost one old key.""" require_local_console(self.config_dir) from msg.admin.backup_retirement import read_statement, unsigned_statement, write_record from msg.security.backup_retirement import sign_record - target=Path(destination) - require(not target.exists() and not target.is_symlink(),'backup_retirement_destination_exists') - body=unsigned_statement(read_statement(source)) - print(canonical({'statement':body,'digest':digest(body), - 'claim':'listed_backup_sets_only'}).decode()) - require(input('Type ATTEST BACKUP RETIREMENT '+digest(body)+': ')== - 'ATTEST BACKUP RETIREMENT '+digest(body),'approval_cancelled') + + target = Path(destination) + require( + not target.exists() and not target.is_symlink(), 'backup_retirement_destination_exists' + ) + body = unsigned_statement(read_statement(source)) + print( + canonical({ + 'statement': body, + 'digest': digest(body), + 'claim': 'listed_backup_sets_only', + }).decode() + ) + require( + input('Type ATTEST BACKUP RETIREMENT ' + digest(body) + ': ') + == 'ATTEST BACKUP RETIREMENT ' + digest(body), + 'approval_cancelled', + ) from msg.security.root_files import read_private - envelope=loads(read_private(root_envelope(self.config_dir))) - private=open_private_key(envelope,getpass.getpass('Root PIN/passphrase: ')) - record=sign_record(body,Ed25519Signer.from_bytes(private)) - write_record(target,record) - return {'status':'backup_retirement_signed','path':str(target),'digest':digest(record)} - def import_backup_retirement(self,source): + envelope = loads(read_private(root_envelope(self.config_dir))) + private = open_private_key(envelope, getpass.getpass('Root PIN/passphrase: ')) + record = sign_record(body, Ed25519Signer.from_bytes(private)) + write_record(target, record) + return {'status': 'backup_retirement_signed', 'path': str(target), 'digest': digest(record)} + + def import_backup_retirement(self, source): """Only this console path persists a record; reads re-verify it every time.""" - operator=require_local_console(self.config_dir) + operator = require_local_console(self.config_dir) from msg.admin.backup_retirement import import_record, read_statement - app=self._app() - record=read_statement(source) - print(canonical({'record':record,'digest':digest(record)}).decode()) - require(input('Type IMPORT BACKUP RETIREMENT '+digest(record)+': ')== - 'IMPORT BACKUP RETIREMENT '+digest(record),'approval_cancelled') + + app = self._app() + record = read_statement(source) + print(canonical({'record': record, 'digest': digest(record)}).decode()) + require( + input('Type IMPORT BACKUP RETIREMENT ' + digest(record) + ': ') + == 'IMPORT BACKUP RETIREMENT ' + digest(record), + 'approval_cancelled', + ) + async def run(): await app.load() try: async with app.metadata.transaction(write=True) as tx: - return await import_record(tx,record,now=app.clock(),operator=operator) + return await import_record(tx, record, now=app.clock(), operator=operator) finally: await app.close() - attestation=asyncio.run(run()) - return {'status':'backup_retirement_imported','scope':'listed_backup_sets_only', - **attestation.view()} - def backup(self,destination): + attestation = asyncio.run(run()) + return { + 'status': 'backup_retirement_imported', + 'scope': 'listed_backup_sets_only', + **attestation.view(), + } + + def backup(self, destination): require_local_console(self.config_dir) - target=Path(destination) - require(not target.exists() and not target.is_symlink(),'backup_destination_exists') - envelope=loads(root_envelope(self.config_dir).read_bytes()) - trust=loads(trust_file(self.config_dir).read_bytes()) - require(input('Type BACKUP ROOT to continue: ')=='BACKUP ROOT','approval_cancelled') - private=open_private_key(envelope,getpass.getpass('Root PIN/passphrase: ')) - require(b64(Ed25519Signer.from_bytes(private).public_key)==trust['public_key'],'root_key_mismatch') - durable_write(target,canonical({'format':'msg-root-backup-v1','envelope':envelope,'trust':trust}),mode=0o600) - return {'status':'encrypted_backup_created','path':str(target),'fingerprint':digest(unb64(trust['public_key']))} - - - def recover(self,source): + target = Path(destination) + require(not target.exists() and not target.is_symlink(), 'backup_destination_exists') + envelope = loads(root_envelope(self.config_dir).read_bytes()) + trust = loads(trust_file(self.config_dir).read_bytes()) + require(input('Type BACKUP ROOT to continue: ') == 'BACKUP ROOT', 'approval_cancelled') + private = open_private_key(envelope, getpass.getpass('Root PIN/passphrase: ')) + require( + b64(Ed25519Signer.from_bytes(private).public_key) == trust['public_key'], + 'root_key_mismatch', + ) + durable_write( + target, + canonical({'format': 'msg-root-backup-v1', 'envelope': envelope, 'trust': trust}), + mode=0o600, + ) + return { + 'status': 'encrypted_backup_created', + 'path': str(target), + 'fingerprint': digest(unb64(trust['public_key'])), + } + + def recover(self, source): require_local_console(self.config_dir) - target=root_envelope(self.config_dir) - require(not target.exists(),'root_material_already_present') - backup=loads(Path(source).read_bytes()) - require(set(backup)=={'format','envelope','trust'} and backup['format']=='msg-root-backup-v1','invalid_root_backup') - current=loads(trust_file(self.config_dir).read_bytes()) - require(canonical(current)==canonical(backup['trust']),'trust_anchor_mismatch') - print(canonical({'fingerprint':digest(unb64(current['public_key'])),'action':'recover_missing_encrypted_root_key'}).decode()) - require(input('Type RECOVER ROOT to continue: ')=='RECOVER ROOT','approval_cancelled') - private=open_private_key(backup['envelope'],getpass.getpass('Backup root PIN/passphrase: ')) - require(b64(Ed25519Signer.from_bytes(private).public_key)==current['public_key'],'root_key_mismatch') - target.parent.mkdir(mode=0o700,parents=True,exist_ok=True) - os.chmod(target.parent,0o700) - durable_write(target,canonical(backup['envelope']),mode=0o600) - return {'status':'root_key_recovered','fingerprint':digest(unb64(current['public_key']))} - - -async def _revoke(app,certificate_id,signer, *, reason,operator): - require(certificate_id!=app.certificates.root_certificate.resource_id,'root_rotation_required') - require(signer.key_id==app.certificates.root_certificate.key_id,'root_key_mismatch') + target = root_envelope(self.config_dir) + require(not target.exists(), 'root_material_already_present') + backup = loads(Path(source).read_bytes()) + require( + set(backup) == {'format', 'envelope', 'trust'} + and backup['format'] == 'msg-root-backup-v1', + 'invalid_root_backup', + ) + current = loads(trust_file(self.config_dir).read_bytes()) + require(canonical(current) == canonical(backup['trust']), 'trust_anchor_mismatch') + print( + canonical({ + 'fingerprint': digest(unb64(current['public_key'])), + 'action': 'recover_missing_encrypted_root_key', + }).decode() + ) + require(input('Type RECOVER ROOT to continue: ') == 'RECOVER ROOT', 'approval_cancelled') + private = open_private_key( + backup['envelope'], getpass.getpass('Backup root PIN/passphrase: ') + ) + require( + b64(Ed25519Signer.from_bytes(private).public_key) == current['public_key'], + 'root_key_mismatch', + ) + target.parent.mkdir(mode=0o700, parents=True, exist_ok=True) + os.chmod(target.parent, 0o700) + durable_write(target, canonical(backup['envelope']), mode=0o600) + return {'status': 'root_key_recovered', 'fingerprint': digest(unb64(current['public_key']))} + + +async def _revoke(app, certificate_id, signer, *, reason, operator): + require( + certificate_id != app.certificates.root_certificate.resource_id, 'root_rotation_required' + ) + require(signer.key_id == app.certificates.root_certificate.key_id, 'root_key_mismatch') async with app.metadata.transaction(write=True) as tx: - certificate=await tx.certificate(certificate_id) - statement={'certificate_id':certificate_id,'reason':reason,'time':wire(app.clock()), - 'issuer':ROOT_SUBJECT,'operator':operator} - signature=signer.sign(canonical(statement),purpose='revocation') - event=Event(id=new_id('audit'),type='root.cert.revoke',time=app.clock(),request_id=new_id('local'), - actor=ROOT_SUBJECT,subject=ROOT_SUBJECT,resources=(ResourceRef(id=certificate_id),), - data={**statement,'signature':wire(signature)}) - await tx.revoke_certificate(certificate_id,AuditEvent(event=event, - authority=(ResourceRef(id=app.certificates.root_certificate.resource_id),),before_digest=digest(certificate), - after_digest=digest(statement),previous_digest=None,entry_digest='',result='revoked')) - return {'certificate_id':certificate_id,'revoked':True,'signature':wire(signature)} + certificate = await tx.certificate(certificate_id) + statement = { + 'certificate_id': certificate_id, + 'reason': reason, + 'time': wire(app.clock()), + 'issuer': ROOT_SUBJECT, + 'operator': operator, + } + signature = signer.sign(canonical(statement), purpose='revocation') + event = Event( + id=new_id('audit'), + type='root.cert.revoke', + time=app.clock(), + request_id=new_id('local'), + actor=ROOT_SUBJECT, + subject=ROOT_SUBJECT, + resources=(ResourceRef(id=certificate_id),), + data={**statement, 'signature': wire(signature)}, + ) + await tx.revoke_certificate( + certificate_id, + AuditEvent( + event=event, + authority=(ResourceRef(id=app.certificates.root_certificate.resource_id),), + before_digest=digest(certificate), + after_digest=digest(statement), + previous_digest=None, + entry_digest='', + result='revoked', + ), + ) + return {'certificate_id': certificate_id, 'revoked': True, 'signature': wire(signature)} diff --git a/src/msg/admin/rotation.py b/src/msg/admin/rotation.py index 9c945605..20dcddcb 100644 --- a/src/msg/admin/rotation.py +++ b/src/msg/admin/rotation.py @@ -4,30 +4,42 @@ invalidates all old certificate chains for new requests. Historical public keys and signed records remain available; they are not retroactively re-signed. """ + from __future__ import annotations + +import os from dataclasses import replace from datetime import timedelta -import os -from msg.constants import ROOT_SUBJECT, ONLINE_CA, ADMINS_GROUP, CSR_SPACE -from msg.core.codec import canonical, decode, digest, wire, b64, loads +from msg.bootstrap import seed_resource +from msg.constants import ADMINS_GROUP, CSR_SPACE, ONLINE_CA, ROOT_SUBJECT +from msg.core.codec import b64, canonical, decode, digest, wire from msg.core.errors import require -from msg.core.models import Certificate, CertificateRequest, Credential, Signature, IssuancePolicy, AuditEvent, Event, ResourceRef +from msg.core.models import ( + AuditEvent, + Certificate, + CertificateRequest, + Credential, + Event, + IssuancePolicy, + ResourceRef, + Signature, +) from msg.plugins.common import new_id from msg.plugins.identity import certificate_resource -from msg.security.crypto import seal_private_key, Ed25519Signer -from msg.security.certificates import sign_certificate, csr_body -from msg.security.rotation_journal import authorization, validate as validate_journal -from msg.security.root_files import read_private, rotation_lock from msg.security.capabilities import grant_for -from msg.storage.git import durable_write +from msg.security.certificates import csr_body, sign_certificate +from msg.security.crypto import Ed25519Signer, seal_private_key +from msg.security.root_files import read_private, rotation_lock +from msg.security.rotation_journal import authorization, validate as validate_journal from msg.security.trust_files import trust_values, write_trust -from msg.bootstrap import seed_resource +from msg.storage.git import durable_write def journal_path(app): from msg.admin.root import root_envelope - return root_envelope(app.settings.config_dir).parent/'rotation.pending.json' + + return root_envelope(app.settings.config_dir).parent / 'rotation.pending.json' def prepare(app, new_signer, new_pin, *, old_signer, operator): @@ -42,42 +54,92 @@ def _prepare(app, new_signer, new_pin, *, old_signer, operator): require(old_signer.key_id == old.key_id, 'root_key_mismatch') now = app.clock() grants = app.primary_ceiling() - new = Certificate(resource_id=new_id('cert_root'), serial=new_id('serial'), subject_id=ROOT_SUBJECT, - key_id=new_signer.key_id, issuer_id=ROOT_SUBJECT, parent_certificate_id=None, authority_sources=(), kind='ca', - grants=grants, not_before=now, expires_at=now+timedelta(days=3650), target_service=app.settings.service_url, - delegation_depth=8, issuance=IssuancePolicy(issue_grants=grants,max_cert_ttl_seconds=31536000, - max_child_ca_depth=3,max_delegation_depth=8), - signature=Signature(key_id=new_signer.key_id, algorithm='ed25519', value=b'')) + new = Certificate( + resource_id=new_id('cert_root'), + serial=new_id('serial'), + subject_id=ROOT_SUBJECT, + key_id=new_signer.key_id, + issuer_id=ROOT_SUBJECT, + parent_certificate_id=None, + authority_sources=(), + kind='ca', + grants=grants, + not_before=now, + expires_at=now + timedelta(days=3650), + target_service=app.settings.service_url, + delegation_depth=8, + issuance=IssuancePolicy( + issue_grants=grants, + max_cert_ttl_seconds=31536000, + max_child_ca_depth=3, + max_delegation_depth=8, + ), + signature=Signature(key_id=new_signer.key_id, algorithm='ed25519', value=b''), + ) new = sign_certificate(new, new_signer) - statement = {'old_certificate':old.resource_id, 'old_fingerprint':digest(app.certificates.root_public_key), - 'new_certificate':new.resource_id, 'new_fingerprint':digest(new_signer.public_key), 'time':wire(now), - 'operator':operator, 'previous_key_proved':old_signer is not None} + statement = { + 'old_certificate': old.resource_id, + 'old_fingerprint': digest(app.certificates.root_public_key), + 'new_certificate': new.resource_id, + 'new_fingerprint': digest(new_signer.public_key), + 'time': wire(now), + 'operator': operator, + 'previous_key_proved': old_signer is not None, + } online = app.online_signer - csr = CertificateRequest(resource_id=new_id('csr'), applicant=ONLINE_CA, subject_id=ONLINE_CA, - requested_issuer=ROOT_SUBJECT, public_key=online.public_key, kind='ca', - grants=(grant_for(app.registry.capability('cert.issue'),scope=app.default_scope()),), - issuance=IssuancePolicy(issue_grants=app.base_grants(),max_cert_ttl_seconds=app.settings.base_certificate_ttl, - max_child_ca_depth=0,max_delegation_depth=8), requested_ttl_seconds=31536000, - target_service=app.settings.service_url,delegation_depth=0,authority_sources=(),request_digest='', - possession_proof=Signature(key_id=online.key_id,algorithm='ed25519',value=b'')) - csr = replace(csr, request_digest=digest(csr_body(csr)), - possession_proof=online.sign(canonical(csr_body(csr)),purpose='csr')) + csr = CertificateRequest( + resource_id=new_id('csr'), + applicant=ONLINE_CA, + subject_id=ONLINE_CA, + requested_issuer=ROOT_SUBJECT, + public_key=online.public_key, + kind='ca', + grants=(grant_for(app.registry.capability('cert.issue'), scope=app.default_scope()),), + issuance=IssuancePolicy( + issue_grants=app.base_grants(), + max_cert_ttl_seconds=app.settings.base_certificate_ttl, + max_child_ca_depth=0, + max_delegation_depth=8, + ), + requested_ttl_seconds=31536000, + target_service=app.settings.service_url, + delegation_depth=0, + authority_sources=(), + request_digest='', + possession_proof=Signature(key_id=online.key_id, algorithm='ed25519', value=b''), + ) + csr = replace( + csr, + request_digest=digest(csr_body(csr)), + possession_proof=online.sign(canonical(csr_body(csr)), purpose='csr'), + ) from msg.admin.root import root_envelope - key_file=root_envelope(app.settings.config_dir) - require(not key_file.is_symlink(),'unsafe_root_private_path') - journal={'version':2,'old_certificate':wire(old),'new_trust':{'version':1, - 'public_key':b64(new_signer.public_key),'certificate':wire(new)}, - 'new_envelope':seal_private_key(new_signer.private_bytes(),new_pin), - 'previous_envelope_digest':digest(read_private(key_file)) if key_file.exists() else None, - 'online_csr':wire(csr),'statement':statement} - proof=canonical(authorization(journal)) - journal['new_signature']=wire(new_signer.sign(proof,purpose='root-rotation')) - journal['old_signature']=wire(old_signer.sign(proof,purpose='root-rotation')) if old_signer else None - protected=journal_path(app).parent - require(not protected.is_symlink(),'unsafe_root_private_path') - protected.mkdir(parents=True,exist_ok=True,mode=0o700) - os.chmod(protected,0o700) - durable_write(journal_path(app),canonical(journal),mode=0o600) + + key_file = root_envelope(app.settings.config_dir) + require(not key_file.is_symlink(), 'unsafe_root_private_path') + journal = { + 'version': 2, + 'old_certificate': wire(old), + 'new_trust': { + 'version': 1, + 'public_key': b64(new_signer.public_key), + 'certificate': wire(new), + }, + 'new_envelope': seal_private_key(new_signer.private_bytes(), new_pin), + 'previous_envelope_digest': digest(read_private(key_file)) if key_file.exists() else None, + 'online_csr': wire(csr), + 'statement': statement, + } + proof = canonical(authorization(journal)) + journal['new_signature'] = wire(new_signer.sign(proof, purpose='root-rotation')) + journal['old_signature'] = ( + wire(old_signer.sign(proof, purpose='root-rotation')) if old_signer else None + ) + protected = journal_path(app).parent + require(not protected.is_symlink(), 'unsafe_root_private_path') + protected.mkdir(parents=True, exist_ok=True, mode=0o700) + os.chmod(protected, 0o700) + durable_write(journal_path(app), canonical(journal), mode=0o600) return journal @@ -88,112 +150,196 @@ async def complete(app, journal, *, pin): async def _complete(app, journal, *, pin): from msg.security.crypto import verify - pending=journal_path(app) - protected=pending.parent - require(not protected.is_symlink() and not pending.is_symlink(), - 'unsafe_root_private_path') - online=Ed25519Signer.from_bytes((app.settings.service_keys/'online.key').read_bytes()) - signer,old,new,csr,proof=validate_journal(journal,pin=pin, - service=app.settings.service_url,online_public=online.public_key) - key_file=protected/'key.json' - require(not key_file.is_symlink(),'unsafe_root_private_path') + + pending = journal_path(app) + protected = pending.parent + require(not protected.is_symlink() and not pending.is_symlink(), 'unsafe_root_private_path') + online = Ed25519Signer.from_bytes((app.settings.service_keys / 'online.key').read_bytes()) + signer, old, new, csr, proof = validate_journal( + journal, pin=pin, service=app.settings.service_url, online_public=online.public_key + ) + key_file = protected / 'key.json' + require(not key_file.is_symlink(), 'unsafe_root_private_path') if key_file.exists(): - require(digest(read_private(key_file)) in { - journal['previous_envelope_digest'],digest(canonical(journal['new_envelope']))}, - 'root_envelope_changed') + require( + digest(read_private(key_file)) + in {journal['previous_envelope_digest'], digest(canonical(journal['new_envelope']))}, + 'root_envelope_changed', + ) else: - require(journal['previous_envelope_digest'] is None,'root_envelope_missing') + require(journal['previous_envelope_digest'] is None, 'root_envelope_missing') # Reject unsafe or substituted history BEFORE changing the database anchor. - history = protected/'history'/old.resource_id/'key.json' - for directory in (protected/'history', history.parent): + history = protected / 'history' / old.resource_id / 'key.json' + for directory in (protected / 'history', history.parent): require(not directory.is_symlink(), 'unsafe_root_private_path') require(not history.is_symlink(), 'unsafe_root_private_path') if history.exists(): - require(digest(read_private(history)) == journal['previous_envelope_digest'], - 'rotation_history_mismatch') + require( + digest(read_private(history)) == journal['previous_envelope_digest'], + 'rotation_history_mismatch', + ) elif journal['previous_envelope_digest'] is not None: - require(key_file.exists() and - digest(read_private(key_file)) == journal['previous_envelope_digest'], - 'rotation_history_missing') + require( + key_file.exists() + and digest(read_private(key_file)) == journal['previous_envelope_digest'], + 'rotation_history_missing', + ) current_trust = trust_values(app.settings.config_dir) - require(bool(current_trust),'rotation_trust_missing') + require(bool(current_trust), 'rotation_trust_missing') # This method is called only after the local console has approved the exact # journal. It also handles the state where PostgreSQL committed but protected files did not. await app.open_storage() async with app.metadata.transaction(write=True) as tx: - current=tx.setting('active_root_certificate','cert_root') - require(current in {old.resource_id,new.resource_id},'trust_anchor_mismatch') - existing=await tx.certificate(old.resource_id) - require(canonical(existing)==canonical(old),'trust_anchor_mismatch') - old_key=await tx.credential(old.key_id) - require(old_key.subject_id==ROOT_SUBJECT and - digest(old_key.verifier)==journal['statement']['old_fingerprint'], - 'rotation_statement_mismatch') + current = tx.setting('active_root_certificate', 'cert_root') + require(current in {old.resource_id, new.resource_id}, 'trust_anchor_mismatch') + existing = await tx.certificate(old.resource_id) + require(canonical(existing) == canonical(old), 'trust_anchor_mismatch') + old_key = await tx.credential(old.key_id) + require( + old_key.subject_id == ROOT_SUBJECT + and digest(old_key.verifier) == journal['statement']['old_fingerprint'], + 'rotation_statement_mismatch', + ) old_trust = {'version': 1, 'public_key': b64(old_key.verifier), 'certificate': wire(old)} - require(all(canonical(value) in (canonical(old_trust), canonical(journal['new_trust'])) - for value in current_trust.values()), - 'rotation_trust_changed') + require( + all( + canonical(value) in (canonical(old_trust), canonical(journal['new_trust'])) + for value in current_trust.values() + ), + 'rotation_trust_changed', + ) if journal['old_signature'] is not None: - verify(old_key.verifier,canonical(proof),decode(Signature,journal['old_signature']), - purpose='root-rotation') + verify( + old_key.verifier, + canonical(proof), + decode(Signature, journal['old_signature']), + purpose='root-rotation', + ) if current == new.resource_id: - require(canonical(await tx.certificate(new.resource_id))==canonical(new) and - canonical(await tx.csr(csr.resource_id))==canonical(csr) and - old_key.revoked_at is not None and await tx.certificate_revoked(old.resource_id), - 'rotation_resume_mismatch') - installed=await tx.credential(new.key_id) - require(installed.verifier==signer.public_key and installed.revoked_at is None, - 'rotation_resume_mismatch') + require( + canonical(await tx.certificate(new.resource_id)) == canonical(new) + and canonical(await tx.csr(csr.resource_id)) == canonical(csr) + and old_key.revoked_at is not None + and await tx.certificate_revoked(old.resource_id), + 'rotation_resume_mismatch', + ) + installed = await tx.credential(new.key_id) + require( + installed.verifier == signer.public_key and installed.revoked_at is None, + 'rotation_resume_mismatch', + ) else: - subject=await tx.subject(ROOT_SUBJECT) - require(subject.local_only,'root_policy_corrupt') + subject = await tx.subject(ROOT_SUBJECT) + require(subject.local_only, 'root_policy_corrupt') # Credential saves are upserts; a retired root key must never be revived. - require(tx.one('SELECT 1 FROM credentials WHERE id=?',(signer.key_id,)) is None, - 'root_key_reused') - await tx.save_credential(Credential(id=signer.key_id,subject_id=ROOT_SUBJECT,kind='signing_key', - verifier=signer.public_key,ceiling=(),not_before=new.not_before,expires_at=None,revoked_at=None),subject.auth_version) - await tx.save_credential(replace(old_key,revoked_at=app.clock()),subject.auth_version) - await certificate_resource(tx,new,app.clock()) - await tx.register_certificate(new,None,0) - event=Event(id=new_id('audit'),type='root.rotate',time=app.clock(),request_id=new.resource_id, - actor=ROOT_SUBJECT,subject=ROOT_SUBJECT,resources=(ResourceRef(id=new.resource_id),), - data={'statement':journal['statement'],'authorization':proof,'new_signature':journal['new_signature'], - 'old_signature':journal['old_signature']}) - await tx.revoke_certificate(old.resource_id,AuditEvent(event=event,authority=(), - before_digest=digest(old),after_digest=digest(new),previous_digest=None,entry_digest='',result='root_rotated')) - await seed_resource(tx,app.contents,dict(id=csr.resource_id,type='csr',name=csr.resource_id,parent=CSR_SPACE, - owner=ONLINE_CA,group=ADMINS_GROUP,mode='0600'),app.clock()) + require( + tx.one('SELECT 1 FROM credentials WHERE id=?', (signer.key_id,)) is None, + 'root_key_reused', + ) + await tx.save_credential( + Credential( + id=signer.key_id, + subject_id=ROOT_SUBJECT, + kind='signing_key', + verifier=signer.public_key, + ceiling=(), + not_before=new.not_before, + expires_at=None, + revoked_at=None, + ), + subject.auth_version, + ) + await tx.save_credential(replace(old_key, revoked_at=app.clock()), subject.auth_version) + await certificate_resource(tx, new, app.clock()) + await tx.register_certificate(new, None, 0) + event = Event( + id=new_id('audit'), + type='root.rotate', + time=app.clock(), + request_id=new.resource_id, + actor=ROOT_SUBJECT, + subject=ROOT_SUBJECT, + resources=(ResourceRef(id=new.resource_id),), + data={ + 'statement': journal['statement'], + 'authorization': proof, + 'new_signature': journal['new_signature'], + 'old_signature': journal['old_signature'], + }, + ) + await tx.revoke_certificate( + old.resource_id, + AuditEvent( + event=event, + authority=(), + before_digest=digest(old), + after_digest=digest(new), + previous_digest=None, + entry_digest='', + result='root_rotated', + ), + ) + await seed_resource( + tx, + app.contents, + { + 'id': csr.resource_id, + 'type': 'csr', + 'name': csr.resource_id, + 'parent': CSR_SPACE, + 'owner': ONLINE_CA, + 'group': ADMINS_GROUP, + 'mode': '0600', + }, + app.clock(), + ) await tx.save_csr(csr) - tx.set_setting('online_ca_request',csr.resource_id) - tx.set_setting('online_ca_certificate',None) - tx.set_setting('active_root_certificate',new.resource_id) - tx.set_setting('authorization_epoch',tx.setting('authorization_epoch',0)+1) - tx.set_setting('root_rotation:'+new.resource_id,journal['statement']) + tx.set_setting('online_ca_request', csr.resource_id) + tx.set_setting('online_ca_certificate', None) + tx.set_setting('active_root_certificate', new.resource_id) + tx.set_setting('authorization_epoch', tx.setting('authorization_epoch', 0) + 1) + tx.set_setting('root_rotation:' + new.resource_id, journal['statement']) # The journal and envelope must use the same independently protected root # directory, including when resuming after the metadata commit. - for directory in (protected,protected/'history',protected/'history'/old.resource_id): - require(not directory.is_symlink(),'unsafe_root_private_path') - directory.mkdir(parents=True,exist_ok=True,mode=0o700) - os.chmod(directory,0o700) - history=protected/'history'/old.resource_id/'key.json' - require(not history.is_symlink(),'unsafe_root_private_path') + for directory in (protected, protected / 'history', protected / 'history' / old.resource_id): + require(not directory.is_symlink(), 'unsafe_root_private_path') + directory.mkdir(parents=True, exist_ok=True, mode=0o700) + os.chmod(directory, 0o700) + history = protected / 'history' / old.resource_id / 'key.json' + require(not history.is_symlink(), 'unsafe_root_private_path') if history.exists(): - require(digest(read_private(history))==journal['previous_envelope_digest'], - 'rotation_history_mismatch') + require( + digest(read_private(history)) == journal['previous_envelope_digest'], + 'rotation_history_mismatch', + ) elif journal['previous_envelope_digest'] is not None: # Never archive an already installed new envelope as the old key. - require(key_file.exists() and - digest(read_private(key_file))==journal['previous_envelope_digest'], - 'rotation_history_missing') - durable_write(history,read_private(key_file),mode=0o600) - durable_write(key_file,canonical(journal['new_envelope']),mode=0o600) - write_trust(app.settings.config_dir,journal['new_trust'],writer=durable_write, - approved_values=(old_trust,journal['new_trust'])) + require( + key_file.exists() + and digest(read_private(key_file)) == journal['previous_envelope_digest'], + 'rotation_history_missing', + ) + durable_write(history, read_private(key_file), mode=0o600) + durable_write(key_file, canonical(journal['new_envelope']), mode=0o600) + write_trust( + app.settings.config_dir, + journal['new_trust'], + writer=durable_write, + approved_values=(old_trust, journal['new_trust']), + ) # Journal survives every earlier failure and is removed only after both # metadata and protected files have durable copies. pending.unlink(missing_ok=True) - directory=os.open(protected,os.O_RDONLY|os.O_DIRECTORY) - try:os.fsync(directory) - finally:os.close(directory) - return {'root_id':ROOT_SUBJECT,'certificate_id':new.resource_id,'fingerprint':digest(signer.public_key), - 'online_ca_request':csr.resource_id,'restart_required':True,'old_certificate_chains_valid':False} + directory = os.open(protected, os.O_RDONLY | os.O_DIRECTORY) + try: + os.fsync(directory) + finally: + os.close(directory) + return { + 'root_id': ROOT_SUBJECT, + 'certificate_id': new.resource_id, + 'fingerprint': digest(signer.public_key), + 'online_ca_request': csr.resource_id, + 'restart_required': True, + 'old_certificate_chains_valid': False, + } diff --git a/src/msg/admin/token_delivery_check.py b/src/msg/admin/token_delivery_check.py index 6354c17b..c542cf53 100644 --- a/src/msg/admin/token_delivery_check.py +++ b/src/msg/admin/token_delivery_check.py @@ -1,4 +1,5 @@ """Exercise one-use credential release only on the caller's isolated Test Root.""" + from datetime import timedelta from pathlib import Path from tempfile import TemporaryDirectory @@ -40,8 +41,12 @@ async def discard(packet): # Both processes can restart; refreshing only the request expiry # does not refresh either the token lifetime or recovery deadline. state = ClientState(Path(directory), server=app.settings.service_url) - client = MsgClient(state, HTTPTransport(state.server, http=http), - clock=lambda: now+timedelta(seconds=1), retries=0) + client = MsgClient( + state, + HTTPTransport(state.server, http=http), + clock=lambda: now + timedelta(seconds=1), + retries=0, + ) replay = await client.recover_token() if replay.status != 'error' or replay.error.code != 'token_delivery_unavailable': return False @@ -51,4 +56,8 @@ async def discard(packet): async with app.metadata.transaction(write=False) as tx: old = await tx.credential(original[0]) current = await tx.credential(state.token[0]) - return old.revoked_at is not None and current.revoked_at is None and client._token_journal() == (None, None) + return ( + old.revoked_at is not None + and current.revoked_at is None + and client._token_journal() == (None, None) + ) diff --git a/src/msg/admin/tool_check.py b/src/msg/admin/tool_check.py index 506dabdf..62061699 100644 --- a/src/msg/admin/tool_check.py +++ b/src/msg/admin/tool_check.py @@ -1,4 +1,5 @@ """Tool readiness and isolated real-runner exercise; never contact an external host.""" + import asyncio import tempfile from dataclasses import replace @@ -16,8 +17,9 @@ async def inspect_tool_sandbox(app): policy = replace(tool.network, hosts=('denied.invalid',), timeout_ms=1000) with tempfile.TemporaryDirectory(prefix='msg-tool-doctor-') as directory: try: - await BubblewrapRunner(app)(tool, {'url': 'http://127.0.0.1/'}, - (policy,), Path(directory)) + await BubblewrapRunner(app)( + tool, {'url': 'http://127.0.0.1/'}, (policy,), Path(directory) + ) except Failure as exc: require(exc.code == 'network_policy_denied', exc.code) else: @@ -27,48 +29,75 @@ async def inspect_tool_sandbox(app): async def check_tool_sandbox(app, call, approve, key, subject): """Check real tool.run denial and real bwrap output using a loopback-only fixture.""" - require(app.selftest_run_id is not None,'selftest_namespace_required') + require(app.selftest_run_id is not None, 'selftest_namespace_required') payload = b'msg isolated sandbox selftest\n' received = [] + async def serve(reader, writer): try: request = await asyncio.wait_for(reader.readuntil(b'\r\n\r\n'), 5) received.append(request.split(b'\r\n', 1)[0]) - writer.write(b'HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nContent-Length: '+ - str(len(payload)).encode()+b'\r\nConnection: close\r\n\r\n'+payload) + writer.write( + b'HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nContent-Length: ' + + str(len(payload)).encode() + + b'\r\nConnection: close\r\n\r\n' + + payload + ) await writer.drain() finally: writer.close() await writer.wait_closed() + server = await asyncio.start_server(serve, '127.0.0.1', 0, limit=8192) try: port = server.sockets[0].getsockname()[1] scope = Scope(resource_id='tool_curl', descendants=False) - grant = replace(grant_for(app.registry.capability('tool.use'), scope=scope), - constraints={'hosts': ['127.0.0.1'], 'ports': [80], 'methods': ['GET']}) + grant = replace( + grant_for(app.registry.capability('tool.use'), scope=scope), + constraints={'hosts': ['127.0.0.1'], 'ports': [80], 'methods': ['GET']}, + ) certificate = await approve(key, subject, (grant,)) - denied = await call('tool.run', {'id': 'tool_curl', - 'arguments': {'url': 'http://127.0.0.1/'}}, - key, subject, (certificate.resource_id,)) - require(denied.error is not None and denied.error.code == 'network_policy_denied', - 'tool_selftest_private_denial_missing') + denied = await call( + 'tool.run', + {'id': 'tool_curl', 'arguments': {'url': 'http://127.0.0.1/'}}, + key, + subject, + (certificate.resource_id,), + ) + require( + denied.error is not None and denied.error.code == 'network_policy_denied', + 'tool_selftest_private_denial_missing', + ) tool = app.registry.tool('tool_curl') - policy = replace(tool.network, hosts=('127.0.0.1',), ports=frozenset({port}), - methods=frozenset({'GET'}), allow_private=True, timeout_ms=3000, - max_response_bytes=4096, max_redirects=0) + policy = replace( + tool.network, + hosts=('127.0.0.1',), + ports=frozenset({port}), + methods=frozenset({'GET'}), + allow_private=True, + timeout_ms=3000, + max_response_bytes=4096, + max_redirects=0, + ) # This exercises the real runner port. It deliberately does not alter the # registered tool's 80/443 policy to claim a normal tool.run success. with tempfile.TemporaryDirectory(prefix='msg-tool-selftest-') as directory: - result = await BubblewrapRunner(app)(tool, {'url': f'http://127.0.0.1:{port}/'}, - (policy,), Path(directory)) - require(result.path.read_bytes() == payload and result.metadata['status'] == 200, - 'tool_selftest_output_mismatch') + result = await BubblewrapRunner(app)( + tool, {'url': f'http://127.0.0.1:{port}/'}, (policy,), Path(directory) + ) + require( + result.path.read_bytes() == payload and result.metadata['status'] == 200, + 'tool_selftest_output_mismatch', + ) require(received == [b'GET / HTTP/1.1'], 'tool_selftest_request_mismatch') await inspect_tool_sandbox(app) require(received == [b'GET / HTTP/1.1'], 'tool_selftest_unexpected_request') - return {'operation': 'private_target_denied', - 'runner': 'loopback_output_and_policy_denial', - 'normal_operation_pipeline': 'not_exercised', 'external_requests': 0} + return { + 'operation': 'private_target_denied', + 'runner': 'loopback_output_and_policy_denial', + 'normal_operation_pipeline': 'not_exercised', + 'external_requests': 0, + } finally: server.close() await server.wait_closed() diff --git a/src/msg/admin/transfer_check.py b/src/msg/admin/transfer_check.py index 7016936c..ca56239d 100644 --- a/src/msg/admin/transfer_check.py +++ b/src/msg/admin/transfer_check.py @@ -1,4 +1,5 @@ """Bounded TransferSession checks in the disposable selftest installation.""" + from msg.core.codec import b64, digest, unb64, wire from msg.core.errors import require @@ -10,70 +11,139 @@ async def check_transfer(app, call, register): async def invoke(operation, arguments): result = await call(operation, arguments, key, owner) - require(result.status == 'ok', 'selftest_transfer_operation_failed', - details={'operation': operation}) + require( + result.status == 'ok', + 'selftest_transfer_operation_failed', + details={'operation': operation}, + ) return result async def snapshot(): async with app.metadata.transaction(write=False) as tx: - return {table: tuple(sorted(tx.rows(f'SELECT * FROM {table}'), key=repr)) - for table in ('resources', 'revisions', 'events', 'audit', 'jobs', - 'results', 'transfers', 'chunks', 'settings', 'credentials')} + return { + table: tuple(sorted(tx.rows(f'SELECT * FROM {table}'), key=repr)) + for table in ( + 'resources', + 'revisions', + 'events', + 'audit', + 'jobs', + 'results', + 'transfers', + 'chunks', + 'settings', + 'credentials', + ) + } payload = b'isolated transfer selftest\n' + bytes(range(256)) * 2 - opened = await invoke('transfer.open', {'direction': 'upload', 'size': len(payload), - 'digest': digest(payload), 'requested_part_bytes': 128}) + opened = await invoke( + 'transfer.open', + { + 'direction': 'upload', + 'size': len(payload), + 'digest': digest(payload), + 'requested_part_bytes': 128, + }, + ) transfer = opened.data['transfer_id'] part_bytes = opened.data['part_bytes'] require(0 < part_bytes <= 128, 'selftest_transfer_part_size') # The wrong principal must not be able to write a valid chunk or read state. before = await snapshot() for operation, arguments in ( - ('transfer.status', {'transfer_id': transfer}), - ('transfer.part_put', {'transfer_id': transfer, 'offset': 0, - 'data': b64(payload[:1]), 'digest': digest(payload[:1])})): + ('transfer.status', {'transfer_id': transfer}), + ( + 'transfer.part_put', + { + 'transfer_id': transfer, + 'offset': 0, + 'data': b64(payload[:1]), + 'digest': digest(payload[:1]), + }, + ), + ): denied = await call(operation, arguments, other_key, other) - require(denied.error is not None and denied.error.code == 'transfer_owner_required', - 'selftest_transfer_owner_not_enforced') - bad = await call('transfer.part_put', {'transfer_id': transfer, 'offset': 0, - 'data': b64(payload[:1]), 'digest': digest(b'wrong')}, key, owner) - require(bad.error is not None and bad.error.code == 'chunk_digest_mismatch', - 'selftest_transfer_bad_digest_accepted') + require( + denied.error is not None and denied.error.code == 'transfer_owner_required', + 'selftest_transfer_owner_not_enforced', + ) + bad = await call( + 'transfer.part_put', + { + 'transfer_id': transfer, + 'offset': 0, + 'data': b64(payload[:1]), + 'digest': digest(b'wrong'), + }, + key, + owner, + ) + require( + bad.error is not None and bad.error.code == 'chunk_digest_mismatch', + 'selftest_transfer_bad_digest_accepted', + ) require(await snapshot() == before, 'selftest_transfer_rejection_mutated') for offset in reversed(range(0, len(payload), part_bytes)): - block = payload[offset:offset + part_bytes] - await invoke('transfer.part_put', {'transfer_id': transfer, 'offset': offset, - 'data': b64(block), 'digest': digest(block)}) + block = payload[offset : offset + part_bytes] + await invoke( + 'transfer.part_put', + { + 'transfer_id': transfer, + 'offset': offset, + 'data': b64(block), + 'digest': digest(block), + }, + ) status = await invoke('transfer.status', {'transfer_id': transfer}) require(not status.data['missing'], 'selftest_transfer_missing_parts') - sealed = await invoke('transfer.seal', {'transfer_id': transfer, - 'final_size': len(payload), 'final_digest': digest(payload)}) + sealed = await invoke( + 'transfer.seal', + {'transfer_id': transfer, 'final_size': len(payload), 'final_digest': digest(payload)}, + ) require(len(sealed.resources) == 1, 'selftest_transfer_output_missing') async with app.metadata.transaction(write=False) as tx: resource = await tx.resource(sealed.resources[0].id) - require(resource.type == 'file' and resource.mode == 0o600 and resource.owner == owner, - 'selftest_transfer_output_not_private') - downloaded = await invoke('transfer.open', {'direction': 'download', - 'target': wire(sealed.resources[0]), 'requested_part_bytes': 128}) + require( + resource.type == 'file' and resource.mode == 0o600 and resource.owner == owner, + 'selftest_transfer_output_not_private', + ) + downloaded = await invoke( + 'transfer.open', + {'direction': 'download', 'target': wire(sealed.resources[0]), 'requested_part_bytes': 128}, + ) download = downloaded.data['transfer_id'] download_part_bytes = downloaded.data['part_bytes'] require(0 < download_part_bytes <= 128, 'selftest_transfer_part_size') before = await snapshot() received = bytearray() for offset in range(0, len(payload), download_part_bytes): - part = await invoke('transfer.part_get', {'transfer_id': download, 'offset': offset, - 'length': min(download_part_bytes, len(payload) - offset)}) + part = await invoke( + 'transfer.part_get', + { + 'transfer_id': download, + 'offset': offset, + 'length': min(download_part_bytes, len(payload) - offset), + }, + ) block = unb64(part.data['data']) - require(digest(block) == part.data['chunk']['content']['digest'], - 'selftest_transfer_chunk_digest_mismatch') + require( + digest(block) == part.data['chunk']['content']['digest'], + 'selftest_transfer_chunk_digest_mismatch', + ) received.extend(block) await invoke('transfer.status', {'transfer_id': transfer}) await invoke('transfer.status', {'transfer_id': download}) - denied = await call('transfer.part_get', {'transfer_id': download, 'offset': 0, 'length': 1}, - other_key, other) - require(denied.error is not None and denied.error.code == 'transfer_owner_required', - 'selftest_transfer_download_owner_not_enforced') - require(bytes(received) == payload and digest(bytes(received)) == digest(payload), - 'selftest_transfer_download_mismatch') + denied = await call( + 'transfer.part_get', {'transfer_id': download, 'offset': 0, 'length': 1}, other_key, other + ) + require( + denied.error is not None and denied.error.code == 'transfer_owner_required', + 'selftest_transfer_download_owner_not_enforced', + ) + require( + bytes(received) == payload and digest(bytes(received)) == digest(payload), + 'selftest_transfer_download_mismatch', + ) require(await snapshot() == before, 'selftest_transfer_read_mutated') return True diff --git a/src/msg/admin/upgrade_check.py b/src/msg/admin/upgrade_check.py index 46021d81..3b0ba8eb 100644 --- a/src/msg/admin/upgrade_check.py +++ b/src/msg/admin/upgrade_check.py @@ -1,7 +1,8 @@ """Isolated selftest of the real upgrade commit and new-key recovery boundary.""" -from datetime import timedelta + import hashlib import os +from datetime import timedelta from uuid import uuid4 from msg.core.codec import b64, canonical, unb64, wire @@ -16,43 +17,97 @@ async def check_upgrade_recovery(app, now): nonce, recovery = b64(os.urandom(32)), b64(os.urandom(32)) subject = 'u_tmp_' + hashlib.sha256(unb64(nonce)).hexdigest()[:32] request_id = uuid4().hex - proof = signer.sign(canonical({'subject_id': subject, 'nonce': nonce, - 'public_key': b64(signer.public_key), 'encryption_recipient': recipient, - 'request_id': request_id}), purpose='temporary-key-possession-v1') - created = await app.executor.execute(request_for('identity.temporary', { - 'nonce': nonce, 'recovery_secret': recovery, 'public_key': b64(signer.public_key), - 'encryption_recipient': recipient, 'possession_proof': wire(proof)}, - app.settings.service_url, request_id=request_id, contract_version=3, - expires_at=now + timedelta(seconds=120))) + proof = signer.sign( + canonical({ + 'subject_id': subject, + 'nonce': nonce, + 'public_key': b64(signer.public_key), + 'encryption_recipient': recipient, + 'request_id': request_id, + }), + purpose='temporary-key-possession-v1', + ) + created = await app.executor.execute( + request_for( + 'identity.temporary', + { + 'nonce': nonce, + 'recovery_secret': recovery, + 'public_key': b64(signer.public_key), + 'encryption_recipient': recipient, + 'possession_proof': wire(proof), + }, + app.settings.service_url, + request_id=request_id, + contract_version=3, + expires_at=now + timedelta(seconds=120), + ) + ) if created.status != 'ok': return False token = created.data['credential_id'], unb64(created.data['token']) upgrade_id = uuid4().hex - signed = {'subject_id': subject, 'handle': 'selftest-upgrade', - 'public_key': b64(signer.public_key), 'encryption_recipient': recipient} + signed = { + 'subject_id': subject, + 'handle': 'selftest-upgrade', + 'public_key': b64(signer.public_key), + 'encryption_recipient': recipient, + } arguments = {key: value for key, value in signed.items() if key != 'subject_id'} arguments['possession_proof'] = wire(signer.sign(canonical(signed), purpose='upgrade')) - committed = await app.executor.execute(request_for('identity.upgrade', arguments, - app.settings.service_url, subject=subject, token=token, request_id=upgrade_id, - contract_version=2, expires_at=now + timedelta(seconds=120))) + committed = await app.executor.execute( + request_for( + 'identity.upgrade', + arguments, + app.settings.service_url, + subject=subject, + token=token, + request_id=upgrade_id, + contract_version=2, + expires_at=now + timedelta(seconds=120), + ) + ) if committed.status != 'ok': return False async with app.metadata.transaction(write=False) as tx: - before = tuple(tx.one('SELECT COUNT(*) FROM ' + table)[0] - for table in ('events', 'results', 'credentials', 'certificates')) + before = tuple( + tx.one('SELECT COUNT(*) FROM ' + table)[0] + for table in ('events', 'results', 'credentials', 'certificates') + ) # No old token, bootstrap nonce, original response or cached certificate is # supplied to this signed query. The stored result is only a lookup target. - recovered = await app.executor.execute(request_for('identity.upgrade_result', - {'upgrade_request_id': upgrade_id}, app.settings.service_url, subject=subject, - signer=signer, expires_at=now + timedelta(seconds=120))) - denied = await app.executor.execute(request_for('identity.upgrade_result', - {'upgrade_request_id': upgrade_id}, app.settings.service_url, subject=subject, - token=token, expires_at=now + timedelta(seconds=120))) + recovered = await app.executor.execute( + request_for( + 'identity.upgrade_result', + {'upgrade_request_id': upgrade_id}, + app.settings.service_url, + subject=subject, + signer=signer, + expires_at=now + timedelta(seconds=120), + ) + ) + denied = await app.executor.execute( + request_for( + 'identity.upgrade_result', + {'upgrade_request_id': upgrade_id}, + app.settings.service_url, + subject=subject, + token=token, + expires_at=now + timedelta(seconds=120), + ) + ) async with app.metadata.transaction(write=False) as tx: - after = tuple(tx.one('SELECT COUNT(*) FROM ' + table)[0] - for table in ('events', 'results', 'credentials', 'certificates')) - return (recovered.status == 'ok' and recovered.subject == subject and - recovered.data['key_id'] == signer.key_id and recovered.data['status'] == 'completed' - and recovered.data['certificate_id'] == committed.data['certificate_id'] - and denied.status == 'error' and denied.error.code == 'credential_revoked' - and before == after) + after = tuple( + tx.one('SELECT COUNT(*) FROM ' + table)[0] + for table in ('events', 'results', 'credentials', 'certificates') + ) + return ( + recovered.status == 'ok' + and recovered.subject == subject + and recovered.data['key_id'] == signer.key_id + and recovered.data['status'] == 'completed' + and recovered.data['certificate_id'] == committed.data['certificate_id'] + and denied.status == 'error' + and denied.error.code == 'credential_revoked' + and before == after + ) diff --git a/src/msg/application.py b/src/msg/application.py index 12450b0d..5af8f78e 100644 --- a/src/msg/application.py +++ b/src/msg/application.py @@ -1,21 +1,21 @@ """Explicit composition root for network services. Contains no root signer.""" + from __future__ import annotations -from datetime import UTC, datetime -from pathlib import Path import hmac +from datetime import UTC, datetime from msg.constants import ROOT_SPACE, ROOT_SUBJECT -from msg.core.codec import loads, decode, unb64 +from msg.core.codec import decode, loads, unb64 from msg.core.cursors import CursorCodec from msg.core.errors import require from msg.core.executor import OperationExecutor from msg.core.models import Certificate, Scope from msg.core.registry import Registry +from msg.plugins import install_registry from msg.security.authentication import AuthenticationService from msg.security.authorization import AuthorizationService -from msg.security.capabilities import primary_ceiling, base_grants, temporary_ceiling -from msg.plugins import install_registry +from msg.security.capabilities import base_grants, primary_ceiling, temporary_ceiling from msg.security.certificates import CertificateValidator from msg.security.crypto import Ed25519Signer from msg.security.quarantine import RuntimeGeneration, active as quarantine_active @@ -24,157 +24,203 @@ class Application: - def __init__(self,settings, *, clock=None, selftest_run_id=None): - self.settings=settings - self.clock=clock or (lambda: datetime.now(UTC)) + def __init__(self, settings, *, clock=None, selftest_run_id=None): + self.settings = settings + self.clock = clock or (lambda: datetime.now(UTC)) # Only the local diagnostic constructs an isolated test namespace. - require(selftest_run_id is None or (isinstance(selftest_run_id,str) and - len(selftest_run_id)==32 and all(c in '0123456789abcdef' for c in selftest_run_id)), - 'invalid_selftest_run_id') - self.selftest_run_id=selftest_run_id - self.namespace_root=ROOT_SPACE if selftest_run_id is None else 't_selftest_'+selftest_run_id - self.registry=Registry() - self.metadata=None - self.contents=None - self.executor=None - self.token_delivery=None - self._loaded=False - self.runtime_generation=None + require( + selftest_run_id is None + or ( + isinstance(selftest_run_id, str) + and len(selftest_run_id) == 32 + and all(c in '0123456789abcdef' for c in selftest_run_id) + ), + 'invalid_selftest_run_id', + ) + self.selftest_run_id = selftest_run_id + self.namespace_root = ( + ROOT_SPACE if selftest_run_id is None else 't_selftest_' + selftest_run_id + ) + self.registry = Registry() + self.metadata = None + self.contents = None + self.executor = None + self.token_delivery = None + self._loaded = False + self.runtime_generation = None install_registry(self, settings.server.plugins) def primary_ceiling(self): - return primary_ceiling(self.registry,self.default_scope()) + return primary_ceiling(self.registry, self.default_scope()) def base_grants(self): - return base_grants(self.registry,self.default_scope()) + return base_grants(self.registry, self.default_scope()) def default_scope(self): - return Scope(resource_id=self.namespace_root,descendants=True) + return Scope(resource_id=self.namespace_root, descendants=True) @property def base_capability_names(self): return frozenset(g.capability for g in self.base_grants()) def temporary_ceiling(self): - return temporary_ceiling(self.registry,self.default_scope()) + return temporary_ceiling(self.registry, self.default_scope()) async def open_storage(self): if self.metadata is None: from msg.storage.postgres import PostgresMetadataStore from msg.storage.valkey_bus import ValkeyOutboxSignal - signal=(ValkeyOutboxSignal(self.settings.server.valkey_url) - if self.settings.server.valkey_url else None) - self.metadata=PostgresMetadataStore(self.settings.server.postgres_dsn,signal=signal) + + signal = ( + ValkeyOutboxSignal(self.settings.server.valkey_url) + if self.settings.server.valkey_url + else None + ) + self.metadata = PostgresMetadataStore(self.settings.server.postgres_dsn, signal=signal) if self.contents is None: - self.contents=GitContentStore(self.settings.server.content_dir, - binary_dir=self.settings.server.blob_dir, - staging_dir=self.settings.server.staging_dir, - group_read=self.settings.hosting_content_group_read) + self.contents = GitContentStore( + self.settings.server.content_dir, + binary_dir=self.settings.server.blob_dir, + staging_dir=self.settings.server.staging_dir, + group_read=self.settings.hosting_content_group_read, + ) async def load(self): - require(self.settings.trust_file.is_file(),'root_not_initialized') + require(self.settings.trust_file.is_file(), 'root_not_initialized') # Pin before reading runtime files, then recheck with validated authority. await self.open_storage() async with self.metadata.transaction(write=False) as tx: if self.runtime_generation is None: - self.runtime_generation=RuntimeGeneration.capture(tx) + self.runtime_generation = RuntimeGeneration.capture(tx) else: self.runtime_generation.require_current(tx) - trust=loads(self.settings.trust_file.read_bytes()) - require(set(trust)=={'version','public_key','certificate'} and trust['version']==1,'invalid_trust_anchor') - root_certificate=decode(Certificate,trust['certificate']) - root_public=unb64(trust['public_key'],limit=32) - keys=self.settings.service_keys - required=('online.key','receipt.key','tokens.key') - require(all((keys/name).is_file() for name in required),'service_keys_missing') - self.online_signer=Ed25519Signer.from_bytes((keys/'online.key').read_bytes()) - self.receipt_signer=Ed25519Signer.from_bytes((keys/'receipt.key').read_bytes()) - self._token_secret=(keys/'tokens.key').read_bytes() - require(len(self._token_secret)==32,'invalid_service_key') - self._vault_key=hmac.digest(self._token_secret,b'custodial-vault-aesgcm-v1','sha256') - self.cursors=CursorCodec(hmac.digest(self._token_secret,b'cursor-key-v1','sha256')) - self.token_delivery=TokenDelivery(metadata=self.metadata,token_secret=self._token_secret, + trust = loads(self.settings.trust_file.read_bytes()) + require( + set(trust) == {'version', 'public_key', 'certificate'} and trust['version'] == 1, + 'invalid_trust_anchor', + ) + root_certificate = decode(Certificate, trust['certificate']) + root_public = unb64(trust['public_key'], limit=32) + keys = self.settings.service_keys + required = ('online.key', 'receipt.key', 'tokens.key') + require(all((keys / name).is_file() for name in required), 'service_keys_missing') + self.online_signer = Ed25519Signer.from_bytes((keys / 'online.key').read_bytes()) + self.receipt_signer = Ed25519Signer.from_bytes((keys / 'receipt.key').read_bytes()) + self._token_secret = (keys / 'tokens.key').read_bytes() + require(len(self._token_secret) == 32, 'invalid_service_key') + self._vault_key = hmac.digest(self._token_secret, b'custodial-vault-aesgcm-v1', 'sha256') + self.cursors = CursorCodec(hmac.digest(self._token_secret, b'cursor-key-v1', 'sha256')) + self.token_delivery = TokenDelivery( + metadata=self.metadata, + token_secret=self._token_secret, clock=lambda: self.clock(), - recovery_window=lambda: self.settings.credential_delivery_recovery_window) - self.certificates=CertificateValidator(self.registry,root_certificate,root_public,self.settings.service_url,self.clock) + recovery_window=lambda: self.settings.credential_delivery_recovery_window, + ) + self.certificates = CertificateValidator( + self.registry, root_certificate, root_public, self.settings.service_url, self.clock + ) async with self.metadata.transaction(write=False) as tx: self.runtime_generation.require_current(tx) - self.token_delivery.runtime_generation=self.runtime_generation - quarantined=quarantine_active(tx) - await self.certificates.validate(root_certificate.resource_id,tx) - root=await tx.subject(ROOT_SUBJECT) - require(root.local_only,'root_policy_corrupt') + self.token_delivery.runtime_generation = self.runtime_generation + quarantined = quarantine_active(tx) + await self.certificates.validate(root_certificate.resource_id, tx) + root = await tx.subject(ROOT_SUBJECT) + require(root.local_only, 'root_policy_corrupt') # Only a verified installation may import immutable release-owned rules. # This is idempotent per source digest/version and rejects source deletion. from msg.bootstrap import sync_system_sources - marker=self.settings.config_dir/'recovery-drill.json' - quarantined=quarantined or marker.exists() or marker.is_symlink() + + marker = self.settings.config_dir / 'recovery-drill.json' + quarantined = quarantined or marker.exists() or marker.is_symlink() if not quarantined: async with self.metadata.transaction(write=True) as tx: # Recheck under the writer lock; loading a restore must not # mutate release resources before recovery has been accepted. self.runtime_generation.require_current(tx) if not quarantine_active(tx): - await sync_system_sources(tx,self.contents,self.clock(),namespace_root=self.namespace_root, - registry=self.registry) + await sync_system_sources( + tx, + self.contents, + self.clock(), + namespace_root=self.namespace_root, + registry=self.registry, + ) else: - quarantined=True - self.authenticator=AuthenticationService(self.registry,self.certificates,self.settings.service_url,self.clock, - self.primary_ceiling,self.temporary_ceiling) - self.authorizer=AuthorizationService(self.registry,self.certificates) - self.authenticator.runtime_generation=self.runtime_generation - self.authorizer.runtime_generation=self.runtime_generation - self.executor=self.new_executor(self.authenticator) - self.executor.recovery_drill_marker=marker - self.executor.recovery_quarantined=quarantined - self._loaded=True + quarantined = True + self.authenticator = AuthenticationService( + self.registry, + self.certificates, + self.settings.service_url, + self.clock, + self.primary_ceiling, + self.temporary_ceiling, + ) + self.authorizer = AuthorizationService(self.registry, self.certificates) + self.authenticator.runtime_generation = self.runtime_generation + self.authorizer.runtime_generation = self.runtime_generation + self.executor = self.new_executor(self.authenticator) + self.executor.recovery_drill_marker = marker + self.executor.recovery_quarantined = quarantined + self._loaded = True return self def new_executor(self, authenticator=None): """Compose identical limits/ports for HTTP and the restricted SSH entry.""" - executor=OperationExecutor(self.registry,self.metadata,self.contents, + executor = OperationExecutor( + self.registry, + self.metadata, + self.contents, self.authenticator if authenticator is None else authenticator, - self.authorizer,self.clock,self.receipt_signer, + self.authorizer, + self.clock, + self.receipt_signer, max_request_bytes=self.settings.server.limits.max_request_bytes, result_projection=self._result_projection, - event_notifications=self._event_notifications) - executor.runtime_generation=self.runtime_generation - executor.response_hook=self.token_delivery.release - executor.recovery_drill_marker=self.settings.config_dir/'recovery-drill.json' + event_notifications=self._event_notifications, + ) + executor.runtime_generation = self.runtime_generation + executor.response_hook = self.token_delivery.release + executor.recovery_drill_marker = self.settings.config_dir / 'recovery-drill.json' if self.executor is not None: - executor.recovery_quarantined=self.executor.recovery_quarantined + executor.recovery_quarantined = self.executor.recovery_quarantined return executor async def _result_projection(self, context, request, session, resource, *, fields): from msg.plugins.discovery import read_projection - return await read_projection(self,context,request,session,resource.id, - revision=resource.revision,fields=fields) + + return await read_projection( + self, context, request, session, resource.id, revision=resource.revision, fields=fields + ) async def _event_notifications(self, session, event): if 'communication' not in self.settings.server.plugins: return from msg.plugins.communication import enqueue_domain_webhooks - await enqueue_domain_webhooks(self,session,event) + + await enqueue_domain_webhooks(self, session, event) from msg.plugins.watches import enqueue - await enqueue(self,session,event) - async def online_issuer(self,tx): - issuer=tx.setting('online_ca_certificate') - require(issuer is not None,'issuer_not_ready') - cert=await self.certificates.validate(issuer,tx) - require(cert.kind=='ca' and cert.key_id==self.online_signer.key_id,'issuer_not_ready') + await enqueue(self, session, event) + + async def online_issuer(self, tx): + issuer = tx.setting('online_ca_certificate') + require(issuer is not None, 'issuer_not_ready') + cert = await self.certificates.validate(issuer, tx) + require(cert.kind == 'ca' and cert.key_id == self.online_signer.key_id, 'issuer_not_ready') return cert - def issued_token(self,request,subject): - return self.token_delivery.issued_token(request,subject) + def issued_token(self, request, subject): + return self.token_delivery.issued_token(request, subject) - recovery_verifier=staticmethod(recovery_verifier) + recovery_verifier = staticmethod(recovery_verifier) - def record_token_delivery(self,tx,request,credential,now,*,recovery_deadline=None): - return self.token_delivery.record_token_delivery(tx,request,credential,now,recovery_deadline=recovery_deadline) + def record_token_delivery(self, tx, request, credential, now, *, recovery_deadline=None): + return self.token_delivery.record_token_delivery( + tx, request, credential, now, recovery_deadline=recovery_deadline + ) - async def _secrets_for_caller(self,request,result): - return await self.token_delivery.release(request,result) + async def _secrets_for_caller(self, request, result): + return await self.token_delivery.release(request, result) async def close(self): if self.metadata is not None: diff --git a/src/msg/atomic_file.py b/src/msg/atomic_file.py index beaf7034..980510fd 100644 --- a/src/msg/atomic_file.py +++ b/src/msg/atomic_file.py @@ -1,22 +1,23 @@ """Atomic local-file replacement with caller-selected mode and durable syncs.""" + from __future__ import annotations import os -from pathlib import Path import tempfile +from pathlib import Path def durable_write(path: Path, data: bytes, mode: int = 0o600): - path.parent.mkdir(parents=True,exist_ok=True) - fd,temporary=tempfile.mkstemp(prefix='.pending-',dir=path.parent) + path.parent.mkdir(parents=True, exist_ok=True) + fd, temporary = tempfile.mkstemp(prefix='.pending-', dir=path.parent) try: - os.fchmod(fd,mode) - with os.fdopen(fd,'wb') as stream: + os.fchmod(fd, mode) + with os.fdopen(fd, 'wb') as stream: stream.write(data) stream.flush() os.fsync(stream.fileno()) - os.replace(temporary,path) - directory=os.open(path.parent,os.O_RDONLY|os.O_DIRECTORY) + os.replace(temporary, path) + directory = os.open(path.parent, os.O_RDONLY | os.O_DIRECTORY) try: os.fsync(directory) finally: diff --git a/src/msg/bootstrap.py b/src/msg/bootstrap.py index 74e2caa3..241a9c9d 100644 --- a/src/msg/bootstrap.py +++ b/src/msg/bootstrap.py @@ -1,235 +1,442 @@ """Versioned public bootstrap. Private-key provisioning lives in admin only.""" + from __future__ import annotations + +import re from dataclasses import replace from importlib.resources import files from pathlib import Path -import re -from msg.constants import * -from msg.core.codec import loads,wire,canonical,digest,decode -from msg.core.errors import require -from msg.core.models import Resource,ResourceRef,Revision,Subject,Organization,Membership +from msg.constants import ( + ADMINS_GROUP as ADMINS_GROUP, + CA_SPACE as CA_SPACE, + CERT_SPACE as CERT_SPACE, + CSR_SPACE as CSR_SPACE, + ONLINE_CA as ONLINE_CA, + PROTOCOL_VERSION as PROTOCOL_VERSION, + PUBLIC_GROUP as PUBLIC_GROUP, + ROOT_SPACE as ROOT_SPACE, + ROOT_SUBJECT as ROOT_SUBJECT, + TOOLS_SPACE as TOOLS_SPACE, +) +from msg.core.codec import canonical, decode, digest, loads, wire +from msg.core.errors import require +from msg.core.models import Organization, Resource, Revision, Subject -RULE_NAMES=('identity','read-write','auth','topics','files','recovery','security','protocol') -SOURCE_HEADER=re.compile(r'\n') -REQUIRES_HEADER=re.compile(r'') -ROOT_WEB_LOGO=files('msg.data').joinpath('logo.svg').read_text(encoding='utf-8') -ROOT_WEB_SAMPLE=('' +RULE_NAMES = ( + 'identity', + 'read-write', + 'auth', + 'topics', + 'files', + 'recovery', + 'security', + 'protocol', +) +SOURCE_HEADER = re.compile(r'\n') +REQUIRES_HEADER = re.compile(r'') +ROOT_WEB_LOGO = files('msg.data').joinpath('logo.svg').read_text(encoding='utf-8') +ROOT_WEB_SAMPLE = ( + '' '' 'msg.lmm.best' - '
'+ROOT_WEB_LOGO+'

msg.lmm.best

' + '
' + ROOT_WEB_LOGO + '

msg.lmm.best

' '

让 Agent 和人清楚地交流、分享与继续工作。

' '

发布、回复、私聊、交换文件；公开和私密由你决定。

' - '

Agent 入口

\n').encode('utf-8') + '

Agent 入口

\n' +).encode('utf-8') # Public identity and location are deliberately independent of the release file. # A release can change a source path only with an explicit migration declaration. -RULE_SPECS=(('msg.bootstrap','r_agents',ROOT_SPACE,'AGENTS.md','AGENTS.md',1000), - ('msg.rules.index','r_rule_index','r_rules','_index.md','rules/_index.md',2400), - *((f'msg.{name}',f'r_rule_{name.replace("-","_")}','r_rules', - name,f'rules/{name}.md',4000) for name in RULE_NAMES)) -RULE_PATHS={rule_id:('/AGENTS.md' if rid=='r_agents' else - '/_rules/_index.md' if rid=='r_rule_index' else '/_rules/'+name) - for rule_id,rid,_,name,_,_ in RULE_SPECS} +RULE_SPECS = ( + ('msg.bootstrap', 'r_agents', ROOT_SPACE, 'AGENTS.md', 'AGENTS.md', 1000), + ('msg.rules.index', 'r_rule_index', 'r_rules', '_index.md', 'rules/_index.md', 2400), + *( + ( + f'msg.{name}', + f'r_rule_{name.replace("-", "_")}', + 'r_rules', + name, + f'rules/{name}.md', + 4000, + ) + for name in RULE_NAMES + ), +) +RULE_PATHS = { + rule_id: ( + '/AGENTS.md' + if rid == 'r_agents' + else '/_rules/_index.md' + if rid == 'r_rule_index' + else '/_rules/' + name + ) + for rule_id, rid, _, name, _, _ in RULE_SPECS +} # Release maintainers declare a relocation in both maps in the same change. -SOURCE_PATH_OVERRIDES={} -SOURCE_MIGRATIONS={} +SOURCE_PATH_OVERRIDES = {} +SOURCE_MIGRATIONS = {} # Release-only pins: rule_id -> {source_path: relative .md path, version: int, # digest: sha256:}. Remove active links/operation dependencies in the same # release; keep RULE_SPECS identity and this pin so upgrades preserve history. # Retirement archives existing resources; fresh installs invent no old revision. # No current rule is retired. A pin mismatch requires an explicit staged upgrade. -SOURCE_RETIREMENTS={} +SOURCE_RETIREMENTS = {} def system_source_root(): - packaged=files('msg.data').joinpath('system') - require(packaged.is_dir(),'system_source_missing') + packaged = files('msg.data').joinpath('system') + require(packaged.is_dir(), 'system_source_missing') return packaged -async def sync_system_sources(tx,contents,now, *, source_root=None,namespace_root=ROOT_SPACE, - source_paths=None,migrations=None,retirements=None,registry=None): +async def sync_system_sources( + tx, + contents, + now, + *, + source_root=None, + namespace_root=ROOT_SPACE, + source_paths=None, + migrations=None, + retirements=None, + registry=None, +): """Sync each release-owned document independently; never delete a source implicitly.""" - root=Path(source_root or system_source_root()) - source_paths=dict(SOURCE_PATH_OVERRIDES if source_paths is None else source_paths) - migrations=dict(SOURCE_MIGRATIONS if migrations is None else migrations) - require(set(source_paths)<=RULE_PATHS.keys() and set(migrations)<=RULE_PATHS.keys(), - 'unknown_rule_id') - retirements=dict(SOURCE_RETIREMENTS if retirements is None else retirements) - require(set(retirements)<=RULE_PATHS.keys()-{'msg.bootstrap','msg.rules.index'}, - 'system_source_invalid_retirement') - require(not set(retirements)&(source_paths.keys()|migrations.keys()), - 'system_source_invalid_retirement') - for rule_id,pin in retirements.items(): - require(isinstance(pin,dict) and set(pin)=={'source_path','version','digest'} and - isinstance(pin['source_path'],str) and - pin['source_path']==Path(pin['source_path']).as_posix() and - not Path(pin['source_path']).is_absolute() and - '..' not in Path(pin['source_path']).parts and - pin['source_path'].endswith('.md') and - type(pin['version']) is int and pin['version']>0 and - isinstance(pin['digest'],str) and - re.fullmatch(r'sha256:[0-9a-f]{64}',pin['digest']) is not None, - 'system_source_invalid_retirement',rule_id) + root = Path(source_root or system_source_root()) + source_paths = dict(SOURCE_PATH_OVERRIDES if source_paths is None else source_paths) + migrations = dict(SOURCE_MIGRATIONS if migrations is None else migrations) + require( + set(source_paths) <= RULE_PATHS.keys() and set(migrations) <= RULE_PATHS.keys(), + 'unknown_rule_id', + ) + retirements = dict(SOURCE_RETIREMENTS if retirements is None else retirements) + require( + set(retirements) <= RULE_PATHS.keys() - {'msg.bootstrap', 'msg.rules.index'}, + 'system_source_invalid_retirement', + ) + require( + not set(retirements) & (source_paths.keys() | migrations.keys()), + 'system_source_invalid_retirement', + ) + for rule_id, pin in retirements.items(): + require( + isinstance(pin, dict) + and set(pin) == {'source_path', 'version', 'digest'} + and isinstance(pin['source_path'], str) + and pin['source_path'] == Path(pin['source_path']).as_posix() + and not Path(pin['source_path']).is_absolute() + and '..' not in Path(pin['source_path']).parts + and pin['source_path'].endswith('.md') + and type(pin['version']) is int + and pin['version'] > 0 + and isinstance(pin['digest'], str) + and re.fullmatch(r'sha256:[0-9a-f]{64}', pin['digest']) is not None, + 'system_source_invalid_retirement', + rule_id, + ) if retirements: - require(registry is not None and all(not set(spec.requires_rules)&retirements.keys() - for spec in registry.operations()),'system_source_retirement_referenced') - specs=[] - for rule_id,rid,parent,name,default_path,max_bytes in RULE_SPECS: + require( + registry is not None + and all( + not set(spec.requires_rules) & retirements.keys() for spec in registry.operations() + ), + 'system_source_retirement_referenced', + ) + specs = [] + for rule_id, rid, parent, name, default_path, max_bytes in RULE_SPECS: if rule_id in retirements: continue - relative=source_paths.get(rule_id,default_path) - require(isinstance(relative,str) and relative.endswith('.md') and - relative==Path(relative).as_posix() and - not Path(relative).is_absolute() and '..' not in Path(relative).parts, - 'system_source_invalid_path',rule_id) - specs.append((relative,rid,namespace_root if parent==ROOT_SPACE else parent, - name,rule_id,max_bytes)) - require(len({s[0] for s in specs})==len(specs),'duplicate_system_source_path') - expected={s[0] for s in specs} - actual={p.relative_to(root).as_posix() for p in root.rglob('*.md')} + relative = source_paths.get(rule_id, default_path) + require( + isinstance(relative, str) + and relative.endswith('.md') + and relative == Path(relative).as_posix() + and not Path(relative).is_absolute() + and '..' not in Path(relative).parts, + 'system_source_invalid_path', + rule_id, + ) + specs.append(( + relative, + rid, + namespace_root if parent == ROOT_SPACE else parent, + name, + rule_id, + max_bytes, + )) + require(len({s[0] for s in specs}) == len(specs), 'duplicate_system_source_path') + expected = {s[0] for s in specs} + actual = {p.relative_to(root).as_posix() for p in root.rglob('*.md')} for relative in sorted(actual): - candidate=root/relative - require(not candidate.is_symlink() and candidate.resolve().is_relative_to(root.resolve()), - 'system_source_invalid_path',relative) - observed={} + candidate = root / relative + require( + not candidate.is_symlink() and candidate.resolve().is_relative_to(root.resolve()), + 'system_source_invalid_path', + relative, + ) + observed = {} for relative in sorted(actual): - match=SOURCE_HEADER.match((root/relative).read_text(encoding='utf-8')) - require(match is not None,'system_source_invalid_header',relative) - rule_id=match.group(1) - require(rule_id in RULE_PATHS,'unknown_rule_id',relative) - require(rule_id not in observed,'duplicate_rule_id',rule_id) - require(rule_id not in retirements,'system_source_retirement_source_present',rule_id) - observed[rule_id]=relative - require(not expected-actual,'system_source_deleted_requires_migration', - details={'paths':sorted(expected-actual)}) - require(not actual-expected,'system_source_inventory_mismatch', - details={'unknown':sorted(actual-expected)}) - prepared=[] - rule_ids=set() - for relative,rid,parent,name,expected_rule_id,max_bytes in specs: - path=root/relative - require(path.is_file() and not path.is_symlink(),'system_source_missing',relative) - raw=path.read_bytes() - require(len(raw)<=max_bytes,'system_source_too_large',relative) + match = SOURCE_HEADER.match((root / relative).read_text(encoding='utf-8')) + require(match is not None, 'system_source_invalid_header', relative) + rule_id = match.group(1) + require(rule_id in RULE_PATHS, 'unknown_rule_id', relative) + require(rule_id not in observed, 'duplicate_rule_id', rule_id) + require(rule_id not in retirements, 'system_source_retirement_source_present', rule_id) + observed[rule_id] = relative + require( + not expected - actual, + 'system_source_deleted_requires_migration', + details={'paths': sorted(expected - actual)}, + ) + require( + not actual - expected, + 'system_source_inventory_mismatch', + details={'unknown': sorted(actual - expected)}, + ) + prepared = [] + rule_ids = set() + for relative, rid, parent, name, expected_rule_id, max_bytes in specs: + path = root / relative + require(path.is_file() and not path.is_symlink(), 'system_source_missing', relative) + raw = path.read_bytes() + require(len(raw) <= max_bytes, 'system_source_too_large', relative) try: - source_text=raw.decode('utf-8') + source_text = raw.decode('utf-8') except UnicodeError as exc: raise ValueError('system_source_invalid_utf8') from exc - header=SOURCE_HEADER.match(source_text) - require(header is not None,'system_source_invalid_header',relative) - rule_id,version=header.group(1),int(header.group(2)) - require(rule_id in RULE_PATHS and rule_id==expected_rule_id,'unknown_rule_id',relative) - require(rule_id not in rule_ids,'duplicate_rule_id',rule_id) + header = SOURCE_HEADER.match(source_text) + require(header is not None, 'system_source_invalid_header', relative) + rule_id, version = header.group(1), int(header.group(2)) + require(rule_id in RULE_PATHS and rule_id == expected_rule_id, 'unknown_rule_id', relative) + require(rule_id not in rule_ids, 'duplicate_rule_id', rule_id) rule_ids.add(rule_id) - note_match=re.search(r'',source_text) - change_note=note_match.group(1) if note_match else None - required=REQUIRES_HEADER.findall(source_text) - require(len(required)==source_text.count('', source_text) + change_note = note_match.group(1) if note_match else None + required = REQUIRES_HEADER.findall(source_text) + require( + len(required) == source_text.count('\nA new sentence.\n') + first = (await tx.resource(rid)).revision + source = await tx.revision( + __import__('msg.core.models', fromlist=['ResourceRef']).ResourceRef(id=rid) + ) + assert (source.source_kind, source.source_version, source.source_digest) == ( + 'release', + 1, + source.content.digest, + ) + copied = tmp_path / 'system' + shutil.copytree(system_source_root(), copied) + path = copied / 'rules' / 'identity.md' + path.write_text( + path.read_text().replace('version: 1', 'version: 2') + + '\n\nA new sentence.\n' + ) async with app.metadata.transaction(write=True) as tx: - await sync_system_sources(tx,app.contents,NOW,source_root=copied) - history=await call(app,'discovery.get',{'id':rid,'view':'history','limit':1}) - assert history.status=='ok' and history.data['cursor'] - older=await call(app,'discovery.get',{'id':rid,'view':'history','limit':1, - 'cursor':history.data['cursor']}) - records=(*history.data['revisions'],*older.data['revisions']) - assert {item['source_version'] for item in records}=={1,2} - assert any(item.get('change_note')=='Clarified key ownership.' for item in records) - current=(await call(app,'discovery.get',{'id':rid})).data['revision'] - diff=await call(app,'discovery.diff_view',{'id':rid,'known_revision':first}) - assert diff.status=='ok' and diff.data['from']['revision']==first - assert diff.data['to']['revision']==current and '+A new sentence.' in diff.data['diff'] - assert diff.data['to_source']['source_version']==2 + await sync_system_sources(tx, app.contents, NOW, source_root=copied) + history = await call(app, 'discovery.get', {'id': rid, 'view': 'history', 'limit': 1}) + assert history.status == 'ok' and history.data['cursor'] + older = await call( + app, + 'discovery.get', + {'id': rid, 'view': 'history', 'limit': 1, 'cursor': history.data['cursor']}, + ) + records = (*history.data['revisions'], *older.data['revisions']) + assert {item['source_version'] for item in records} == {1, 2} + assert any(item.get('change_note') == 'Clarified key ownership.' for item in records) + current = (await call(app, 'discovery.get', {'id': rid})).data['revision'] + diff = await call(app, 'discovery.diff_view', {'id': rid, 'known_revision': first}) + assert diff.status == 'ok' and diff.data['from']['revision'] == first + assert diff.data['to']['revision'] == current and '+A new sentence.' in diff.data['diff'] + assert diff.data['to_source']['source_version'] == 2 @pytest.mark.asyncio async def test_user_cannot_forge_release_source_and_rule_links_resolve(installed): app, _ = installed - key,subject,_=await register(app,'source-user') - forged=await call(app,'content.post_create',{'parent':'/main','body':'a post', - 'source_kind':'release','source_version':999},key=key,subject=subject) - assert forged.status=='error' and forged.error.code=='schema_validation' + key, subject, _ = await register(app, 'source-user') + forged = await call( + app, + 'content.post_create', + {'parent': '/main', 'body': 'a post', 'source_kind': 'release', 'source_version': 999}, + key=key, + subject=subject, + ) + assert forged.status == 'error' and forged.error.code == 'schema_validation' for operation in app.registry.operations('network'): - described=app.registry.describe(operation) - links=described['requires_rules'] + described = app.registry.describe(operation) + links = described['requires_rules'] assert links for link in links: assert link['rule_id'].startswith('msg.') and link['path'].startswith('/_rules/') - rule=await call(app,'discovery.get',{'id':link['path']}) - assert rule.status=='ok' and link['rule_id'] in rule.data['content'] - schema=await call(app,'discovery.schema',{'operation':'identity.register'}) - assert schema.status=='ok' and schema.data['operation']['requires_rules'] + rule = await call(app, 'discovery.get', {'id': link['path']}) + assert rule.status == 'ok' and link['rule_id'] in rule.data['content'] + schema = await call(app, 'discovery.schema', {'operation': 'identity.register'}) + assert schema.status == 'ok' and schema.data['operation']['requires_rules'] @pytest.mark.asyncio async def test_history_cursor_rechecks_principal_and_current_visibility(installed): app, _ = installed - key,subject,_=await register(app,'history-owner') - created=await call(app,'content.post_create',{'parent':'/main','body':'first'}, - key=key,subject=subject) - rid=created.resources[0].id - edited=await call(app,'content.post_edit',{'id':rid, - 'expected_revision':created.resources[0].revision,'body':'second'}, - key=key,subject=subject,expected=((rid,created.data['generation']),)) - assert edited.status=='ok' - page=await call(app,'discovery.get',{'id':rid,'view':'history','limit':1}) - assert page.status=='ok' and page.data['cursor'] - wrong=await call(app,'discovery.get',{'id':rid,'view':'history','limit':1, - 'cursor':page.data['cursor']},key=key,subject=subject) - assert wrong.status=='error' and wrong.error.code=='cursor_principal_mismatch' - hidden=await call(app,'content.chmod',{'id':rid,'mode':'0600'},key=key,subject=subject, - expected=((rid,edited.data['generation']),)) - assert hidden.status=='ok' - stale=await call(app,'discovery.get',{'id':rid,'view':'history','limit':1, - 'cursor':page.data['cursor']}) - assert stale.status=='error' + key, subject, _ = await register(app, 'history-owner') + created = await call( + app, 'content.post_create', {'parent': '/main', 'body': 'first'}, key=key, subject=subject + ) + rid = created.resources[0].id + edited = await call( + app, + 'content.post_edit', + {'id': rid, 'expected_revision': created.resources[0].revision, 'body': 'second'}, + key=key, + subject=subject, + expected=((rid, created.data['generation']),), + ) + assert edited.status == 'ok' + page = await call(app, 'discovery.get', {'id': rid, 'view': 'history', 'limit': 1}) + assert page.status == 'ok' and page.data['cursor'] + wrong = await call( + app, + 'discovery.get', + {'id': rid, 'view': 'history', 'limit': 1, 'cursor': page.data['cursor']}, + key=key, + subject=subject, + ) + assert wrong.status == 'error' and wrong.error.code == 'cursor_principal_mismatch' + hidden = await call( + app, + 'content.chmod', + {'id': rid, 'mode': '0600'}, + key=key, + subject=subject, + expected=((rid, edited.data['generation']),), + ) + assert hidden.status == 'ok' + stale = await call( + app, + 'discovery.get', + {'id': rid, 'view': 'history', 'limit': 1, 'cursor': page.data['cursor']}, + ) + assert stale.status == 'error' diff --git a/tests/test_root_rotation.py b/tests/test_root_rotation.py index ac15e34b..ddd88f31 100644 --- a/tests/test_root_rotation.py +++ b/tests/test_root_rotation.py @@ -1,37 +1,49 @@ import pytest -from msg.admin.rotation import prepare,complete,journal_path -from msg.security.crypto import Ed25519Signer,open_private_key -from msg.core.codec import loads,wire +from test_service import NOW, call, register + +from msg.admin.root import _approve_csr, root_envelope +from msg.admin.rotation import complete, journal_path, prepare from msg.application import Application -from msg.admin.root import _approve_csr,root_envelope -from test_service import call,register,NOW +from msg.core.codec import loads, wire +from msg.security.crypto import Ed25519Signer, open_private_key + @pytest.mark.asyncio async def test_root_rotation_retires_old_chain_and_requires_explicit_online_ca_approval(installed): - app,root=installed - key,uid,cert=await register(app,'rotation-agent') - next_key=Ed25519Signer.generate() - pin='new-root-long-passphrase' - journal=prepare(app,next_key,pin,old_signer=root,operator='isolated-test-console') - assert journal['new_trust']['certificate']['issuance']['max_child_ca_depth']==3 - assert journal['online_csr']['issuance']['max_child_ca_depth']==0 + app, root = installed + key, uid, cert = await register(app, 'rotation-agent') + next_key = Ed25519Signer.generate() + pin = 'new-root-long-passphrase' + journal = prepare(app, next_key, pin, old_signer=root, operator='isolated-test-console') + assert journal['new_trust']['certificate']['issuance']['max_child_ca_depth'] == 3 + assert journal['online_csr']['issuance']['max_child_ca_depth'] == 0 assert journal_path(app).exists() assert next_key.private_bytes() not in journal_path(app).read_bytes() - outcome=await complete(app,journal,pin=pin) + outcome = await complete(app, journal, pin=pin) assert outcome['restart_required'] and not journal_path(app).exists() - stale=await call(app,'discovery.get',{'id':'/main'}) - assert stale.error.code=='service_restart_required' - fresh=Application(app.settings,clock=lambda:NOW);await fresh.load() + stale = await call(app, 'discovery.get', {'id': '/main'}) + assert stale.error.code == 'service_restart_required' + fresh = Application(app.settings, clock=lambda: NOW) + await fresh.load() try: - denied=await call(fresh,'discovery.get',{'id':'/main'},key=key,subject=uid,certs=(cert,)) - assert denied.error.code=='certificate_revoked' - renewal=await call(fresh,'identity.certificate_renew',{},key=key,subject=uid) - assert renewal.error.code=='issuer_not_ready' - await _approve_csr(fresh,outcome['online_ca_request'],next_key,expected_digest=None,operator='isolated-test-console') - renewal=await call(fresh,'identity.certificate_renew',{},key=key,subject=uid) - assert renewal.error.code=='renewal_source_required',wire(renewal) - assert fresh.certificates.root_public_key==next_key.public_key - assert not (app.settings.config_dir/'root').exists() - envelope=loads(root_envelope(app.settings.config_dir).read_bytes()) - assert open_private_key(envelope,pin)==next_key.private_bytes() - finally:await fresh.close() + denied = await call( + fresh, 'discovery.get', {'id': '/main'}, key=key, subject=uid, certs=(cert,) + ) + assert denied.error.code == 'certificate_revoked' + renewal = await call(fresh, 'identity.certificate_renew', {}, key=key, subject=uid) + assert renewal.error.code == 'issuer_not_ready' + await _approve_csr( + fresh, + outcome['online_ca_request'], + next_key, + expected_digest=None, + operator='isolated-test-console', + ) + renewal = await call(fresh, 'identity.certificate_renew', {}, key=key, subject=uid) + assert renewal.error.code == 'renewal_source_required', wire(renewal) + assert fresh.certificates.root_public_key == next_key.public_key + assert not (app.settings.config_dir / 'root').exists() + envelope = loads(root_envelope(app.settings.config_dir).read_bytes()) + assert open_private_key(envelope, pin) == next_key.private_bytes() + finally: + await fresh.close() diff --git a/tests/test_root_rotation_resume.py b/tests/test_root_rotation_resume.py index e7f6e68f..02904c20 100644 --- a/tests/test_root_rotation_resume.py +++ b/tests/test_root_rotation_resume.py @@ -1,16 +1,17 @@ """Use real isolated PostgreSQL; local use cases do not simulate a physical console.""" + from copy import deepcopy import pytest +from test_service import NOW from msg.admin.root import root_envelope -from msg.admin.rotation import prepare, complete, journal_path +from msg.admin.rotation import complete, journal_path, prepare from msg.application import Application from msg.core.codec import canonical, loads from msg.core.errors import Failure -from msg.security.crypto import Ed25519Signer, seal_private_key, open_private_key +from msg.security.crypto import Ed25519Signer, open_private_key, seal_private_key from msg.storage.git import durable_write -from test_service import NOW OLD_PIN = 'old-protected-envelope-test-passphrase' NEW_PIN = 'new-protected-envelope-test-passphrase' @@ -25,8 +26,11 @@ def old_envelope(app, root): @pytest.mark.asyncio @pytest.mark.parametrize('failure_at', ['key', 'trust']) -async def test_rotation_resumes_without_archiving_new_key_as_old(installed, monkeypatch, failure_at): +async def test_rotation_resumes_without_archiving_new_key_as_old( + installed, monkeypatch, failure_at +): from msg.admin import rotation + app, root = installed path, original_key = old_envelope(app, root) successor = Ed25519Signer.generate() @@ -35,32 +39,37 @@ async def test_rotation_resumes_without_archiving_new_key_as_old(installed, monk assert journal['version'] == 2 and pending.parent.stat().st_mode & 0o777 == 0o700 fail_path = path if failure_at == 'key' else app.settings.trust_file real_write = rotation.durable_write + def interrupted(target, data, **kwargs): if target == fail_path: raise OSError('injected protected-file interruption') return real_write(target, data, **kwargs) + with monkeypatch.context() as patch: patch.setattr(rotation, 'durable_write', interrupted) with pytest.raises(OSError, match='injected'): await complete(app, journal, pin=NEW_PIN) assert pending.exists() async with app.metadata.transaction(write=False) as tx: - assert tx.setting('active_root_certificate') == journal['new_trust']['certificate']['resource_id'] + assert ( + tx.setting('active_root_certificate') + == journal['new_trust']['certificate']['resource_id'] + ) assert await tx.certificate_revoked(journal['old_certificate']['resource_id']) before = tx.one('SELECT COUNT(*) FROM audit')[0] # Resume must work before Application.load can validate partially switched trust files. - fresh = Application(app.settings, clock=lambda:NOW) + fresh = Application(app.settings, clock=lambda: NOW) try: result = await complete(fresh, loads(pending.read_bytes()), pin=NEW_PIN) assert result['restart_required'] and not pending.exists() assert root_envelope(app.settings.config_dir) == path assert open_private_key(loads(path.read_bytes()), NEW_PIN) == successor.private_bytes() - history = path.parent/'history'/journal['old_certificate']['resource_id']/'key.json' + history = path.parent / 'history' / journal['old_certificate']['resource_id'] / 'key.json' assert history.read_bytes() == original_key assert open_private_key(loads(history.read_bytes()), OLD_PIN) == root.private_bytes() assert path.stat().st_mode & 0o777 == history.stat().st_mode & 0o777 == 0o600 assert history.parent.stat().st_mode & 0o777 == 0o700 - assert not (app.settings.config_dir/'root').exists() + assert not (app.settings.config_dir / 'root').exists() async with fresh.metadata.transaction(write=False) as tx: assert tx.one('SELECT COUNT(*) FROM audit')[0] == before await fresh.load() @@ -73,7 +82,9 @@ def interrupted(target, data, **kwargs): async def test_replacement_csr_is_rejected_before_any_commit(installed): app, root = installed path, original = old_envelope(app, root) - journal = prepare(app, Ed25519Signer.generate(), NEW_PIN, old_signer=root, operator='isolated-test') + journal = prepare( + app, Ed25519Signer.generate(), NEW_PIN, old_signer=root, operator='isolated-test' + ) changed = deepcopy(journal) changed['online_csr']['resource_id'] = 'csr_replaced_after_confirmation' with pytest.raises(Failure, match='invalid_signature'): @@ -93,7 +104,9 @@ async def test_explicit_lost_root_key_rotation_uses_bound_journal(installed): assert journal['statement']['previous_key_proved'] is False and journal['old_signature'] is None await complete(app, journal, pin=NEW_PIN) assert open_private_key(loads(path.read_bytes()), NEW_PIN) == successor.private_bytes() - assert (path.parent/'history'/journal['old_certificate']['resource_id']/'key.json').read_bytes() == original + assert ( + path.parent / 'history' / journal['old_certificate']['resource_id'] / 'key.json' + ).read_bytes() == original @pytest.mark.asyncio @@ -101,9 +114,12 @@ async def test_rotation_back_to_retired_root_key_is_rejected_before_commit(insta app, root = installed old_envelope(app, root) successor = Ed25519Signer.generate() - await complete(app, prepare(app, successor, NEW_PIN, old_signer=root, operator='isolated-test'), - pin=NEW_PIN) - fresh = Application(app.settings, clock=lambda:NOW) + await complete( + app, + prepare(app, successor, NEW_PIN, old_signer=root, operator='isolated-test'), + pin=NEW_PIN, + ) + fresh = Application(app.settings, clock=lambda: NOW) await fresh.load() try: path = root_envelope(app.settings.config_dir) @@ -114,7 +130,9 @@ async def test_rotation_back_to_retired_root_key_is_rejected_before_commit(insta assert journal_path(app).exists() assert path.read_bytes() == installed_key and app.settings.trust_file.read_bytes() == trust async with fresh.metadata.transaction(write=False) as tx: - assert tx.setting('active_root_certificate') == journal['old_certificate']['resource_id'] + assert ( + tx.setting('active_root_certificate') == journal['old_certificate']['resource_id'] + ) assert (await tx.credential(root.key_id)).revoked_at is not None assert (await tx.credential(successor.key_id)).revoked_at is None finally: diff --git a/tests/test_route_effect_matrix.py b/tests/test_route_effect_matrix.py index 3493e43d..6e4fefd2 100644 --- a/tests/test_route_effect_matrix.py +++ b/tests/test_route_effect_matrix.py @@ -1,119 +1,166 @@ """Executable ordinary-route matrix: reads and rejected writes preserve state.""" + from datetime import timedelta import httpx import pytest +from test_service import NOW, call, register from msg.core.codec import b64, canonical, wire from msg.core.requests import request_for from msg.transports.http import RouteEffect, classify_route, create_app -from test_service import NOW, call, register async def database_snapshot(app): """Compare complete rows, so a read counter or an updated ACK cannot hide.""" async with app.metadata.transaction(write=False) as tx: - tables=[row[0] for row in tx.rows( - "SELECT tablename FROM pg_tables WHERE schemaname='public' " - "AND tablename<>'schema_version' ORDER BY tablename")] - return {table:sorted(repr(tuple(row)) for row in tx.rows( - f'SELECT * FROM {table}')) for table in tables} + tables = [ + row[0] + for row in tx.rows( + "SELECT tablename FROM pg_tables WHERE schemaname='public' " + "AND tablename<>'schema_version' ORDER BY tablename" + ) + ] + return { + table: sorted(repr(tuple(row)) for row in tx.rows(f'SELECT * FROM {table}')) + for table in tables + } @pytest.mark.asyncio async def test_ordinary_read_and_failed_route_matrix_preserves_all_authority_rows(installed): - app,_=installed - key,subject,_=await register(app,'route-effect-owner') - post=await call(app,'content.post_create', - {'parent':'/main','name':'effect-probe','body':'route effect marker'}, - key=key,subject=subject) - repo=await call(app,'git.create',{'parent':'/@route-effect-owner','name':'code.git'}, - key=key,subject=subject) - assert post.status==repo.status=='ok' - rid=post.resources[0].id - repo_id=repo.resources[0].id - read_repo=repo.data['read_url'].removeprefix(app.settings.service_url.rstrip('/')) - write=request_for('content.post_create', - {'parent':'/main','body':'must never be written by an ordinary path'}, - app.settings.service_url,signer=key,subject=subject, - request_id='ordinary-route-matrix',expires_at=NOW+timedelta(seconds=90)) - encoded=b64(canonical(write)) - before=await database_snapshot(app) - async with httpx.AsyncClient(transport=httpx.ASGITransport(app=create_app(app)), - base_url='http://testserver',follow_redirects=False) as http: - reads=( - ('GET','/',200),('GET','/main',200),('HEAD','/main',200), - ('GET','/main/effect-probe',308),('HEAD','/main/effect-probe',308), - ('GET',f'/_read/{rid}/json',200),('HEAD',f'/_r/{rid}/meta',200), - ('GET','/rss',200),('GET','/latest/post',200), - ('GET','/_search?query=route',200), - ('GET','/_s/q/2/s/%2Fmain/t/route/f/b/fc/type,tag',200), - ('GET',f'{read_repo}/info/refs?service=git-upload-pack',200), + app, _ = installed + key, subject, _ = await register(app, 'route-effect-owner') + post = await call( + app, + 'content.post_create', + {'parent': '/main', 'name': 'effect-probe', 'body': 'route effect marker'}, + key=key, + subject=subject, + ) + repo = await call( + app, + 'git.create', + {'parent': '/@route-effect-owner', 'name': 'code.git'}, + key=key, + subject=subject, + ) + assert post.status == repo.status == 'ok' + rid = post.resources[0].id + repo_id = repo.resources[0].id + read_repo = repo.data['read_url'].removeprefix(app.settings.service_url.rstrip('/')) + write = request_for( + 'content.post_create', + {'parent': '/main', 'body': 'must never be written by an ordinary path'}, + app.settings.service_url, + signer=key, + subject=subject, + request_id='ordinary-route-matrix', + expires_at=NOW + timedelta(seconds=90), + ) + encoded = b64(canonical(write)) + before = await database_snapshot(app) + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=create_app(app)), + base_url='http://testserver', + follow_redirects=False, + ) as http: + reads = ( + ('GET', '/', 200), + ('GET', '/main', 200), + ('HEAD', '/main', 200), + ('GET', '/main/effect-probe', 308), + ('HEAD', '/main/effect-probe', 308), + ('GET', f'/_read/{rid}/json', 200), + ('HEAD', f'/_r/{rid}/meta', 200), + ('GET', '/rss', 200), + ('GET', '/latest/post', 200), + ('GET', '/_search?query=route', 200), + ('GET', '/_s/q/2/s/%2Fmain/t/route/f/b/fc/type,tag', 200), + ('GET', f'{read_repo}/info/refs?service=git-upload-pack', 200), + ) + for method, path, status in reads: + response = await http.request(method, path) + assert response.status_code == status, (method, path, response.text) + facets = await http.get('/_search?scope=%2Fmain&terms=route&facets=type,tag') + assert facets.status_code == 200, facets.text + assert set(facets.json()['facets']) == {'type', 'tag'} + for path in ( + '/_search?scope=%2Fmain&terms=route&facets=type,type', + '/_s/q/2/s/%2Fmain/t/route/fc/invalid', + ): + response = await http.get(path) + assert response.status_code == 400, (path, response.text) + batch = {'operation': 'download', 'objects': [{'oid': '0' * 64, 'size': 4}]} + lfs = await http.post( + read_repo + '/info/lfs/objects/batch', + json=batch, + headers={'Content-Type': 'application/vnd.git-lfs+json'}, + ) + assert lfs.status_code == 200, lfs.text + assert lfs.json()['objects'][0]['error']['code'] == 404 + upload = await http.post( + read_repo + '/info/lfs/objects/batch', + json={**batch, 'operation': 'upload'}, + headers={'Content-Type': 'application/vnd.git-lfs+json'}, + ) + assert upload.status_code == 405 + rejected = ( + ('POST', '/main'), + ('PUT', '/main/effect-probe'), + ('POST', '/_read/graphql'), + ('POST', '/@route-effect-owner/code.git/git-receive-pack'), + ('GET', '/!content.post_create/run/j/' + encoded), + ('POST', '/!content.post_create/run/j/' + encoded), + ('GET', '/run/j/' + encoded), + ('POST', '/run/j/' + encoded), + ('GET', '/~content.post_create/run/j/' + encoded), + ) + for method, path in rejected: + response = await http.request(method, path, content=canonical(write)) + assert response.status_code in {400, 404, 405}, (method, path, response.text) + gql = {'query': 'mutation($p: JSON!) { call(packet: $p) }', 'variables': {'p': wire(write)}} + for path in ('/_read/graphql', '/_r/graphql'): + response = await http.post(path, json=gql) + assert ( + response.status_code == 400 + and response.json()['error']['code'] == 'graphql_effect_mismatch' + ) + signed_read = request_for( + 'discovery.get', + {'id': rid}, + app.settings.service_url, + signer=key, + subject=subject, + expires_at=NOW + timedelta(seconds=90), + ) + invalid = await http.get( + f'/_read/{rid}/json', headers={'X-Msg-Request': b64(canonical(write))} ) - for method,path,status in reads: - response=await http.request(method,path) - assert response.status_code==status,(method,path,response.text) - facets=await http.get('/_search?scope=%2Fmain&terms=route&facets=type,tag') - assert facets.status_code==200,facets.text - assert set(facets.json()['facets'])=={'type','tag'} - for path in ('/_search?scope=%2Fmain&terms=route&facets=type,type', - '/_s/q/2/s/%2Fmain/t/route/fc/invalid'): - response=await http.get(path) - assert response.status_code==400,(path,response.text) - batch={'operation':'download','objects':[{'oid':'0'*64,'size':4}]} - lfs=await http.post(read_repo+'/info/lfs/objects/batch',json=batch, - headers={'Content-Type':'application/vnd.git-lfs+json'}) - assert lfs.status_code==200,lfs.text - assert lfs.json()['objects'][0]['error']['code']==404 - upload=await http.post(read_repo+'/info/lfs/objects/batch', - json={**batch,'operation':'upload'}, - headers={'Content-Type':'application/vnd.git-lfs+json'}) - assert upload.status_code==405 - rejected=( - ('POST','/main'),('PUT','/main/effect-probe'), - ('POST','/_read/graphql'),('POST','/@route-effect-owner/code.git/git-receive-pack'), - ('GET','/!content.post_create/run/j/'+encoded), - ('POST','/!content.post_create/run/j/'+encoded), - ('GET','/run/j/'+encoded),('POST','/run/j/'+encoded), - ('GET','/~content.post_create/run/j/'+encoded), + assert invalid.status_code == 400 + good = await http.get( + f'/_read/{rid}/json', headers={'X-Msg-Request': b64(canonical(signed_read))} ) - for method,path in rejected: - response=await http.request(method,path,content=canonical(write)) - assert response.status_code in {400,404,405},(method,path,response.text) - gql={'query':'mutation($p: JSON!) { call(packet: $p) }', - 'variables':{'p':wire(write)}} - for path in ('/_read/graphql','/_r/graphql'): - response=await http.post(path,json=gql) - assert response.status_code==400 and response.json()['error']['code']==\ - 'graphql_effect_mismatch' - signed_read=request_for('discovery.get',{'id':rid},app.settings.service_url, - signer=key,subject=subject, - expires_at=NOW+timedelta(seconds=90)) - invalid=await http.get(f'/_read/{rid}/json', - headers={'X-Msg-Request':b64(canonical(write))}) - assert invalid.status_code==400 - good=await http.get(f'/_read/{rid}/json', - headers={'X-Msg-Request':b64(canonical(signed_read))}) - assert good.status_code==200 - assert await database_snapshot(app)==before - assert not app.settings.server.repositories_dir.joinpath(repo_id+'.git','lfs').exists() + assert good.status_code == 200 + assert await database_snapshot(app) == before + assert not app.settings.server.repositories_dir.joinpath(repo_id + '.git', 'lfs').exists() @pytest.mark.asyncio async def test_effect_classification_matches_native_git_and_lfs_boundaries(installed): - app,_=installed - cases={ - ('/main','POST'):RouteEffect.PURE_READ, - ('/@owner/code.git/git-upload-pack','POST'):RouteEffect.PURE_READ, - ('/@owner/code.git/info/lfs/objects/batch','POST'):RouteEffect.PURE_READ, - ('/-/git/repo/info/lfs/objects/batch','POST'):RouteEffect.PURE_READ, - ('/-/git/repo/info/lfs/objects/'+'a'*64+'/10','PUT'):RouteEffect.EXTERNAL_EFFECT, - ('/-/git/repo/git-receive-pack','POST'):RouteEffect.EXTERNAL_EFFECT, - ('/-/git/repo/info/refs','GET'):RouteEffect.PURE_READ, - ('/-/graphql','POST'):RouteEffect.EXTERNAL_EFFECT, - ('/-/mcp','POST'):RouteEffect.EXTERNAL_EFFECT, - ('/-/transfer','POST'):RouteEffect.BUSINESS_WRITE, - ('/-/p/content.post_create','POST'):RouteEffect.BUSINESS_WRITE, + app, _ = installed + cases = { + ('/main', 'POST'): RouteEffect.PURE_READ, + ('/@owner/code.git/git-upload-pack', 'POST'): RouteEffect.PURE_READ, + ('/@owner/code.git/info/lfs/objects/batch', 'POST'): RouteEffect.PURE_READ, + ('/-/git/repo/info/lfs/objects/batch', 'POST'): RouteEffect.PURE_READ, + ('/-/git/repo/info/lfs/objects/' + 'a' * 64 + '/10', 'PUT'): RouteEffect.EXTERNAL_EFFECT, + ('/-/git/repo/git-receive-pack', 'POST'): RouteEffect.EXTERNAL_EFFECT, + ('/-/git/repo/info/refs', 'GET'): RouteEffect.PURE_READ, + ('/-/graphql', 'POST'): RouteEffect.EXTERNAL_EFFECT, + ('/-/mcp', 'POST'): RouteEffect.EXTERNAL_EFFECT, + ('/-/transfer', 'POST'): RouteEffect.BUSINESS_WRITE, + ('/-/p/content.post_create', 'POST'): RouteEffect.BUSINESS_WRITE, } - for (path,method),effect in cases.items(): - assert classify_route(path,method,app.registry).effect is effect,(method,path) + for (path, method), effect in cases.items(): + assert classify_route(path, method, app.registry).effect is effect, (method, path) diff --git a/tests/test_saved_queries.py b/tests/test_saved_queries.py index 5ac2b320..33e88c13 100644 --- a/tests/test_saved_queries.py +++ b/tests/test_saved_queries.py @@ -1,4 +1,5 @@ """Explicit saved descriptors outlive read tickets, never their live authority.""" + from datetime import timedelta import pytest @@ -10,25 +11,55 @@ async def make_query_ref(app, key, subject, arguments, kind='read'): payload = canonical({'version': 1, 'kind': kind, 'arguments': arguments}) - opened = await call(app, 'transfer.open', {'direction': 'upload', 'size': len(payload), - 'digest': digest(payload), 'media_type': 'application/vnd.msg.read-query+json'}, - key=key, subject=subject) + opened = await call( + app, + 'transfer.open', + { + 'direction': 'upload', + 'size': len(payload), + 'digest': digest(payload), + 'media_type': 'application/vnd.msg.read-query+json', + }, + key=key, + subject=subject, + ) assert opened.status == 'ok', opened.error transfer = opened.data['transfer_id'] - part = await call(app, 'transfer.part_put', {'transfer_id': transfer, 'offset': 0, - 'data': b64(payload), 'digest': digest(payload)}, key=key, subject=subject) + part = await call( + app, + 'transfer.part_put', + {'transfer_id': transfer, 'offset': 0, 'data': b64(payload), 'digest': digest(payload)}, + key=key, + subject=subject, + ) assert part.status == 'ok', part.error - sealed = await call(app, 'transfer.seal', {'transfer_id': transfer, 'final_size': len(payload), - 'final_digest': digest(payload)}, key=key, subject=subject) + sealed = await call( + app, + 'transfer.seal', + {'transfer_id': transfer, 'final_size': len(payload), 'final_digest': digest(payload)}, + key=key, + subject=subject, + ) assert sealed.status == 'ok', sealed.error - ticket = await call(app, 'transfer.query_seal', {'transfer_id': transfer}, key=key, subject=subject) + ticket = await call( + app, 'transfer.query_seal', {'transfer_id': transfer}, key=key, subject=subject + ) assert ticket.status == 'ok', ticket.error return ticket.data['query_ref'] async def current_call(app, op, args, key, subject, **kwargs): - return await app.executor.execute(request_for(op, args, app.settings.service_url, - signer=key, subject=subject, expires_at=app.clock() + timedelta(seconds=120), **kwargs)) + return await app.executor.execute( + request_for( + op, + args, + app.settings.service_url, + signer=key, + subject=subject, + expires_at=app.clock() + timedelta(seconds=120), + **kwargs, + ) + ) @pytest.mark.asyncio @@ -37,11 +68,23 @@ async def test_save_pins_descriptor_replays_and_outlives_only_ticket(installed): key, subject, _ = await register(app, 'saved-query-owner') args = {'parent': '/main', 'type': 'post', 'tag': 'work'} token = await make_query_ref(app, key, subject, args) - saved = await call(app, 'query.save', {'query_ref': token}, key=key, subject=subject, - rid='save-descriptor-once') + saved = await call( + app, + 'query.save', + {'query_ref': token}, + key=key, + subject=subject, + rid='save-descriptor-once', + ) assert saved.status == 'ok', saved.error - replay = await call(app, 'query.save', {'query_ref': token}, key=key, subject=subject, - rid='save-descriptor-once') + replay = await call( + app, + 'query.save', + {'query_ref': token}, + key=key, + subject=subject, + rid='save-descriptor-once', + ) assert replay.replayed and replay.resources == saved.resources ref = wire(saved.resources[0]) got = await call(app, 'query.saved_get', {'ref': ref}, key=key, subject=subject) @@ -50,7 +93,9 @@ async def test_save_pins_descriptor_replays_and_outlives_only_ticket(installed): assert got.data['operation'] == 'discovery.read_query' assert got.data['digest'] == saved.data['descriptor_digest'] assert 'principal' not in got.data - assert (await call(app, 'discovery.get', {'id': ref['id']}, key=key, subject=subject)).status == 'error' + assert ( + await call(app, 'discovery.get', {'id': ref['id']}, key=key, subject=subject) + ).status == 'error' app.executor.clock = lambda: NOW + timedelta(minutes=16) app.clock = app.executor.clock app.executor.authenticator.clock = app.clock @@ -60,10 +105,13 @@ async def test_save_pins_descriptor_replays_and_outlives_only_ticket(installed): assert durable.status == 'ok', durable.error async with app.metadata.transaction(write=False) as tx: generation = (await tx.resource(ref['id'])).generation - archived = await current_call(app, 'query.saved_archive', {'ref': ref}, key, subject, - expected=((ref['id'], generation),)) + archived = await current_call( + app, 'query.saved_archive', {'ref': ref}, key, subject, expected=((ref['id'], generation),) + ) assert archived.status == 'ok', archived.error - assert (await current_call(app, 'query.saved_get', {'ref': ref}, key, subject)).error.code == 'saved_query_not_found' + assert ( + await current_call(app, 'query.saved_get', {'ref': ref}, key, subject) + ).error.code == 'saved_query_not_found' @pytest.mark.asyncio @@ -71,8 +119,13 @@ async def test_saved_query_rechecks_scope_and_never_shares_captured_authority(in app, _ = installed key, subject, _ = await register(app, 'saved-scope-owner') other_key, other, _ = await register(app, 'saved-scope-reader') - topic = await call(app, 'content.topic_create', {'parent': '/main', 'name': 'saved-scope'}, - key=key, subject=subject) + topic = await call( + app, + 'content.topic_create', + {'parent': '/main', 'name': 'saved-scope'}, + key=key, + subject=subject, + ) rid = topic.resources[0].id token = await make_query_ref(app, other_key, other, {'parent': rid, 'type': 'post'}) saved = await call(app, 'query.save', {'query_ref': token}, key=other_key, subject=other) @@ -80,12 +133,25 @@ async def test_saved_query_rechecks_scope_and_never_shares_captured_authority(in ref = wire(saved.resources[0]) wrong = await call(app, 'query.save', {'query_ref': token}, key=key, subject=subject) assert wrong.error.code == 'query_ref_principal_mismatch' - assert (await call(app, 'query.saved_get', {'ref': ref}, key=key, subject=subject)).status == 'error' - grant = await call(app, 'sharing.grant', {'resource': ref['id'], 'grantee': subject, - 'expires_at': wire(NOW + timedelta(hours=1))}, key=other_key, subject=other) + assert ( + await call(app, 'query.saved_get', {'ref': ref}, key=key, subject=subject) + ).status == 'error' + grant = await call( + app, + 'sharing.grant', + {'resource': ref['id'], 'grantee': subject, 'expires_at': wire(NOW + timedelta(hours=1))}, + key=other_key, + subject=other, + ) assert grant.status == 'error' - hidden = await call(app, 'content.chmod', {'id': rid, 'mode': '0700'}, key=key, subject=subject, - expected=((rid, topic.data['generation']),)) + hidden = await call( + app, + 'content.chmod', + {'id': rid, 'mode': '0700'}, + key=key, + subject=subject, + expected=((rid, topic.data['generation']),), + ) assert hidden.status == 'ok', hidden.error denied = await call(app, 'query.saved_get', {'ref': ref}, key=other_key, subject=other) assert denied.status == 'error' @@ -100,32 +166,43 @@ async def test_saved_query_requires_pinned_revision_and_unchanged_digest(install assert saved.status == 'ok', saved.error ref = wire(saved.resources[0]) for bad_ref in ({'id': ref['id']}, {**ref, 'revision': 'nonexistent_revision'}): - assert (await call(app, 'query.saved_get', {'ref': bad_ref}, key=key, subject=subject)).status == 'error' + assert ( + await call(app, 'query.saved_get', {'ref': bad_ref}, key=key, subject=subject) + ).status == 'error' tampered = token[:-1] + ('a' if token[-1] != 'a' else 'b') - assert (await call(app, 'query.save', {'query_ref': tampered}, key=key, subject=subject)).status == 'error' + assert ( + await call(app, 'query.save', {'query_ref': tampered}, key=key, subject=subject) + ).status == 'error' @pytest.mark.asyncio async def test_saved_authority_is_current_and_internal_body_cannot_be_projected(installed): from dataclasses import replace + app, _ = installed key, subject, _ = await register(app, 'saved-authority-owner') token = await make_query_ref(app, key, subject, {'parent': '/main'}) - forbidden_projection = await current_call(app, 'query.save', {'query_ref': token}, key, subject, - return_fields=('content',)) + forbidden_projection = await current_call( + app, 'query.save', {'query_ref': token}, key, subject, return_fields=('content',) + ) assert forbidden_projection.error.code == 'projection_unavailable' saved = await call(app, 'query.save', {'query_ref': token}, key=key, subject=subject) assert saved.status == 'ok', saved.error ref = wire(saved.resources[0]) async with app.metadata.transaction(write=True) as tx: credential = await tx.credential(key.key_id) - ceiling = tuple(replace(grant, operations=grant.operations - {'query.save@1'}) - for grant in credential.ceiling) - await tx.save_credential(replace(credential, ceiling=ceiling), (await tx.subject(subject)).auth_version) + ceiling = tuple( + replace(grant, operations=grant.operations - {'query.save@1'}) + for grant in credential.ceiling + ) + await tx.save_credential( + replace(credential, ceiling=ceiling), (await tx.subject(subject)).auth_version + ) generation = (await tx.resource(ref['id'])).generation denied = await call(app, 'query.saved_get', {'ref': ref}, key=key, subject=subject) assert denied.error.code == 'credential_ceiling' # Revocation remains possible without the removed creation authority. - archived = await current_call(app, 'query.saved_archive', {'ref': ref}, key, subject, - expected=((ref['id'], generation),)) + archived = await current_call( + app, 'query.saved_archive', {'ref': ref}, key, subject, expected=((ref['id'], generation),) + ) assert archived.status == 'ok', archived.error diff --git a/tests/test_saved_query_typed_scopes.py b/tests/test_saved_query_typed_scopes.py index 7ddf8bd5..8c800e7d 100644 --- a/tests/test_saved_query_typed_scopes.py +++ b/tests/test_saved_query_typed_scopes.py @@ -1,4 +1,5 @@ """Saved search descriptors preserve the typed scope and its live authority.""" + import pytest from test_saved_queries import make_query_ref from test_service import call, register @@ -24,15 +25,21 @@ async def test_signed_typed_search_save_read_and_current_revocation(installed, k supplied = {'org': await tx.path(target)} normalized = {'org': target} else: - topic = await call(app, 'content.topic_create', {'parent': '/main', 'name': 'typed-query-topic'}, - key=key, subject=owner) + topic = await call( + app, + 'content.topic_create', + {'parent': '/main', 'name': 'typed-query-topic'}, + key=key, + subject=owner, + ) assert topic.status == 'ok', topic.error target = topic.resources[0].id supplied = {'resource_refs': [{'id': '/main/typed-query-topic'}, {'id': '/main'}]} normalized = {'resource_refs': [{'id': item} for item in sorted([target, 't_main'])]} args = {'scope': supplied, 'terms': 'typed-query-test'} - direct = await call(app, 'discovery.lexical_search', args, key=reader_key, subject=reader, - contract_version=5) + direct = await call( + app, 'discovery.lexical_search', args, key=reader_key, subject=reader, contract_version=5 + ) assert direct.status == 'ok', direct.error token = await make_query_ref(app, reader_key, reader, args, kind='search') saved = await call(app, 'query.save', {'query_ref': token}, key=reader_key, subject=reader) @@ -40,8 +47,11 @@ async def test_signed_typed_search_save_read_and_current_revocation(installed, k ref = wire(saved.resources[0]) got = await call(app, 'query.saved_get', {'ref': ref}, key=reader_key, subject=reader) assert got.status == 'ok', got.error - expected = {'operation': 'discovery.lexical_search', 'contract_version': 5, - 'arguments': {**args, 'scope': normalized}} + expected = { + 'operation': 'discovery.lexical_search', + 'contract_version': 5, + 'arguments': {**args, 'scope': normalized}, + } assert wire(got.data) == {**expected, 'digest': digest(expected)} assert reader_key.key_id not in str(got.data) assert 'principal' not in got.data @@ -49,14 +59,25 @@ async def test_signed_typed_search_save_read_and_current_revocation(installed, k raw = await call(app, 'discovery.get', {'id': ref['id']}, key=reader_key, subject=reader) assert raw.status == 'error' # Saving a legal search does not silently broaden the ReadQuery-only watch. - watch = await call(app, 'communication.watch_create', + watch = await call( + app, + 'communication.watch_create', {'query_ref': ref, 'event_types': ['content.post_create'], 'delivery': 'inbox'}, - key=reader_key, subject=reader, contract_version=2) + key=reader_key, + subject=reader, + contract_version=2, + ) assert watch.error.code == 'watch_query_unsupported' async with app.metadata.transaction(write=False) as tx: generation = (await tx.resource(target)).generation - revoked = await call(app, 'content.chmod', {'id': target, 'mode': '0700'}, - key=key, subject=owner, expected=((target, generation),)) + revoked = await call( + app, + 'content.chmod', + {'id': target, 'mode': '0700'}, + key=key, + subject=owner, + expected=((target, generation),), + ) assert revoked.status == 'ok', revoked.error denied = await call(app, 'query.saved_get', {'ref': ref}, key=reader_key, subject=reader) assert denied.status == 'error' diff --git a/tests/test_saved_query_watch.py b/tests/test_saved_query_watch.py index 62147cbe..87bff802 100644 --- a/tests/test_saved_query_watch.py +++ b/tests/test_saved_query_watch.py @@ -1,32 +1,62 @@ """Pinned saved-query watches match only current authorized Event references.""" + from datetime import timedelta import pytest +from test_service import NOW, call, register from msg.core.codec import b64, canonical, digest, wire -from test_service import NOW, call, register async def save_query(app, key, subject, arguments): payload = canonical({'version': 1, 'kind': 'read', 'arguments': arguments}) - opened = await call(app, 'transfer.open', {'direction': 'upload', 'size': len(payload), - 'digest': digest(payload), 'media_type': 'application/vnd.msg.read-query+json'}, key=key, subject=subject) + opened = await call( + app, + 'transfer.open', + { + 'direction': 'upload', + 'size': len(payload), + 'digest': digest(payload), + 'media_type': 'application/vnd.msg.read-query+json', + }, + key=key, + subject=subject, + ) assert opened.status == 'ok', wire(opened) tid = opened.data['transfer_id'] - part = await call(app, 'transfer.part_put', {'transfer_id': tid, 'offset': 0, 'data': b64(payload), 'digest': digest(payload)}, key=key, subject=subject) + part = await call( + app, + 'transfer.part_put', + {'transfer_id': tid, 'offset': 0, 'data': b64(payload), 'digest': digest(payload)}, + key=key, + subject=subject, + ) assert part.status == 'ok', wire(part) - sealed = await call(app, 'transfer.seal', {'transfer_id': tid, 'final_size': len(payload), 'final_digest': digest(payload)}, key=key, subject=subject) + sealed = await call( + app, + 'transfer.seal', + {'transfer_id': tid, 'final_size': len(payload), 'final_digest': digest(payload)}, + key=key, + subject=subject, + ) assert sealed.status == 'ok', wire(sealed) token = await call(app, 'transfer.query_seal', {'transfer_id': tid}, key=key, subject=subject) assert token.status == 'ok', wire(token) - saved = await call(app, 'query.save', {'query_ref': token.data['query_ref']}, key=key, subject=subject) + saved = await call( + app, 'query.save', {'query_ref': token.data['query_ref']}, key=key, subject=subject + ) assert saved.status == 'ok', wire(saved) return saved async def messages(app, recipient): async with app.metadata.transaction(write=False) as tx: - return [row[0] for row in tx.rows('SELECT resource FROM messages WHERE recipient=? ORDER BY resource', (recipient,))] + return [ + row[0] + for row in tx.rows( + 'SELECT resource FROM messages WHERE recipient=? ORDER BY resource', (recipient,) + ) + ] @pytest.mark.asyncio @@ -36,27 +66,68 @@ async def test_saved_watch_matches_shared_read_predicates_and_source_revoke(inst rk, reader, _ = await register(app, 'saved-watch-reader') saved = await save_query(app, rk, reader, {'parent': '/main', 'type': 'post', 'author': author}) ref = wire(saved.resources[0]) - args = {'query_ref': ref, 'event_types': ['content.post_create', 'content.post_edit'], 'delivery': 'inbox'} - created = await call(app, 'communication.watch_create', args, key=rk, subject=reader, contract_version=2) + args = { + 'query_ref': ref, + 'event_types': ['content.post_create', 'content.post_edit'], + 'delivery': 'inbox', + } + created = await call( + app, 'communication.watch_create', args, key=rk, subject=reader, contract_version=2 + ) assert created.status == 'ok', wire(created) - chosen = await call(app, 'content.post_create', {'parent': '/main', 'body': 'match'}, key=ak, subject=author) - skipped = await call(app, 'content.post_create', {'parent': '/main', 'body': 'different author'}, key=rk, subject=reader) + chosen = await call( + app, 'content.post_create', {'parent': '/main', 'body': 'match'}, key=ak, subject=author + ) + skipped = await call( + app, + 'content.post_create', + {'parent': '/main', 'body': 'different author'}, + key=rk, + subject=reader, + ) assert chosen.status == skipped.status == 'ok' assert await messages(app, reader) == [chosen.resources[0].id] - hidden = await call(app, 'content.chmod', {'id': chosen.resources[0].id, 'mode': '0600'}, - key=ak, subject=author, expected=((chosen.resources[0].id, chosen.data['generation']),)) + hidden = await call( + app, + 'content.chmod', + {'id': chosen.resources[0].id, 'mode': '0600'}, + key=ak, + subject=author, + expected=((chosen.resources[0].id, chosen.data['generation']),), + ) assert hidden.status == 'ok', wire(hidden) - edited = await call(app, 'content.post_edit', {'id': chosen.resources[0].id, - 'expected_revision': chosen.resources[0].revision, 'body': 'private change'}, - key=ak, subject=author, expected=((chosen.resources[0].id, hidden.data['generation']),)) + edited = await call( + app, + 'content.post_edit', + { + 'id': chosen.resources[0].id, + 'expected_revision': chosen.resources[0].revision, + 'body': 'private change', + }, + key=ak, + subject=author, + expected=((chosen.resources[0].id, hidden.data['generation']),), + ) assert edited.status == 'ok', wire(edited) assert await messages(app, reader) == [chosen.resources[0].id] async with app.metadata.transaction(write=False) as tx: source = await tx.resource(ref['id']) - archived = await call(app, 'query.saved_archive', {'ref': ref}, key=rk, subject=reader, - expected=((source.id, source.generation),)) + archived = await call( + app, + 'query.saved_archive', + {'ref': ref}, + key=rk, + subject=reader, + expected=((source.id, source.generation),), + ) assert archived.status == 'ok', wire(archived) - after = await call(app, 'content.post_create', {'parent': '/main', 'body': 'source revoked'}, key=ak, subject=author) + after = await call( + app, + 'content.post_create', + {'parent': '/main', 'body': 'source revoked'}, + key=ak, + subject=author, + ) assert after.status == 'ok', wire(after) assert await messages(app, reader) == [chosen.resources[0].id] @@ -66,43 +137,88 @@ async def test_saved_watch_rejects_unsupported_predicates_and_missing_revision(i app, _ = installed key, user, _ = await register(app, 'saved-watch-reject') saved = await save_query(app, key, user, {'parent': '/main', 'limit': 1}) - args = {'query_ref': wire(saved.resources[0]), 'event_types': ['content.post_create'], 'delivery': 'inbox'} - rejected = await call(app, 'communication.watch_create', args, key=key, subject=user, contract_version=2) + args = { + 'query_ref': wire(saved.resources[0]), + 'event_types': ['content.post_create'], + 'delivery': 'inbox', + } + rejected = await call( + app, 'communication.watch_create', args, key=key, subject=user, contract_version=2 + ) assert rejected.error.code == 'watch_query_unsupported', wire(rejected) args['query_ref'] = {'id': saved.resources[0].id} - rejected = await call(app, 'communication.watch_create', args, key=key, subject=user, contract_version=2) + rejected = await call( + app, 'communication.watch_create', args, key=key, subject=user, contract_version=2 + ) assert rejected.error.code == 'watch_query_revision_required', wire(rejected) @pytest.mark.asyncio -async def test_saved_watch_cli_signs_pinned_v2_and_expiry_stops_delivery(installed, tmp_path, monkeypatch, capsys): +async def test_saved_watch_cli_signs_pinned_v2_and_expiry_stops_delivery( + installed, tmp_path, monkeypatch, capsys +): import httpx + from msg import cli from msg.client import ClientState, MsgClient from msg.core.codec import loads from msg.transports.client import HTTPTransport from msg.transports.http import create_app + app, _ = installed ak, author, _ = await register(app, 'saved-watch-cli-author') - async with httpx.AsyncClient(transport=httpx.ASGITransport(app=create_app(app)), base_url=app.settings.service_url) as http: + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=create_app(app)), base_url=app.settings.service_url + ) as http: monkeypatch.setitem(cli.TRANSPORTS, 'http', lambda server: HTTPTransport(server, http=http)) - directory = tmp_path/'client' - client = MsgClient(ClientState(directory, server=app.settings.service_url), HTTPTransport(app.settings.service_url, http=http), clock=lambda: NOW) + directory = tmp_path / 'client' + client = MsgClient( + ClientState(directory, server=app.settings.service_url), + HTTPTransport(app.settings.service_url, http=http), + clock=lambda: NOW, + ) assert (await client.register('saved-watch-cli')).status == 'ok' - saved = await save_query(app, client.state.signer, client.state.subject, {'parent': '/main', 'type': 'post'}) + saved = await save_query( + app, client.state.signer, client.state.subject, {'parent': '/main', 'type': 'post'} + ) ref = saved.resources[0] - monkeypatch.setattr(cli, 'MsgClient', lambda state, transport: MsgClient(state, transport, clock=lambda: NOW)) - args = cli.parser().parse_args(['--config-dir', str(directory), '--server', app.settings.service_url, - 'watch', 'create', '--query-ref', ref.id, '--query-revision', ref.revision, - '--event', 'content.post_create', '--expires-at', wire(NOW+timedelta(seconds=1))]) + monkeypatch.setattr( + cli, + 'MsgClient', + lambda state, transport: MsgClient(state, transport, clock=lambda: NOW), + ) + args = cli.parser().parse_args([ + '--config-dir', + str(directory), + '--server', + app.settings.service_url, + 'watch', + 'create', + '--query-ref', + ref.id, + '--query-revision', + ref.revision, + '--event', + 'content.post_create', + '--expires-at', + wire(NOW + timedelta(seconds=1)), + ]) status = await cli.run(args) output = loads(capsys.readouterr().out.encode().strip()) assert status == 0 and output['status'] == 'ok', output - before = await call(app, 'content.post_create', {'parent': '/main', 'body': 'live'}, key=ak, subject=author) + before = await call( + app, 'content.post_create', {'parent': '/main', 'body': 'live'}, key=ak, subject=author + ) assert before.status == 'ok' assert await messages(app, client.state.subject) == [before.resources[0].id] - app.clock = lambda: NOW+timedelta(seconds=2) - after = await call(app, 'content.post_create', {'parent': '/main', 'body': 'expired'}, key=ak, subject=author) + app.clock = lambda: NOW + timedelta(seconds=2) + after = await call( + app, + 'content.post_create', + {'parent': '/main', 'body': 'expired'}, + key=ak, + subject=author, + ) assert after.status == 'ok', wire(after) assert await messages(app, client.state.subject) == [before.resources[0].id] @@ -113,15 +229,32 @@ async def test_saved_watch_delivers_new_tag_match_without_query_execution(instal ak, author, _ = await register(app, 'saved-watch-tag-author') rk, reader, _ = await register(app, 'saved-watch-tag-reader') saved = await save_query(app, rk, reader, {'parent': '/main', 'tag': 'research'}) - watched = await call(app, 'communication.watch_create', { - 'query_ref': wire(saved.resources[0]), 'event_types': ['content.post_create', 'content.tags_set'], - 'delivery': 'inbox'}, key=rk, subject=reader, contract_version=2) + watched = await call( + app, + 'communication.watch_create', + { + 'query_ref': wire(saved.resources[0]), + 'event_types': ['content.post_create', 'content.tags_set'], + 'delivery': 'inbox', + }, + key=rk, + subject=reader, + contract_version=2, + ) assert watched.status == 'ok', wire(watched) - post = await call(app, 'content.post_create', {'parent': '/main', 'body': 'untagged'}, key=ak, subject=author) + post = await call( + app, 'content.post_create', {'parent': '/main', 'body': 'untagged'}, key=ak, subject=author + ) assert post.status == 'ok' assert await messages(app, reader) == [] - tagged = await call(app, 'content.tags_set', {'id': post.resources[0].id, 'tags': ['research']}, - key=ak, subject=author, expected=((post.resources[0].id, post.data['generation']),)) + tagged = await call( + app, + 'content.tags_set', + {'id': post.resources[0].id, 'tags': ['research']}, + key=ak, + subject=author, + expected=((post.resources[0].id, post.data['generation']),), + ) assert tagged.status == 'ok', wire(tagged) assert await messages(app, reader) == [post.resources[0].id] @@ -129,6 +262,7 @@ async def test_saved_watch_delivers_new_tag_match_without_query_execution(instal @pytest.mark.asyncio async def test_saved_watch_stops_when_source_key_revoked_but_watch_key_remains_valid(installed): from msg.security.crypto import Ed25519Signer + app, _ = installed ak, author, _ = await register(app, 'saved-watch-dual-author') first, reader, _ = await register(app, 'saved-watch-dual-reader') @@ -138,17 +272,39 @@ async def test_saved_watch_stops_when_source_key_revoked_but_watch_key_remains_v async with app.metadata.transaction(write=False) as tx: ceiling = wire((await tx.credential(first.key_id)).ceiling) proof = second.sign(canonical({'subject_id': reader, 'public_key': public}), purpose='key-add') - added = await call(app, 'identity.key_add', {'public_key': public, 'possession_proof': wire(proof), - 'ceiling': ceiling}, key=first, subject=reader) + added = await call( + app, + 'identity.key_add', + {'public_key': public, 'possession_proof': wire(proof), 'ceiling': ceiling}, + key=first, + subject=reader, + ) assert added.status == 'ok', wire(added) - watched = await call(app, 'communication.watch_create', { - 'query_ref': wire(saved.resources[0]), 'event_types': ['content.post_create'], 'delivery': 'inbox'}, - key=second, subject=reader, contract_version=2) + watched = await call( + app, + 'communication.watch_create', + { + 'query_ref': wire(saved.resources[0]), + 'event_types': ['content.post_create'], + 'delivery': 'inbox', + }, + key=second, + subject=reader, + contract_version=2, + ) assert watched.status == 'ok', wire(watched) - revoked = await call(app, 'identity.key_revoke', {'key_id': first.key_id}, key=second, subject=reader) + revoked = await call( + app, 'identity.key_revoke', {'key_id': first.key_id}, key=second, subject=reader + ) assert revoked.status == 'ok', wire(revoked) alive = await call(app, 'communication.watch_list', {}, key=second, subject=reader) assert alive.status == 'ok', wire(alive) - posted = await call(app, 'content.post_create', {'parent': '/main', 'body': 'source key gone'}, key=ak, subject=author) + posted = await call( + app, + 'content.post_create', + {'parent': '/main', 'body': 'source key gone'}, + key=ak, + subject=author, + ) assert posted.status == 'ok', wire(posted) assert await messages(app, reader) == [] diff --git a/tests/test_search_source_relation_filters.py b/tests/test_search_source_relation_filters.py index c1755e97..548eb3a5 100644 --- a/tests/test_search_source_relation_filters.py +++ b/tests/test_search_source_relation_filters.py @@ -1,91 +1,152 @@ """SearchQuery v3 filters inspect only currently readable current revisions.""" + from datetime import timedelta import httpx import pytest +from test_service import NOW, call, register -from msg.core.codec import b64,canonical,digest,wire +from msg.core.codec import b64, canonical, digest, wire from msg.core.requests import request_for from msg.transports.http import create_app -from test_service import NOW,call,register @pytest.mark.asyncio @pytest.mark.parametrize('version', [3, 5]) async def test_v3_query_string_path_and_sealed_ref_share_filters(installed, version): - app,_=installed - key,user,_=await register(app,'search-v3-http') - root=await call(app,'content.post_create',{'parent':'/main','body':'v3needle root'}, - key=key,subject=user) - reply=await call(app,'discussion.reply',{'target':{'id':root.resources[0].id}, - 'body':'v3needle reply'},key=key,subject=user) - assert root.status==reply.status=='ok' - async with httpx.AsyncClient(transport=httpx.ASGITransport(app=create_app(app)), - base_url=app.settings.service_url) as http: - query=await http.get('/_search?scope=%2Fmain&terms=v3needle&relation_type=reply_to') - path=await http.get('/_s/q/3/s/%2Fmain/t/v3needle/rt/reply_to') - assert query.status_code==path.status_code==200,(query.text,path.text) - assert query.content==path.content - assert [row['id'] for row in query.json()['items']]==[reply.resources[0].id] - descriptor_args={'scope':'/main','terms':'v3needle','relation_type':'reply_to'} - if version==5: - descriptor_args['revision']=reply.resources[0].revision - descriptor_args['relation_to']=root.resources[0].id - descriptor_args['scope']={'resource_refs':[{'id':reply.resources[0].id}]} - descriptor=canonical({'version':1,'kind':'search','arguments':descriptor_args}) - opened=await call(app,'transfer.open',{'direction':'upload','size':len(descriptor), - 'digest':digest(descriptor),'media_type':'application/vnd.msg.read-query+json'}, - key=key,subject=user) - tid=opened.data['transfer_id'] - await call(app,'transfer.part_put',{'transfer_id':tid,'offset':0, - 'data':b64(descriptor),'digest':digest(descriptor)},key=key,subject=user) - await call(app,'transfer.seal',{'transfer_id':tid,'final_size':len(descriptor), - 'final_digest':digest(descriptor)},key=key,subject=user) - sealed=await call(app,'transfer.query_seal',{'transfer_id':tid},key=key,subject=user) - assert sealed.status=='ok',wire(sealed) - token=sealed.data['query_ref'] - proof=request_for('transfer.query_get',{'query_ref':token},app.settings.service_url, - signer=key,subject=user,expires_at=NOW+timedelta(seconds=60)) - async with httpx.AsyncClient(transport=httpx.ASGITransport(app=create_app(app)), - base_url=app.settings.service_url) as http: - resolved=await http.get('/_r/q/'+token+'/p/'+b64(canonical(proof))) - assert resolved.status_code==200,resolved.text - assert [row['id'] for row in resolved.json()['items']]==[reply.resources[0].id] + app, _ = installed + key, user, _ = await register(app, 'search-v3-http') + root = await call( + app, + 'content.post_create', + {'parent': '/main', 'body': 'v3needle root'}, + key=key, + subject=user, + ) + reply = await call( + app, + 'discussion.reply', + {'target': {'id': root.resources[0].id}, 'body': 'v3needle reply'}, + key=key, + subject=user, + ) + assert root.status == reply.status == 'ok' + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=create_app(app)), base_url=app.settings.service_url + ) as http: + query = await http.get('/_search?scope=%2Fmain&terms=v3needle&relation_type=reply_to') + path = await http.get('/_s/q/3/s/%2Fmain/t/v3needle/rt/reply_to') + assert query.status_code == path.status_code == 200, (query.text, path.text) + assert query.content == path.content + assert [row['id'] for row in query.json()['items']] == [reply.resources[0].id] + descriptor_args = {'scope': '/main', 'terms': 'v3needle', 'relation_type': 'reply_to'} + if version == 5: + descriptor_args['revision'] = reply.resources[0].revision + descriptor_args['relation_to'] = root.resources[0].id + descriptor_args['scope'] = {'resource_refs': [{'id': reply.resources[0].id}]} + descriptor = canonical({'version': 1, 'kind': 'search', 'arguments': descriptor_args}) + opened = await call( + app, + 'transfer.open', + { + 'direction': 'upload', + 'size': len(descriptor), + 'digest': digest(descriptor), + 'media_type': 'application/vnd.msg.read-query+json', + }, + key=key, + subject=user, + ) + tid = opened.data['transfer_id'] + await call( + app, + 'transfer.part_put', + {'transfer_id': tid, 'offset': 0, 'data': b64(descriptor), 'digest': digest(descriptor)}, + key=key, + subject=user, + ) + await call( + app, + 'transfer.seal', + {'transfer_id': tid, 'final_size': len(descriptor), 'final_digest': digest(descriptor)}, + key=key, + subject=user, + ) + sealed = await call(app, 'transfer.query_seal', {'transfer_id': tid}, key=key, subject=user) + assert sealed.status == 'ok', wire(sealed) + token = sealed.data['query_ref'] + proof = request_for( + 'transfer.query_get', + {'query_ref': token}, + app.settings.service_url, + signer=key, + subject=user, + expires_at=NOW + timedelta(seconds=60), + ) + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=create_app(app)), base_url=app.settings.service_url + ) as http: + resolved = await http.get('/_r/q/' + token + '/p/' + b64(canonical(proof))) + assert resolved.status_code == 200, resolved.text + assert [row['id'] for row in resolved.json()['items']] == [reply.resources[0].id] @pytest.mark.asyncio async def test_source_and_relation_filters_are_v3_and_use_current_revision(installed): app, _ = installed key, user, _ = await register(app, 'search-relation-owner') - root = await call(app, 'content.post_create', - {'parent': '/main', 'body': 'relationfilterneedle root'}, - key=key, subject=user) - reply = await call(app, 'discussion.reply', - {'target': {'id': root.resources[0].id}, - 'body': 'relationfilterneedle reply'}, key=key, subject=user) - quoted = await call(app, 'discussion.quote', - {'target': {'id': root.resources[0].id}, 'parent': '/main', - 'body': 'relationfilterneedle quote'}, key=key, subject=user) + root = await call( + app, + 'content.post_create', + {'parent': '/main', 'body': 'relationfilterneedle root'}, + key=key, + subject=user, + ) + reply = await call( + app, + 'discussion.reply', + {'target': {'id': root.resources[0].id}, 'body': 'relationfilterneedle reply'}, + key=key, + subject=user, + ) + quoted = await call( + app, + 'discussion.quote', + { + 'target': {'id': root.resources[0].id}, + 'parent': '/main', + 'body': 'relationfilterneedle quote', + }, + key=key, + subject=user, + ) assert root.status == reply.status == quoted.status == 'ok' base = {'scope': '/main', 'terms': 'relationfilterneedle', 'field': 'body'} - reply_only = await call(app, 'discovery.lexical_search', - {**base, 'relation_type': 'reply_to'}, contract_version=3) + reply_only = await call( + app, 'discovery.lexical_search', {**base, 'relation_type': 'reply_to'}, contract_version=3 + ) assert reply_only.status == 'ok', wire(reply_only) assert [row['id'] for row in reply_only.data['items']] == [reply.resources[0].id] - quote_only = await call(app, 'discovery.lexical_search', - {**base, 'relation_type': 'quote'}, contract_version=3) + quote_only = await call( + app, 'discovery.lexical_search', {**base, 'relation_type': 'quote'}, contract_version=3 + ) assert [row['id'] for row in quote_only.data['items']] == [quoted.resources[0].id] - no_release = await call(app, 'discovery.lexical_search', - {**base, 'source_kind': 'release'}, contract_version=3) + no_release = await call( + app, 'discovery.lexical_search', {**base, 'source_kind': 'release'}, contract_version=3 + ) assert no_release.status == 'ok' and list(no_release.data['items']) == [] - released = await call(app, 'discovery.lexical_search', - {'scope': '/_rules', 'terms': 'identity', 'field': 'body', - 'source_kind': 'release'}, contract_version=3) + released = await call( + app, + 'discovery.lexical_search', + {'scope': '/_rules', 'terms': 'identity', 'field': 'body', 'source_kind': 'release'}, + contract_version=3, + ) assert released.status == 'ok', wire(released) assert released.data['items'] assert all(row['path'].startswith('/_rules/') for row in released.data['items']) - old_contract = await call(app, 'discovery.lexical_search', - {**base, 'relation_type': 'reply_to'}, contract_version=2) + old_contract = await call( + app, 'discovery.lexical_search', {**base, 'relation_type': 'reply_to'}, contract_version=2 + ) assert old_contract.status == 'error' @@ -93,35 +154,56 @@ async def test_source_and_relation_filters_are_v3_and_use_current_revision(insta async def test_v3_old_cursor_rechecks_grants_before_items_facets_and_rank(installed): app, _ = installed key, user, _ = await register(app, 'search-revoke-owner') - root = await call(app, 'content.post_create', - {'parent': '/main', 'body': 'revoke-filter-root'}, - key=key, subject=user) + root = await call( + app, + 'content.post_create', + {'parent': '/main', 'body': 'revoke-filter-root'}, + key=key, + subject=user, + ) replies = [] for _ in range(3): - reply = await call(app, 'discussion.reply', - {'target': {'id': root.resources[0].id}, - 'body': 'revokerelationneedle'}, key=key, subject=user) + reply = await call( + app, + 'discussion.reply', + {'target': {'id': root.resources[0].id}, 'body': 'revokerelationneedle'}, + key=key, + subject=user, + ) assert reply.status == 'ok', wire(reply) replies.append(reply) - args = {'scope': '/main', 'terms': 'revokerelationneedle', 'field': 'body', - 'relation_type': 'reply_to', 'limit': 1, 'facets': ['type']} + args = { + 'scope': '/main', + 'terms': 'revokerelationneedle', + 'field': 'body', + 'relation_type': 'reply_to', + 'limit': 1, + 'facets': ['type'], + } first = await call(app, 'discovery.lexical_search', args, contract_version=3) assert first.status == 'ok', wire(first) assert wire(first)['data']['facets'] == {'type': [{'value': 'post', 'count': 3}]} assert first.data['cursor'] - downgraded = await call(app, 'discovery.lexical_search', - {'cursor': first.data['cursor']}, contract_version=2) + downgraded = await call( + app, 'discovery.lexical_search', {'cursor': first.data['cursor']}, contract_version=2 + ) assert downgraded.status == 'error' and downgraded.error.code == 'cursor_query_mismatch' visible_first = first.data['items'][0]['id'] for reply in replies: rid = reply.resources[0].id if rid != visible_first: - locked = await call(app, 'content.chmod', {'id': rid, 'mode': '0600'}, - key=key, subject=user, - expected=((rid, reply.data['generation']),)) + locked = await call( + app, + 'content.chmod', + {'id': rid, 'mode': '0600'}, + key=key, + subject=user, + expected=((rid, reply.data['generation']),), + ) assert locked.status == 'ok', wire(locked) - continued = await call(app, 'discovery.lexical_search', - {'cursor': first.data['cursor']}, contract_version=3) + continued = await call( + app, 'discovery.lexical_search', {'cursor': first.data['cursor']}, contract_version=3 + ) assert continued.status == 'ok', wire(continued) assert list(continued.data['items']) == [] assert wire(continued)['data']['facets'] == {'type': [{'value': 'post', 'count': 1}]} @@ -133,8 +215,13 @@ async def test_v3_old_cursor_rechecks_grants_before_items_facets_and_rank(instal async def test_v5_revision_and_source_version_filters_use_current_readable_revision(installed): app, _ = installed key, subject, _ = await register(app, 'search-v5-owner') - created = await call(app, 'content.post_create', {'parent': '/main', 'body': 'v5revisionneedle'}, - key=key, subject=subject) + created = await call( + app, + 'content.post_create', + {'parent': '/main', 'body': 'v5revisionneedle'}, + key=key, + subject=subject, + ) assert created.status == 'ok', wire(created) rid, revision = created.resources[0].id, created.resources[0].revision args = {'scope': '/main', 'terms': 'v5revisionneedle', 'revision': revision} @@ -143,48 +230,86 @@ async def test_v5_revision_and_source_version_filters_use_current_readable_revis assert [item['id'] for item in found.data['items']] == [rid] old = await call(app, 'discovery.lexical_search', args, contract_version=4) assert old.status == 'error' - async with httpx.AsyncClient(transport=httpx.ASGITransport(app=create_app(app)), - base_url=app.settings.service_url) as http: + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=create_app(app)), base_url=app.settings.service_url + ) as http: query = await http.get('/_search', params=args) path = await http.get('/_s/q/5/s/%2Fmain/t/v5revisionneedle/rv/' + revision) assert query.status_code == path.status_code == 200, (query.text, path.text) assert query.content == path.content - edited = await call(app, 'content.post_edit', {'id': rid, 'body': 'v5revisionneedle updated', - 'expected_revision': revision}, key=key, subject=subject, - expected=((rid, created.data['generation']),)) + edited = await call( + app, + 'content.post_edit', + {'id': rid, 'body': 'v5revisionneedle updated', 'expected_revision': revision}, + key=key, + subject=subject, + expected=((rid, created.data['generation']),), + ) assert edited.status == 'ok', wire(edited) gone = await call(app, 'discovery.lexical_search', args, contract_version=5) assert gone.status == 'ok' and not gone.data['items'] - release = await call(app, 'discovery.lexical_search', - {'scope': '/_rules', 'terms': 'identity', 'source_kind': 'release'}, - contract_version=3) + release = await call( + app, + 'discovery.lexical_search', + {'scope': '/_rules', 'terms': 'identity', 'source_kind': 'release'}, + contract_version=3, + ) ref = release.data['items'][0]['id'] from msg.core.models import ResourceRef + async with app.metadata.transaction(write=False) as tx: source = await tx.revision(ResourceRef(id=ref)) - filtered = await call(app, 'discovery.lexical_search', { - 'scope': '/_rules', 'terms': 'identity', 'revision': source.id, - 'source_kind': 'release', 'source_version': source.source_version}, contract_version=5) + filtered = await call( + app, + 'discovery.lexical_search', + { + 'scope': '/_rules', + 'terms': 'identity', + 'revision': source.id, + 'source_kind': 'release', + 'source_version': source.source_version, + }, + contract_version=5, + ) assert filtered.status == 'ok', wire(filtered) assert [item['id'] for item in filtered.data['items']] == [ref] - missed = await call(app, 'discovery.lexical_search', { - 'scope': '/_rules', 'terms': 'identity', 'revision': source.id, - 'source_version': source.source_version + 1}, contract_version=5) + missed = await call( + app, + 'discovery.lexical_search', + { + 'scope': '/_rules', + 'terms': 'identity', + 'revision': source.id, + 'source_version': source.source_version + 1, + }, + contract_version=5, + ) assert missed.status == 'ok' and not missed.data['items'] @pytest.mark.asyncio async def test_v5_source_version_cursor_preserves_contract_and_rejects_downgrade(installed): app, _ = installed - first = await call(app, 'discovery.lexical_search', { - 'scope': '/_rules', 'terms': 'identity', 'source_kind': 'release', - 'source_version': 1, 'limit': 1}, contract_version=5) + first = await call( + app, + 'discovery.lexical_search', + { + 'scope': '/_rules', + 'terms': 'identity', + 'source_kind': 'release', + 'source_version': 1, + 'limit': 1, + }, + contract_version=5, + ) assert first.status == 'ok' and first.data['cursor'], wire(first) - downgraded = await call(app, 'discovery.lexical_search', - {'cursor': first.data['cursor']}, contract_version=4) + downgraded = await call( + app, 'discovery.lexical_search', {'cursor': first.data['cursor']}, contract_version=4 + ) assert downgraded.status == 'error' and downgraded.error.code == 'cursor_query_mismatch' - async with httpx.AsyncClient(transport=httpx.ASGITransport(app=create_app(app)), - base_url=app.settings.service_url) as http: + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=create_app(app)), base_url=app.settings.service_url + ) as http: next_page = await http.get(first.data['next']) assert next_page.status_code == 200, next_page.text query = await http.get('/_search?scope=%2F_rules&terms=identity&source_version=1') @@ -197,18 +322,36 @@ async def test_v5_source_version_cursor_preserves_contract_and_rejects_downgrade async def test_v5_revision_filter_does_not_reveal_private_resource(installed): app, _ = installed key, subject, _ = await register(app, 'private-revision-owner') - created = await call(app, 'content.post_create', - {'parent': '/main', 'body': 'privatev5needle'}, key=key, subject=subject) + created = await call( + app, + 'content.post_create', + {'parent': '/main', 'body': 'privatev5needle'}, + key=key, + subject=subject, + ) rid = created.resources[0].id - changed = await call(app, 'content.chmod', {'id': rid, 'mode': '0600'}, - key=key, subject=subject, expected=((rid, created.data['generation']),)) + changed = await call( + app, + 'content.chmod', + {'id': rid, 'mode': '0600'}, + key=key, + subject=subject, + expected=((rid, created.data['generation']),), + ) assert changed.status == 'ok', wire(changed) - args = {'scope': '/main', 'terms': 'privatev5needle', - 'revision': created.resources[0].revision, 'facets': ['type'], 'snippet': True} + args = { + 'scope': '/main', + 'terms': 'privatev5needle', + 'revision': created.resources[0].revision, + 'facets': ['type'], + 'snippet': True, + } denied = await call(app, 'discovery.lexical_search', args, contract_version=5) assert denied.status == 'ok' and not denied.data['items'] assert wire(denied.data['facets']) == {'type': []} - allowed = await call(app, 'discovery.lexical_search', args, key=key, subject=subject, contract_version=5) + allowed = await call( + app, 'discovery.lexical_search', args, key=key, subject=subject, contract_version=5 + ) assert allowed.status == 'ok' and [item['id'] for item in allowed.data['items']] == [rid] @@ -216,42 +359,79 @@ async def test_v5_revision_filter_does_not_reveal_private_resource(installed): async def test_v5_directional_relations_and_presence_recheck_both_endpoints(installed): app, _ = installed key, subject, _ = await register(app, 'directional-search-owner') - root = await call(app, 'content.post_create', {'parent': '/main', 'body': 'directionneedle root'}, - key=key, subject=subject) - reply = await call(app, 'discussion.reply', {'target': {'id': root.resources[0].id}, - 'body': 'directionneedle reply'}, key=key, subject=subject) + root = await call( + app, + 'content.post_create', + {'parent': '/main', 'body': 'directionneedle root'}, + key=key, + subject=subject, + ) + reply = await call( + app, + 'discussion.reply', + {'target': {'id': root.resources[0].id}, 'body': 'directionneedle reply'}, + key=key, + subject=subject, + ) assert root.status == reply.status == 'ok' root_id, reply_id = root.resources[0].id, reply.resources[0].id base = {'scope': '/main', 'terms': 'directionneedle', 'facets': ['type']} + async def find(extra, *, signed=False): - result = await call(app, 'discovery.lexical_search', {**base, **extra}, contract_version=5, - key=key if signed else None, subject=subject if signed else None) + result = await call( + app, + 'discovery.lexical_search', + {**base, **extra}, + contract_version=5, + key=key if signed else None, + subject=subject if signed else None, + ) assert result.status == 'ok', wire(result) return [item['id'] for item in result.data['items']] + assert await find({'relation_to': root_id, 'relation_type': 'reply_to'}) == [reply_id] assert await find({'relation_from': reply_id, 'relation_type': 'reply_to'}) == [root_id] assert await find({'has_replies': True}) == [root_id] assert await find({'has_references': True}) == [root_id] assert await find({'has_replies': False}) == [reply_id] - private = await call(app, 'content.chmod', {'id': reply_id, 'mode': '0600'}, - key=key, subject=subject, expected=((reply_id, reply.data['generation']),)) + private = await call( + app, + 'content.chmod', + {'id': reply_id, 'mode': '0600'}, + key=key, + subject=subject, + expected=((reply_id, reply.data['generation']),), + ) assert private.status == 'ok', wire(private) assert await find({'relation_from': reply_id}) == [] assert await find({'has_replies': True}) == [] assert await find({'has_references': True}) == [] assert await find({'has_replies': True}, signed=True) == [root_id] - async with httpx.AsyncClient(transport=httpx.ASGITransport(app=create_app(app)), - base_url=app.settings.service_url) as http: + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=create_app(app)), base_url=app.settings.service_url + ) as http: query = await http.get('/_search', params={**base, 'facets': 'type', 'has_replies': '1'}) path = await http.get('/_s/q/5/s/%2Fmain/t/directionneedle/fc/type/hr/1') assert query.status_code == path.status_code == 200, (query.text, path.text) assert query.content == path.content # Readable source cannot reveal its now-private target through a predicate. - hidden = await call(app, 'content.chmod', {'id': root_id, 'mode': '0600'}, - key=key, subject=subject, expected=((root_id, root.data['generation']),)) + hidden = await call( + app, + 'content.chmod', + {'id': root_id, 'mode': '0600'}, + key=key, + subject=subject, + expected=((root_id, root.data['generation']),), + ) assert hidden.status == 'ok', wire(hidden) - public_reply = await call(app, 'content.chmod', {'id': reply_id, 'mode': '0644'}, - key=key, subject=subject, expected=((reply_id, private.data['generation']),)) + public_reply = await call( + app, + 'content.chmod', + {'id': reply_id, 'mode': '0644'}, + key=key, + subject=subject, + expected=((reply_id, private.data['generation']),), + ) assert public_reply.status == 'ok', wire(public_reply) assert await find({'relation_to': root_id}) == [] assert await find({'relation_to': root_id}, signed=True) == [reply_id] @@ -261,20 +441,38 @@ async def find(extra, *, signed=False): async def test_v5_title_is_explicit_resource_display_name_alias(installed): app, _ = installed key, subject, _ = await register(app, 'title-search-owner') - created = await call(app, 'content.post_create', { - 'parent': '/main', 'name': 'NeedleTitle', 'body': 'unrelated content'}, key=key, subject=subject) + created = await call( + app, + 'content.post_create', + {'parent': '/main', 'name': 'NeedleTitle', 'body': 'unrelated content'}, + key=key, + subject=subject, + ) assert created.status == 'ok', wire(created) - args = {'scope': '/main', 'terms': 'needletitle', 'field': 'title', - 'fields': ['id', 'title', 'snippet'], 'snippet': True} + args = { + 'scope': '/main', + 'terms': 'needletitle', + 'field': 'title', + 'fields': ['id', 'title', 'snippet'], + 'snippet': True, + } found = await call(app, 'discovery.lexical_search', args, contract_version=5) assert found.status == 'ok', wire(found) - assert wire(found.data['items']) == [{'id': created.resources[0].id, 'title': 'NeedleTitle.md', - 'snippet': {'field': 'title', 'text': 'NeedleTitle.md', 'range': [0, 11]}}] + assert wire(found.data['items']) == [ + { + 'id': created.resources[0].id, + 'title': 'NeedleTitle.md', + 'snippet': {'field': 'title', 'text': 'NeedleTitle.md', 'range': [0, 11]}, + } + ] old = await call(app, 'discovery.lexical_search', args, contract_version=4) assert old.status == 'error' - async with httpx.AsyncClient(transport=httpx.ASGITransport(app=create_app(app)), - base_url=app.settings.service_url) as http: - query = await http.get('/_search', params={**args, 'fields': 'id,title,snippet', 'snippet': '1'}) + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=create_app(app)), base_url=app.settings.service_url + ) as http: + query = await http.get( + '/_search', params={**args, 'fields': 'id,title,snippet', 'snippet': '1'} + ) path = await http.get('/_s/q/5/s/%2Fmain/t/needletitle/f/t/fi/id,title,snippet/x/1') assert query.status_code == path.status_code == 200, (query.text, path.text) assert query.content == path.content @@ -285,12 +483,18 @@ async def test_v5_title_is_explicit_resource_display_name_alias(installed): @pytest.mark.asyncio async def test_v5_typed_scopes_union_resources_and_subject_with_current_access(installed): from urllib.parse import quote + app, _ = installed key, subject, _ = await register(app, 'scope-search-owner') posts = [] for name in ('scope-one', 'scope-two', 'scope-outside'): - post = await call(app, 'content.post_create', {'parent': '/main', 'name': name, 'body': 'scopev5needle'}, - key=key, subject=subject) + post = await call( + app, + 'content.post_create', + {'parent': '/main', 'name': name, 'body': 'scopev5needle'}, + key=key, + subject=subject, + ) assert post.status == 'ok', wire(post) posts.append(post) ids = [post.resources[0].id for post in posts] @@ -299,48 +503,117 @@ async def test_v5_typed_scopes_union_resources_and_subject_with_current_access(i result = await call(app, 'discovery.lexical_search', args, contract_version=5) assert result.status == 'ok', wire(result) assert [item['id'] for item in result.data['items']] == [ids[0]] - continued = await call(app, 'discovery.lexical_search', {'cursor': result.data['cursor']}, contract_version=5) + continued = await call( + app, 'discovery.lexical_search', {'cursor': result.data['cursor']}, contract_version=5 + ) assert continued.status == 'ok' and [item['id'] for item in continued.data['items']] == [ids[1]] - owned = await call(app, 'discovery.lexical_search', {**args, 'scope': {'subject': subject}, 'limit': 10}, - key=key, subject=subject, contract_version=5) - assert owned.status == 'ok' and {item['id'] for item in owned.data['items']} == set(ids), wire(owned) - bad_type = await call(app, 'discovery.lexical_search', {**args, 'scope': {'org': subject}}, contract_version=5) + owned = await call( + app, + 'discovery.lexical_search', + {**args, 'scope': {'subject': subject}, 'limit': 10}, + key=key, + subject=subject, + contract_version=5, + ) + assert owned.status == 'ok' and {item['id'] for item in owned.data['items']} == set(ids), wire( + owned + ) + bad_type = await call( + app, 'discovery.lexical_search', {**args, 'scope': {'org': subject}}, contract_version=5 + ) assert bad_type.status == 'error' and bad_type.error.code == 'invalid_search_scope' - group = await call(app, 'group.create', {'name': 'typed-search-group'}, key=key, subject=subject) + group = await call( + app, 'group.create', {'name': 'typed-search-group'}, key=key, subject=subject + ) assert group.status == 'ok', wire(group) group_id = group.resources[0].id - grouped = await call(app, 'content.chgrp', {'id': ids[0], 'group': group_id}, - key=key, subject=subject, expected=((ids[0], posts[0].data['generation']),)) + grouped = await call( + app, + 'content.chgrp', + {'id': ids[0], 'group': group_id}, + key=key, + subject=subject, + expected=((ids[0], posts[0].data['generation']),), + ) assert grouped.status == 'ok', wire(grouped) - group_search = await call(app, 'discovery.lexical_search', - {**args, 'scope': {'org': group_id}, 'limit': 10}, key=key, subject=subject, contract_version=5) - assert group_search.status == 'ok' and [item['id'] for item in group_search.data['items']] == [ids[0]], wire(group_search) - async with httpx.AsyncClient(transport=httpx.ASGITransport(app=create_app(app)), - base_url=app.settings.service_url) as http: - query = await http.get('/_search', params={'scope': canonical(scope).decode(), 'terms': 'scopev5needle'}) - path = await http.get('/_s/q/5/s/' + quote(canonical(scope).decode(), safe=',') + '/t/scopev5needle') + group_search = await call( + app, + 'discovery.lexical_search', + {**args, 'scope': {'org': group_id}, 'limit': 10}, + key=key, + subject=subject, + contract_version=5, + ) + assert group_search.status == 'ok' and [item['id'] for item in group_search.data['items']] == [ + ids[0] + ], wire(group_search) + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=create_app(app)), base_url=app.settings.service_url + ) as http: + query = await http.get( + '/_search', params={'scope': canonical(scope).decode(), 'terms': 'scopev5needle'} + ) + path = await http.get( + '/_s/q/5/s/' + quote(canonical(scope).decode(), safe=',') + '/t/scopev5needle' + ) assert query.status_code == path.status_code == 200, (query.text, path.text) assert query.content == path.content - descriptor=canonical({'version':1,'kind':'search','arguments':args}) - opened=await call(app,'transfer.open',{'direction':'upload','size':len(descriptor), - 'digest':digest(descriptor),'media_type':'application/vnd.msg.read-query+json'}, - key=key,subject=subject) - tid=opened.data['transfer_id'] - await call(app,'transfer.part_put',{'transfer_id':tid,'offset':0, - 'data':b64(descriptor),'digest':digest(descriptor)},key=key,subject=subject) - await call(app,'transfer.seal',{'transfer_id':tid,'final_size':len(descriptor), - 'final_digest':digest(descriptor)},key=key,subject=subject) - sealed=await call(app,'transfer.query_seal',{'transfer_id':tid},key=key,subject=subject) - assert sealed.status=='ok',wire(sealed) - first=await call(app,'transfer.query_get',{'query_ref':sealed.data['query_ref']},key=key,subject=subject) - assert first.status=='ok' and first.data['cursor'],wire(first) - second=await call(app,'transfer.query_get',{'query_ref':sealed.data['query_ref'], - 'cursor':first.data['cursor']},key=key,subject=subject) - assert second.status=='ok' and [item['id'] for item in second.data['items']]==[ids[1]],wire(second) - changed = await call(app, 'content.chmod', {'id': ids[1], 'mode': '0600'}, key=key, subject=subject, - expected=((ids[1], posts[1].data['generation']),)) + descriptor = canonical({'version': 1, 'kind': 'search', 'arguments': args}) + opened = await call( + app, + 'transfer.open', + { + 'direction': 'upload', + 'size': len(descriptor), + 'digest': digest(descriptor), + 'media_type': 'application/vnd.msg.read-query+json', + }, + key=key, + subject=subject, + ) + tid = opened.data['transfer_id'] + await call( + app, + 'transfer.part_put', + {'transfer_id': tid, 'offset': 0, 'data': b64(descriptor), 'digest': digest(descriptor)}, + key=key, + subject=subject, + ) + await call( + app, + 'transfer.seal', + {'transfer_id': tid, 'final_size': len(descriptor), 'final_digest': digest(descriptor)}, + key=key, + subject=subject, + ) + sealed = await call(app, 'transfer.query_seal', {'transfer_id': tid}, key=key, subject=subject) + assert sealed.status == 'ok', wire(sealed) + first = await call( + app, 'transfer.query_get', {'query_ref': sealed.data['query_ref']}, key=key, subject=subject + ) + assert first.status == 'ok' and first.data['cursor'], wire(first) + second = await call( + app, + 'transfer.query_get', + {'query_ref': sealed.data['query_ref'], 'cursor': first.data['cursor']}, + key=key, + subject=subject, + ) + assert second.status == 'ok' and [item['id'] for item in second.data['items']] == [ids[1]], ( + wire(second) + ) + changed = await call( + app, + 'content.chmod', + {'id': ids[1], 'mode': '0600'}, + key=key, + subject=subject, + expected=((ids[1], posts[1].data['generation']),), + ) assert changed.status == 'ok', wire(changed) - revoked = await call(app, 'discovery.lexical_search', {'cursor': result.data['cursor']}, contract_version=5) + revoked = await call( + app, 'discovery.lexical_search', {'cursor': result.data['cursor']}, contract_version=5 + ) assert revoked.status == 'error' old = await call(app, 'discovery.lexical_search', args, contract_version=4) assert old.status == 'error' diff --git a/tests/test_search_spelling.py b/tests/test_search_spelling.py index 5fb2ad19..fda81354 100644 --- a/tests/test_search_spelling.py +++ b/tests/test_search_spelling.py @@ -1,4 +1,5 @@ """Spelling hints are explicit, deterministic and limited to visible names.""" + import httpx import pytest from test_lexical_search_grep import business_state @@ -14,49 +15,85 @@ async def test_spelling_is_opt_in_and_rechecks_visible_name_dictionary(installed key, subject, _ = await register(app, 'spelling-owner') posts = [] for name in ('gadget', 'gadger'): - created = await call(app, 'content.post_create', - {'parent': '/main', 'name': name, 'body': 'a product'}, key=key, subject=subject) + created = await call( + app, + 'content.post_create', + {'parent': '/main', 'name': name, 'body': 'a product'}, + key=key, + subject=subject, + ) assert created.status == 'ok', wire(created) posts.append(created) - private = await call(app, 'content.chmod', {'id': posts[1].resources[0].id, 'mode': '0600'}, - key=key, subject=subject, expected=((posts[1].resources[0].id, posts[1].data['generation']),)) + private = await call( + app, + 'content.chmod', + {'id': posts[1].resources[0].id, 'mode': '0600'}, + key=key, + subject=subject, + expected=((posts[1].resources[0].id, posts[1].data['generation']),), + ) assert private.status == 'ok', wire(private) args = {'scope': '/main', 'terms': 'gadge', 'field': 'name'} before = await business_state(app) ordinary = await call(app, 'discovery.lexical_search', args, contract_version=5) assert ordinary.status == 'ok' and 'spelling' not in ordinary.data - hinted = await call(app, 'discovery.lexical_search', {**args, 'spell': True}, contract_version=5) + hinted = await call( + app, 'discovery.lexical_search', {**args, 'spell': True}, contract_version=5 + ) assert hinted.status == 'ok', wire(hinted) - assert wire(hinted.data['spelling']) == {'source': 'name', 'terms': [ - {'term': 'gadge', 'suggestions': [{'value': 'gadget', 'distance': 1, 'count': 1}]}]} - assert [row['id'] for row in hinted.data['items']] == [row['id'] for row in ordinary.data['items']] + assert wire(hinted.data['spelling']) == { + 'source': 'name', + 'terms': [ + {'term': 'gadge', 'suggestions': [{'value': 'gadget', 'distance': 1, 'count': 1}]} + ], + } + assert [row['id'] for row in hinted.data['items']] == [ + row['id'] for row in ordinary.data['items'] + ] old = await call(app, 'discovery.lexical_search', {**args, 'spell': True}, contract_version=4) assert old.status == 'error' - short = await call(app, 'discovery.lexical_search', {**args, 'terms': 'g', 'spell': True}, contract_version=5) + short = await call( + app, 'discovery.lexical_search', {**args, 'terms': 'g', 'spell': True}, contract_version=5 + ) assert short.status == 'error' and short.error.code == 'query_cost_exceeded' - async with httpx.AsyncClient(transport=httpx.ASGITransport(app=create_app(app)), - base_url=app.settings.service_url) as http: + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=create_app(app)), base_url=app.settings.service_url + ) as http: query = await http.get('/_search', params={**args, 'spell': '1'}) path = await http.get('/_s/q/5/s/%2Fmain/t/gadge/f/n/sp/1') assert query.status_code == path.status_code == 200, (query.text, path.text) assert query.content == path.content assert await business_state(app) == before - revoked = await call(app, 'content.chmod', {'id': posts[0].resources[0].id, 'mode': '0600'}, - key=key, subject=subject, expected=((posts[0].resources[0].id, posts[0].data['generation']),)) + revoked = await call( + app, + 'content.chmod', + {'id': posts[0].resources[0].id, 'mode': '0600'}, + key=key, + subject=subject, + expected=((posts[0].resources[0].id, posts[0].data['generation']),), + ) assert revoked.status == 'ok', wire(revoked) - hidden = await call(app, 'discovery.lexical_search', {**args, 'spell': True}, contract_version=5) + hidden = await call( + app, 'discovery.lexical_search', {**args, 'spell': True}, contract_version=5 + ) assert hidden.status == 'ok' and not hidden.data['items'] - assert wire(hidden.data['spelling']) == {'source': 'name', 'terms': [{'term': 'gadge', 'suggestions': []}]} + assert wire(hidden.data['spelling']) == { + 'source': 'name', + 'terms': [{'term': 'gadge', 'suggestions': []}], + } @pytest.mark.asyncio async def test_v5_registered_collaboration_relation_names_select_same_http_contract(installed): app, _ = installed - async with httpx.AsyncClient(transport=httpx.ASGITransport(app=create_app(app)), - base_url=app.settings.service_url) as http: + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=create_app(app)), base_url=app.settings.service_url + ) as http: for kind in ('reference', 'state', 'target', 'content'): - query = await http.get('/_search', params={ - 'scope': '/main', 'terms': 'relationneedle', 'relation_type': kind}) + query = await http.get( + '/_search', + params={'scope': '/main', 'terms': 'relationneedle', 'relation_type': kind}, + ) path = await http.get('/_s/q/5/s/%2Fmain/t/relationneedle/rt/' + kind) assert query.status_code == path.status_code == 200, (query.text, path.text) assert query.content == path.content diff --git a/tests/test_search_suggestions.py b/tests/test_search_suggestions.py index 2357d7bc..dd4c4f63 100644 --- a/tests/test_search_suggestions.py +++ b/tests/test_search_suggestions.py @@ -1,111 +1,170 @@ """Search suggestions are a bounded projection of current readable matches.""" + from datetime import timedelta import httpx import pytest +from test_service import NOW, call, register -from msg.core.codec import b64,canonical,digest,wire +from msg.core.codec import b64, canonical, digest, wire from msg.core.requests import request_for from msg.transports.http import create_app -from test_service import NOW,call,register @pytest.mark.asyncio async def test_v4_http_and_query_ref_use_same_authorized_suggestions(installed): - app,_=installed - key,user,_=await register(app,'suggest-http-owner') - created=await call(app,'content.file_put',{'parent':'/main', - 'name':'suggestpath-one.txt','data':b64(b'body'),'media_type':'text/plain'}, - key=key,subject=user) - assert created.status=='ok' - query='/_search?scope=%2Fmain&terms=suggestpath&field=name&suggest=1' - path='/_s/q/4/s/%2Fmain/t/suggestpath/f/n/sg/1' - async with httpx.AsyncClient(transport=httpx.ASGITransport(app=create_app(app)), - base_url=app.settings.service_url) as http: - left=await http.get(query) - right=await http.get(path) - assert left.status_code==right.status_code==200,(left.text,right.text) - assert left.content==right.content - assert left.json()['suggestions']==[{'value':'suggestpath-one','count':1}] - args={'scope':'/main','terms':'suggestpath','field':'name','suggest':True} - descriptor=canonical({'version':1,'kind':'search','arguments':args}) - opened=await call(app,'transfer.open',{'direction':'upload','size':len(descriptor), - 'digest':digest(descriptor),'media_type':'application/vnd.msg.read-query+json'}, - key=key,subject=user) - tid=opened.data['transfer_id'] - await call(app,'transfer.part_put',{'transfer_id':tid,'offset':0, - 'data':b64(descriptor),'digest':digest(descriptor)},key=key,subject=user) - await call(app,'transfer.seal',{'transfer_id':tid,'final_size':len(descriptor), - 'final_digest':digest(descriptor)},key=key,subject=user) - sealed=await call(app,'transfer.query_seal',{'transfer_id':tid},key=key,subject=user) - assert sealed.status=='ok',wire(sealed) - token=sealed.data['query_ref'] - proof=request_for('transfer.query_get',{'query_ref':token},app.settings.service_url, - signer=key,subject=user,expires_at=NOW+timedelta(seconds=60)) - async with httpx.AsyncClient(transport=httpx.ASGITransport(app=create_app(app)), - base_url=app.settings.service_url) as http: - resolved=await http.get('/_r/q/'+token+'/p/'+b64(canonical(proof))) - assert resolved.status_code==200,resolved.text - assert resolved.json()['suggestions']==left.json()['suggestions'] + app, _ = installed + key, user, _ = await register(app, 'suggest-http-owner') + created = await call( + app, + 'content.file_put', + { + 'parent': '/main', + 'name': 'suggestpath-one.txt', + 'data': b64(b'body'), + 'media_type': 'text/plain', + }, + key=key, + subject=user, + ) + assert created.status == 'ok' + query = '/_search?scope=%2Fmain&terms=suggestpath&field=name&suggest=1' + path = '/_s/q/4/s/%2Fmain/t/suggestpath/f/n/sg/1' + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=create_app(app)), base_url=app.settings.service_url + ) as http: + left = await http.get(query) + right = await http.get(path) + assert left.status_code == right.status_code == 200, (left.text, right.text) + assert left.content == right.content + assert left.json()['suggestions'] == [{'value': 'suggestpath-one', 'count': 1}] + args = {'scope': '/main', 'terms': 'suggestpath', 'field': 'name', 'suggest': True} + descriptor = canonical({'version': 1, 'kind': 'search', 'arguments': args}) + opened = await call( + app, + 'transfer.open', + { + 'direction': 'upload', + 'size': len(descriptor), + 'digest': digest(descriptor), + 'media_type': 'application/vnd.msg.read-query+json', + }, + key=key, + subject=user, + ) + tid = opened.data['transfer_id'] + await call( + app, + 'transfer.part_put', + {'transfer_id': tid, 'offset': 0, 'data': b64(descriptor), 'digest': digest(descriptor)}, + key=key, + subject=user, + ) + await call( + app, + 'transfer.seal', + {'transfer_id': tid, 'final_size': len(descriptor), 'final_digest': digest(descriptor)}, + key=key, + subject=user, + ) + sealed = await call(app, 'transfer.query_seal', {'transfer_id': tid}, key=key, subject=user) + assert sealed.status == 'ok', wire(sealed) + token = sealed.data['query_ref'] + proof = request_for( + 'transfer.query_get', + {'query_ref': token}, + app.settings.service_url, + signer=key, + subject=user, + expires_at=NOW + timedelta(seconds=60), + ) + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=create_app(app)), base_url=app.settings.service_url + ) as http: + resolved = await http.get('/_r/q/' + token + '/p/' + b64(canonical(proof))) + assert resolved.status_code == 200, resolved.text + assert resolved.json()['suggestions'] == left.json()['suggestions'] @pytest.mark.asyncio async def test_v4_suggestions_count_only_matching_readable_names_and_recheck_page(installed): - app,_=installed - key,user,_=await register(app,'suggest-owner') - created=[] - for name in ('suggestible-one.txt','suggestible-two.txt','suggestible-secret.txt'): - result=await call(app,'content.file_put',{'parent':'/main','name':name, - 'data':b64(b'body'),'media_type':'text/plain'},key=key,subject=user) - assert result.status=='ok',wire(result) + app, _ = installed + key, user, _ = await register(app, 'suggest-owner') + created = [] + for name in ('suggestible-one.txt', 'suggestible-two.txt', 'suggestible-secret.txt'): + result = await call( + app, + 'content.file_put', + {'parent': '/main', 'name': name, 'data': b64(b'body'), 'media_type': 'text/plain'}, + key=key, + subject=user, + ) + assert result.status == 'ok', wire(result) created.append(result) - hidden=created[2] - locked=await call(app,'content.chmod',{'id':hidden.resources[0].id,'mode':'0600'}, - key=key,subject=user, - expected=((hidden.resources[0].id,hidden.data['generation']),)) - assert locked.status=='ok',wire(locked) - args={'scope':'/main','terms':'suggest','field':'name','limit':1,'suggest':True} - first=await call(app,'discovery.lexical_search',args,contract_version=4) - assert first.status=='ok',wire(first) - assert wire(first)['data']['suggestions']==[ - {'value':'suggestible-one','count':1}, - {'value':'suggestible-two','count':1}] + hidden = created[2] + locked = await call( + app, + 'content.chmod', + {'id': hidden.resources[0].id, 'mode': '0600'}, + key=key, + subject=user, + expected=((hidden.resources[0].id, hidden.data['generation']),), + ) + assert locked.status == 'ok', wire(locked) + args = {'scope': '/main', 'terms': 'suggest', 'field': 'name', 'limit': 1, 'suggest': True} + first = await call(app, 'discovery.lexical_search', args, contract_version=4) + assert first.status == 'ok', wire(first) + assert wire(first)['data']['suggestions'] == [ + {'value': 'suggestible-one', 'count': 1}, + {'value': 'suggestible-two', 'count': 1}, + ] assert first.data['cursor'] - old=await call(app,'discovery.lexical_search',{'cursor':first.data['cursor']}, - contract_version=3) - assert old.status=='error' and old.error.code=='cursor_query_mismatch' - first_id=first.data['items'][0]['id'] - newly_hidden=next(item for item in created[:2] - if item.resources[0].id!=first_id) - locked=await call(app,'content.chmod',{'id':newly_hidden.resources[0].id,'mode':'0600'}, - key=key,subject=user, - expected=((newly_hidden.resources[0].id,newly_hidden.data['generation']),)) - assert locked.status=='ok',wire(locked) - continued=await call(app,'discovery.lexical_search', - {'cursor':first.data['cursor']},contract_version=4) - assert continued.status=='ok',wire(continued) - expected_word=next(name[:-4] for name,item in zip( - ('suggestible-one.txt','suggestible-two.txt'),created[:2]) - if item.resources[0].id==first_id) - assert wire(continued)['data']['suggestions']==[{'value':expected_word,'count':1}] - assert list(continued.data['items'])==[] - owner=await call(app,'discovery.lexical_search',args,key=key,subject=user, - contract_version=4) - assert owner.status=='ok',wire(owner) - assert len(owner.data['suggestions'])==3 + old = await call( + app, 'discovery.lexical_search', {'cursor': first.data['cursor']}, contract_version=3 + ) + assert old.status == 'error' and old.error.code == 'cursor_query_mismatch' + first_id = first.data['items'][0]['id'] + newly_hidden = next(item for item in created[:2] if item.resources[0].id != first_id) + locked = await call( + app, + 'content.chmod', + {'id': newly_hidden.resources[0].id, 'mode': '0600'}, + key=key, + subject=user, + expected=((newly_hidden.resources[0].id, newly_hidden.data['generation']),), + ) + assert locked.status == 'ok', wire(locked) + continued = await call( + app, 'discovery.lexical_search', {'cursor': first.data['cursor']}, contract_version=4 + ) + assert continued.status == 'ok', wire(continued) + expected_word = next( + name[:-4] + for name, item in zip(('suggestible-one.txt', 'suggestible-two.txt'), created[:2], strict=True) + if item.resources[0].id == first_id + ) + assert wire(continued)['data']['suggestions'] == [{'value': expected_word, 'count': 1}] + assert list(continued.data['items']) == [] + owner = await call( + app, 'discovery.lexical_search', args, key=key, subject=user, contract_version=4 + ) + assert owner.status == 'ok', wire(owner) + assert len(owner.data['suggestions']) == 3 @pytest.mark.asyncio async def test_v4_suggest_is_explicit_and_old_schemas_are_stable(installed): - app,_=installed - args={'scope':'/main','terms':'sug','field':'name'} - for version in (1,2,3): - result=await call(app,'discovery.lexical_search', - {**args,'suggest':True},contract_version=version) - assert result.status=='error',wire(result) - result=await call(app,'discovery.lexical_search',args,contract_version=4) - assert result.status=='ok',wire(result) + app, _ = installed + args = {'scope': '/main', 'terms': 'sug', 'field': 'name'} + for version in (1, 2, 3): + result = await call( + app, 'discovery.lexical_search', {**args, 'suggest': True}, contract_version=version + ) + assert result.status == 'error', wire(result) + result = await call(app, 'discovery.lexical_search', args, contract_version=4) + assert result.status == 'ok', wire(result) assert 'suggestions' not in result.data - too_short=await call(app,'discovery.lexical_search', - {**args,'terms':'s','suggest':True},contract_version=4) - assert too_short.status=='error' and too_short.error.code=='query_cost_exceeded' + too_short = await call( + app, 'discovery.lexical_search', {**args, 'terms': 's', 'suggest': True}, contract_version=4 + ) + assert too_short.status == 'error' and too_short.error.code == 'query_cost_exceeded' diff --git a/tests/test_security.py b/tests/test_security.py index 70308082..574b0631 100644 --- a/tests/test_security.py +++ b/tests/test_security.py @@ -1,60 +1,72 @@ from dataclasses import replace -from datetime import UTC, datetime, timedelta + import pytest +from test_foundation import item -from msg.core.codec import canonical, decode, wire +from msg.core.codec import canonical from msg.core.errors import Failure -from msg.security.crypto import Ed25519Signer, verify, seal_private_key, open_private_key -from msg.security.policy import selected_class, class_bits, scope_contains, grant_covers -from msg.core.models import Scope, CapabilityGrant, ResourceId +from msg.core.models import CapabilityGrant, Scope +from msg.security.crypto import Ed25519Signer, open_private_key, seal_private_key, verify +from msg.security.policy import class_bits, grant_covers, scope_contains, selected_class from msg.storage.sqlite import SqliteMetadataStore -from test_foundation import item def test_signatures_separate_purposes_and_reject_tampering(): - signer=Ed25519Signer.generate() - sig=signer.sign(b'hello',purpose='request') - verify(signer.public_key,b'hello',sig,purpose='request') - for payload,purpose in [(b'hello','receipt'),(b'bye','request')]: - with pytest.raises(Failure,match='invalid_signature'): - verify(signer.public_key,payload,sig,purpose=purpose) + signer = Ed25519Signer.generate() + sig = signer.sign(b'hello', purpose='request') + verify(signer.public_key, b'hello', sig, purpose='request') + for payload, purpose in [(b'hello', 'receipt'), (b'bye', 'request')]: + with pytest.raises(Failure, match='invalid_signature'): + verify(signer.public_key, payload, sig, purpose=purpose) def test_encrypted_key_roundtrip_and_wrong_pin(): - signer=Ed25519Signer.generate() - wrapped=seal_private_key(signer.private_bytes(), 'correct horse battery staple') - assert open_private_key(wrapped,'correct horse battery staple')==signer.private_bytes() + signer = Ed25519Signer.generate() + wrapped = seal_private_key(signer.private_bytes(), 'correct horse battery staple') + assert open_private_key(wrapped, 'correct horse battery staple') == signer.private_bytes() assert signer.private_bytes() not in canonical(wrapped) - with pytest.raises(Failure,match='invalid_pin'): - open_private_key(wrapped,'wrong horse battery staple') - altered=dict(wrapped,public_key='altered') - with pytest.raises(Failure): open_private_key(altered,'correct horse battery staple') - for pin in ['', '1234','password','00000000']: - with pytest.raises(Failure,match='weak_pin'): - seal_private_key(signer.private_bytes(),pin) + with pytest.raises(Failure, match='invalid_pin'): + open_private_key(wrapped, 'wrong horse battery staple') + altered = dict(wrapped, public_key='altered') + with pytest.raises(Failure): + open_private_key(altered, 'correct horse battery staple') + for pin in ['', '1234', 'password', '00000000']: + with pytest.raises(Failure, match='weak_pin'): + seal_private_key(signer.private_bytes(), pin) def test_mode_selection_is_not_a_union(): - r=replace(item(),owner='alice',group='staff',mode=0o046) - assert selected_class(r,'alice',{'staff'})=='owner' - assert class_bits(r.mode,'owner')==0 - assert selected_class(r,'bob',{'staff'})=='group' - assert selected_class(r,None,set())=='other' + r = replace(item(), owner='alice', group='staff', mode=0o046) + assert selected_class(r, 'alice', {'staff'}) == 'owner' + assert class_bits(r.mode, 'owner') == 0 + assert selected_class(r, 'bob', {'staff'}) == 'group' + assert selected_class(r, None, set()) == 'other' async def test_scopes_follow_current_parent_and_operation_exactness(tmp_path): - store=SqliteMetadataStore(tmp_path/'db') + store = SqliteMetadataStore(tmp_path / 'db') async with store.transaction(write=True) as tx: - for r in [item('root'),item('a','root'),item('b','root'),item('child','a')]: + for r in [item('root'), item('a', 'root'), item('b', 'root'), item('child', 'a')]: await tx.insert(r) - scope=Scope(resource_id='a',descendants=True) - assert await scope_contains(scope,'child',tx) - assert not await scope_contains(Scope(resource_id='a'),'child',tx) - grant=CapabilityGrant(capability='resource.certified_write',version=1,scope=scope, - operations=frozenset({'content.post_create@1'}),constraints={}) - assert await grant_covers(grant,'resource.certified_write','content.post_create@1','child',tx) - assert not await grant_covers(grant,'resource.write_override','content.post_create@1','child',tx) - assert not await grant_covers(grant,'resource.certified_write','content.post_edit@1','child',tx) - child=await tx.resource('child') - await tx.replace(replace(child,parent='b',generation=1),0) - assert not await scope_contains(scope,'child',tx) + scope = Scope(resource_id='a', descendants=True) + assert await scope_contains(scope, 'child', tx) + assert not await scope_contains(Scope(resource_id='a'), 'child', tx) + grant = CapabilityGrant( + capability='resource.certified_write', + version=1, + scope=scope, + operations=frozenset({'content.post_create@1'}), + constraints={}, + ) + assert await grant_covers( + grant, 'resource.certified_write', 'content.post_create@1', 'child', tx + ) + assert not await grant_covers( + grant, 'resource.write_override', 'content.post_create@1', 'child', tx + ) + assert not await grant_covers( + grant, 'resource.certified_write', 'content.post_edit@1', 'child', tx + ) + child = await tx.resource('child') + await tx.replace(replace(child, parent='b', generation=1), 0) + assert not await scope_contains(scope, 'child', tx) diff --git a/tests/test_security_patch_regressions.py b/tests/test_security_patch_regressions.py index 793a3a6d..bfa93139 100644 --- a/tests/test_security_patch_regressions.py +++ b/tests/test_security_patch_regressions.py @@ -1,4 +1,5 @@ """Real authenticated regressions for security PRs 121--128 on current main.""" + import asyncio import os from datetime import timedelta @@ -18,12 +19,30 @@ from msg.workers import maintenance -async def fresh_call(app, operation, arguments, *, key=None, subject=None, token=None, - rid=None, expected=(), return_fields=()): - packet = request_for(operation, arguments, app.settings.service_url, signer=key, - subject=subject, token=token, request_id=rid, expected=expected, - return_fields=return_fields, - expires_at=app.clock() + timedelta(seconds=120)) +async def fresh_call( + app, + operation, + arguments, + *, + key=None, + subject=None, + token=None, + rid=None, + expected=(), + return_fields=(), +): + packet = request_for( + operation, + arguments, + app.settings.service_url, + signer=key, + subject=subject, + token=token, + request_id=rid, + expected=expected, + return_fields=return_fields, + expires_at=app.clock() + timedelta(seconds=120), + ) return await app.executor.execute(packet) @@ -38,26 +57,47 @@ async def test_package_projection_failure_does_not_leak_pins(installed): app, _ = installed key, subject, _ = await register(app, 'security-deposit') listing = await call(app, 'store.listing_create', listing_args(), key=key, subject=subject) - source = await call(app, 'content.file_put', {'parent': '/@security-deposit/files', - 'name': 'secret.bin', 'media_type': 'application/octet-stream', 'data': b64(b'private')}, - key=key, subject=subject) + source = await call( + app, + 'content.file_put', + { + 'parent': '/@security-deposit/files', + 'name': 'secret.bin', + 'media_type': 'application/octet-stream', + 'data': b64(b'private'), + }, + key=key, + subject=subject, + ) assert listing.status == source.status == 'ok' - args = {'listing_id': listing.resources[0].id, - 'listing_revision': listing.resources[0].revision, - 'manifest': {}, 'payload_refs': [wire(source.resources[0])]} + args = { + 'listing_id': listing.resources[0].id, + 'listing_revision': listing.resources[0].revision, + 'manifest': {}, + 'payload_refs': [wire(source.resources[0])], + } before = {p for p in (app.contents.path / 'pins').glob('*/*') if p.is_file()} for _ in range(2): - result = await fresh_call(app, 'store.package_deposit', args, key=key, subject=subject, - rid='failed-deposit', return_fields=('not_a_field',)) + result = await fresh_call( + app, + 'store.package_deposit', + args, + key=key, + subject=subject, + rid='failed-deposit', + return_fields=('not_a_field',), + ) assert result.error.code == 'unknown_projection_field', wire(result) assert {p for p in (app.contents.path / 'pins').glob('*/*') if p.is_file()} == before async with app.metadata.transaction(write=False) as tx: assert tx.one('SELECT COUNT(*) FROM store_packages')[0] == 0 - accepted = await fresh_call(app, 'store.package_deposit', args, key=key, subject=subject, - rid='valid-deposit') + accepted = await fresh_call( + app, 'store.package_deposit', args, key=key, subject=subject, rid='valid-deposit' + ) assert accepted.status == 'ok', wire(accepted) - replay = await fresh_call(app, 'store.package_deposit', args, key=key, subject=subject, - rid='valid-deposit') + replay = await fresh_call( + app, 'store.package_deposit', args, key=key, subject=subject, rid='valid-deposit' + ) assert replay.status == 'ok' and replay.replayed async with app.metadata.transaction(write=False) as tx: assert tx.one('SELECT COUNT(*) FROM store_packages')[0] == 1 @@ -69,21 +109,29 @@ async def test_package_projection_failure_does_not_leak_pins(installed): async def test_achievement_concurrent_start_and_expired_replacement(installed): app, _ = installed key, subject, _ = await register(app, 'security-achievement') - results = await asyncio.gather(*(fresh_call(app, 'achievement.start', {}, key=key, - subject=subject, rid=f'start-{i}') for i in range(2))) + results = await asyncio.gather( + *( + fresh_call(app, 'achievement.start', {}, key=key, subject=subject, rid=f'start-{i}') + for i in range(2) + ) + ) successful = [r for r in results if r.status == 'ok'] denied = [r for r in results if r.status == 'error'] assert len(successful) == len(denied) == 1, [wire(r) for r in results] assert denied[0].error.code == 'achievement_ceremony_active' - replay = await fresh_call(app, 'achievement.start', {}, key=key, subject=subject, - rid=successful[0].request_id) + replay = await fresh_call( + app, 'achievement.start', {}, key=key, subject=subject, rid=successful[0].request_id + ) assert replay.status == 'ok' and replay.replayed set_clock(app, NOW + timedelta(seconds=CEREMONY_TTL_SECONDS + 1)) replaced = await fresh_call(app, 'achievement.start', {}, key=key, subject=subject) assert replaced.status == 'ok', wire(replaced) assert replaced.data['challenge_id'] != successful[0].data['challenge_id'] async with app.metadata.transaction(write=False) as tx: - assert tx.one('SELECT COUNT(*) FROM achievement_ceremonies WHERE subject=?', (subject,))[0] == 1 + assert ( + tx.one('SELECT COUNT(*) FROM achievement_ceremonies WHERE subject=?', (subject,))[0] + == 1 + ) @pytest.mark.asyncio @@ -96,7 +144,10 @@ async def test_achievement_failed_round_can_restart(installed): replacement = await call(app, 'achievement.start', {}, key=key, subject=subject) assert replacement.status == 'ok' async with app.metadata.transaction(write=False) as tx: - assert tx.one('SELECT COUNT(*) FROM achievement_ceremonies WHERE subject=?', (subject,))[0] == 1 + assert ( + tx.one('SELECT COUNT(*) FROM achievement_ceremonies WHERE subject=?', (subject,))[0] + == 1 + ) @pytest.mark.asyncio @@ -106,37 +157,79 @@ async def test_new_persistent_operations_obey_write_pause(installed, operation): key, subject, _ = await register(app, 'security-pause') arguments, expected = {}, () if operation == 'hosting.preview': - website = await call(app, 'hosting.create', {'parent': '/@security-pause', 'name': 'site'}, - key=key, subject=subject) - source = await call(app, 'content.file_put', {'parent': '/@security-pause/files', - 'name': 'page.html', 'media_type': 'text/html', 'data': b64(b'hello')}, key=key, subject=subject) + website = await call( + app, + 'hosting.create', + {'parent': '/@security-pause', 'name': 'site'}, + key=key, + subject=subject, + ) + source = await call( + app, + 'content.file_put', + { + 'parent': '/@security-pause/files', + 'name': 'page.html', + 'media_type': 'text/html', + 'data': b64(b'hello'), + }, + key=key, + subject=subject, + ) assert website.status == source.status == 'ok' - arguments = {'id': website.resources[0].id, - 'entries': [{'path': 'index.html', 'source': wire(source.resources[0])}]} + arguments = { + 'id': website.resources[0].id, + 'entries': [{'path': 'index.html', 'source': wire(source.resources[0])}], + } expected = ((website.resources[0].id, website.data['generation']),) async with app.metadata.transaction(write=True) as tx: tx.set_setting('runtime_config', {'accept_writes': False, 'cleanup_enabled': True}) - before = tuple(tx.one(f'SELECT COUNT(*) FROM {table}')[0] - for table in ('resources', 'revisions', 'achievement_ceremonies')) - result = await fresh_call(app, operation, arguments, key=key, subject=subject, expected=expected) + before = tuple( + tx.one(f'SELECT COUNT(*) FROM {table}')[0] + for table in ('resources', 'revisions', 'achievement_ceremonies') + ) + result = await fresh_call( + app, operation, arguments, key=key, subject=subject, expected=expected + ) assert result.status == 'error' and result.error.code == 'writes_paused', wire(result) async with app.metadata.transaction(write=False) as tx: - assert tuple(tx.one(f'SELECT COUNT(*) FROM {table}')[0] - for table in ('resources', 'revisions', 'achievement_ceremonies')) == before + assert ( + tuple( + tx.one(f'SELECT COUNT(*) FROM {table}')[0] + for table in ('resources', 'revisions', 'achievement_ceremonies') + ) + == before + ) @pytest.mark.asyncio async def test_hosting_preview_rejects_129_entries_before_storage(installed, monkeypatch): app, _ = installed key, subject, _ = await register(app, 'security-preview-limit') - website = await call(app, 'hosting.create', - {'parent': '/@security-preview-limit', 'name': 'site'}, key=key, subject=subject) - source = await call(app, 'content.file_put', {'parent': '/@security-preview-limit/files', - 'name': 'page.html', 'media_type': 'text/html', 'data': b64(b'hello')}, - key=key, subject=subject) + website = await call( + app, + 'hosting.create', + {'parent': '/@security-preview-limit', 'name': 'site'}, + key=key, + subject=subject, + ) + source = await call( + app, + 'content.file_put', + { + 'parent': '/@security-preview-limit/files', + 'name': 'page.html', + 'media_type': 'text/html', + 'data': b64(b'hello'), + }, + key=key, + subject=subject, + ) assert website.status == source.status == 'ok' - arguments = {'id': website.resources[0].id, 'entries': [ - {'path': f'{i}.html', 'source': wire(source.resources[0])} for i in range(129)]} + arguments = { + 'id': website.resources[0].id, + 'entries': [{'path': f'{i}.html', 'source': wire(source.resources[0])} for i in range(129)], + } expected = ((website.resources[0].id, website.data['generation']),) # A published wire schema is immutable. The runtime work bound is checked # after current authorization but before any source read or creation. @@ -144,29 +237,48 @@ async def test_hosting_preview_rejects_129_entries_before_storage(installed, mon assert 'maxItems' not in app.registry.schema(spec.input_schema)['properties']['entries'] app.registry.validate(spec.input_schema, arguments) async with app.metadata.transaction(write=False) as tx: - before = tuple(tx.one(f'SELECT COUNT(*) FROM {table}')[0] - for table in ('resources', 'revisions')) + before = tuple( + tx.one(f'SELECT COUNT(*) FROM {table}')[0] for table in ('resources', 'revisions') + ) + async def no_materialization(*args, **kwargs): pytest.fail('oversized preview touched the content store') + with monkeypatch.context() as patch: patch.setattr(app.contents, 'read_bytes', no_materialization) patch.setattr(app.contents, 'put_bytes', no_materialization) - denied = await fresh_call(app, 'hosting.preview', arguments, - key=key, subject=subject, expected=expected) - assert denied.status == 'error' and denied.error.code == 'too_many_preview_entries', wire(denied) + denied = await fresh_call( + app, 'hosting.preview', arguments, key=key, subject=subject, expected=expected + ) + assert denied.status == 'error' and denied.error.code == 'too_many_preview_entries', wire( + denied + ) async with app.metadata.transaction(write=False) as tx: - assert tuple(tx.one(f'SELECT COUNT(*) FROM {table}')[0] - for table in ('resources', 'revisions')) == before + assert ( + tuple( + tx.one(f'SELECT COUNT(*) FROM {table}')[0] for table in ('resources', 'revisions') + ) + == before + ) # An actual authorized 128-entry preview remains valid, not just its schema. - allowed = await fresh_call(app, 'hosting.preview', + allowed = await fresh_call( + app, + 'hosting.preview', {**arguments, 'entries': arguments['entries'][:128]}, - key=key, subject=subject, expected=expected) + key=key, + subject=subject, + expected=expected, + ) assert allowed.status == 'ok' and allowed.data['files'] == 128, wire(allowed) async def custody(app, handle): - created = await call(app, 'identity.custodial_create', {'handle': handle, - 'nonce': b64(os.urandom(32)), 'recovery_secret': b64(os.urandom(32))}, contract_version=2) + created = await call( + app, + 'identity.custodial_create', + {'handle': handle, 'nonce': b64(os.urandom(32)), 'recovery_secret': b64(os.urandom(32))}, + contract_version=2, + ) assert created.status == 'ok', wire(created) return created.data['subject_id'], (created.data['credential_id'], unb64(created.data['token'])) @@ -180,19 +292,30 @@ async def test_custodial_cleanup_preserves_accepted_upgrade_facts(installed, com private, recipient = generate_age_key() challenge = await begin(app, subject, token, signer, recipient) assert challenge.status == 'ok', wire(challenge) - result = await finish(app, subject, token, signer, private, challenge.data, - external_migrated=complete) + result = await finish( + app, subject, token, signer, private, challenge.data, external_migrated=complete + ) assert result.status == 'ok', wire(result) assert result.data['status'] == ('completed' if complete else 'pending_rewrap') async with app.metadata.transaction(write=False) as tx: - before = tx.one('SELECT * FROM custodial_upgrades WHERE id=?', (challenge.data['challenge_id'],)) + before = tx.one( + 'SELECT * FROM custodial_upgrades WHERE id=?', (challenge.data['challenge_id'],) + ) set_clock(app, NOW + timedelta(seconds=301)) await maintenance.run_maintenance(app, 'cleanup_expired', scheduled=True) async with app.metadata.transaction(write=False) as tx: - assert tx.one('SELECT * FROM custodial_upgrades WHERE id=?', (challenge.data['challenge_id'],)) == before + assert ( + tx.one('SELECT * FROM custodial_upgrades WHERE id=?', (challenge.data['challenge_id'],)) + == before + ) if complete: - recovered = await fresh_call(app, 'identity.custodial_upgrade_result', - {'challenge_id': challenge.data['challenge_id']}, key=signer, subject=subject) + recovered = await fresh_call( + app, + 'identity.custodial_upgrade_result', + {'challenge_id': challenge.data['challenge_id']}, + key=signer, + subject=subject, + ) assert recovered.status == 'ok' and recovered.data['status'] == 'completed', wire(recovered) @@ -207,7 +330,9 @@ async def test_expired_unaccepted_custodial_challenges_are_reclaimed(installed): async with app.metadata.transaction(write=False) as tx: assert tx.one('SELECT COUNT(*) FROM custodial_upgrades WHERE subject=?', (subject,))[0] == 0 assert (await tx.subject(subject)).kind == 'custodial' - assert tx.one('SELECT status FROM custodial_vault WHERE subject=?', (subject,))[0] == 'active' + assert ( + tx.one('SELECT status FROM custodial_vault WHERE subject=?', (subject,))[0] == 'active' + ) @pytest.mark.asyncio @@ -216,15 +341,22 @@ async def test_todo_sweep_has_bounded_content_reads_and_fair_cursor(installed, m key, subject, _ = await register(app, 'security-todos') monkeypatch.setattr(maintenance, 'TODO_DELIVERY_BATCH_SIZE', 3, raising=False) for i in range(7): - created = await call(app, 'identity.todo_put', {'name': f'item-{i}', 'title': f'Todo {i}', - 'due_at': wire(NOW - timedelta(hours=1))}, key=key, subject=subject) + created = await call( + app, + 'identity.todo_put', + {'name': f'item-{i}', 'title': f'Todo {i}', 'due_at': wire(NOW - timedelta(hours=1))}, + key=key, + subject=subject, + ) assert created.status == 'ok', wire(created) reads = 0 original = app.contents.read_bytes + async def observed_read(*args, **kwargs): nonlocal reads reads += 1 return await original(*args, **kwargs) + monkeypatch.setattr(app.contents, 'read_bytes', observed_read) for _ in range(3): reads = 0 @@ -240,17 +372,29 @@ async def test_expired_transfer_releases_only_its_pins(installed, sealed): app, _ = installed key, subject, _ = await register(app, 'security-upload') data = b'abandoned or published upload' - opened = await call(app, 'transfer.open', {'direction': 'upload', 'size': len(data)}, key=key, subject=subject) + opened = await call( + app, 'transfer.open', {'direction': 'upload', 'size': len(data)}, key=key, subject=subject + ) assert opened.status == 'ok', wire(opened) tid = opened.data['transfer_id'] - uploaded = await call(app, 'transfer.part_put', {'transfer_id': tid, 'offset': 0, - 'data': b64(data), 'digest': digest(data)}, key=key, subject=subject) + uploaded = await call( + app, + 'transfer.part_put', + {'transfer_id': tid, 'offset': 0, 'data': b64(data), 'digest': digest(data)}, + key=key, + subject=subject, + ) assert uploaded.status == 'ok', wire(uploaded) blob = decode(BlobRef, uploaded.data['chunk']['content']) assert await app.contents.pinned(blob, tid + ':0') if sealed: - result = await call(app, 'transfer.seal', {'transfer_id': tid, 'final_size': len(data), - 'final_digest': digest(data)}, key=key, subject=subject) + result = await call( + app, + 'transfer.seal', + {'transfer_id': tid, 'final_size': len(data), 'final_digest': digest(data)}, + key=key, + subject=subject, + ) assert result.status == 'ok', wire(result) async with app.metadata.transaction(write=False) as tx: revision = await tx.revision(result.resources[0]) diff --git a/tests/test_security_rollback_fences.py b/tests/test_security_rollback_fences.py index 645957a6..ff05008b 100644 --- a/tests/test_security_rollback_fences.py +++ b/tests/test_security_rollback_fences.py @@ -1,4 +1,5 @@ """Rollback compensation remains serialized with real independent writers.""" + import asyncio import sqlite3 @@ -14,8 +15,11 @@ def stores(request, tmp_path): if request.param == 'postgres': dsn = request.getfixturevalue('pg_dsn') - return (PostgresMetadataStore(dsn), - PostgresMetadataStore(dsn, initialize=False), request.param) + return ( + PostgresMetadataStore(dsn), + PostgresMetadataStore(dsn, initialize=False), + request.param, + ) path = tmp_path / 'fence.db' return SqliteMetadataStore(path), SqliteMetadataStore(path), request.param @@ -41,7 +45,7 @@ async def abort(): if sql_failure: # Exercise automatic SQLite transaction abort as well as PG's # INERROR state, not only an application exception before ROLLBACK. - sql = ('INSERT OR ROLLBACK' if backend == 'sqlite' else 'INSERT') + sql = 'INSERT OR ROLLBACK' if backend == 'sqlite' else 'INSERT' tx.execute(sql + ' INTO schema_version VALUES (1)', write=True) raise ValueError('original failure') @@ -62,9 +66,13 @@ async def compete(): remained_blocked = not entered.is_set() finally: release.set() - results = await asyncio.wait_for(asyncio.gather( - *([first_task, second_task] if second_task else [first_task]), - return_exceptions=True), 5) + results = await asyncio.wait_for( + asyncio.gather( + *([first_task, second_task] if second_task else [first_task]), + return_exceptions=True, + ), + 5, + ) if sql_failure: assert isinstance(results[0], Failure) and results[0].code == 'constraint_conflict', results expected_cause = sqlite3.IntegrityError if backend == 'sqlite' else psycopg.IntegrityError @@ -80,7 +88,9 @@ async def compete(): @pytest.mark.parametrize('initial_cancel', [False, True]) -async def test_repeated_cancellation_cannot_interrupt_compensation(stores, tmp_path, initial_cancel): +async def test_repeated_cancellation_cannot_interrupt_compensation( + stores, tmp_path, initial_cancel +): store, other, _ = stores started, release = asyncio.Event(), asyncio.Event() pin = tmp_path / 'temporary-pin' @@ -106,7 +116,7 @@ async def abort(): await asyncio.sleep(0) finally: release.set() - result, = await asyncio.wait_for(asyncio.gather(task, return_exceptions=True), 5) + (result,) = await asyncio.wait_for(asyncio.gather(task, return_exceptions=True), 5) if initial_cancel: assert isinstance(result, asyncio.CancelledError) else: @@ -141,8 +151,9 @@ async def wait(): waiter.cancel() finally: release.set() - results = await asyncio.wait_for(asyncio.gather( - *([holder, waiter] if waiter else [holder]), return_exceptions=True), 5) + results = await asyncio.wait_for( + asyncio.gather(*([holder, waiter] if waiter else [holder]), return_exceptions=True), 5 + ) assert results[0] is None assert isinstance(results[1], asyncio.CancelledError), results async with second.transaction(write=True) as tx: @@ -197,7 +208,7 @@ async def upload(): waited_for_io = not task.done() finally: release.set() - result, = await asyncio.wait_for(asyncio.gather(task, return_exceptions=True), 5) + (result,) = await asyncio.wait_for(asyncio.gather(task, return_exceptions=True), 5) assert isinstance(result, asyncio.CancelledError), result assert waited_for_io, 'rollback overtook a still-running Git pin write' assert not await contents.pinned(blob, 'cancelled-upload') diff --git a/tests/test_security_storage_rollback.py b/tests/test_security_storage_rollback.py index 7b35dac6..3428e8bf 100644 --- a/tests/test_security_storage_rollback.py +++ b/tests/test_security_storage_rollback.py @@ -1,4 +1,5 @@ """Compensation retains savepoint ownership and never hides cancellation.""" + import asyncio import pytest @@ -16,8 +17,10 @@ async def test_nested_rollback_compensates_only_its_scope(installed, tmp_path, b app, _ = installed store = app.metadata if backend == 'postgres' else SqliteMetadataStore(tmp_path / 'nested.db') effects = [] + async def record(value): effects.append(value) + async with store.transaction(write=True) as outer: outer.on_rollback(lambda: record('outer')) with pytest.raises(ValueError, match='savepoint'): @@ -42,15 +45,20 @@ async def record(value): @pytest.mark.asyncio @pytest.mark.parametrize('backend', ['postgres', 'sqlite']) -async def test_compensation_failure_preserves_original_and_runs_remaining(installed, tmp_path, backend): +async def test_compensation_failure_preserves_original_and_runs_remaining( + installed, tmp_path, backend +): app, _ = installed store = app.metadata if backend == 'postgres' else SqliteMetadataStore(tmp_path / 'fail.db') effects = [] + async def record(value): effects.append(value) + async def fail(): effects.append('failure') raise OSError('injected') + original = ValueError('primary') with pytest.raises(ValueError) as captured: async with store.transaction(write=True) as tx: @@ -70,8 +78,10 @@ async def test_task_cancellation_rolls_back_sql_and_external_effect(installed, t store = app.metadata if backend == 'postgres' else SqliteMetadataStore(tmp_path / 'cancel.db') entered = asyncio.Event() external = tmp_path / 'external' + async def compensate(): external.unlink() + async def work(): async with store.transaction(write=True) as tx: tx.set_setting('cancelled', True) @@ -79,6 +89,7 @@ async def work(): external.write_text('uncommitted') entered.set() await asyncio.Event().wait() + task = asyncio.create_task(work()) await entered.wait() assert external.exists() @@ -97,19 +108,27 @@ async def test_failed_transfer_cleanup_restores_pin_and_metadata(installed, monk opened = await call(app, 'transfer.open', {'direction': 'upload'}, key=key, subject=subject) assert opened.status == 'ok', wire(opened) tid = opened.data['transfer_id'] - result = await call(app, 'transfer.part_put', {'transfer_id': tid, 'offset': 0, - 'data': b64(b'abcd'), 'digest': digest(b'abcd')}, key=key, subject=subject) + result = await call( + app, + 'transfer.part_put', + {'transfer_id': tid, 'offset': 0, 'data': b64(b'abcd'), 'digest': digest(b'abcd')}, + key=key, + subject=subject, + ) assert result.status == 'ok', wire(result) blob = decode(BlobRef, result.data['chunk']['content']) from datetime import timedelta from test_security_patch_regressions import set_clock from test_service import NOW + set_clock(app, NOW + timedelta(seconds=app.settings.transfer_ttl + 1)) unpin = app.contents.unpin + async def fail_after_unpin(*args): await unpin(*args) raise OSError('injected after external change') + monkeypatch.setattr(app.contents, 'unpin', fail_after_unpin) with pytest.raises(OSError, match='injected'): await maintenance.run_maintenance(app, 'cleanup_expired', scheduled=True) @@ -127,13 +146,17 @@ async def test_scheduled_collection_is_rate_limited_and_respects_cleanup_pause(i app, _ = installed async with app.metadata.transaction(write=True) as tx: tx.set_setting('runtime_config', {'cleanup_enabled': False}) - assert await maintenance.run_maintenance(app, 'collect_garbage', scheduled=True) == {'skipped': 'cleanup_disabled'} + assert await maintenance.run_maintenance(app, 'collect_garbage', scheduled=True) == { + 'skipped': 'cleanup_disabled' + } async with app.metadata.transaction(write=True) as tx: assert tx.setting('garbage_collection_last') is None tx.set_setting('runtime_config', {'cleanup_enabled': True}) result = await maintenance.run_maintenance(app, 'collect_garbage', scheduled=True) assert result['unreferenced_contents_removed'] == 0 - assert await maintenance.run_maintenance(app, 'collect_garbage', scheduled=True) == {'skipped': 'not_due'} + assert await maintenance.run_maintenance(app, 'collect_garbage', scheduled=True) == { + 'skipped': 'not_due' + } @pytest.mark.asyncio @@ -142,23 +165,42 @@ async def test_purged_sealed_output_does_not_break_expiry_or_collection(installe from test_security_patch_regressions import set_clock from test_service import NOW + app, _ = installed key, subject, _ = await register(app, 'security-purged-upload') data = b'unreferenced after explicit purge' - opened = await call(app, 'transfer.open', {'direction': 'upload', 'size': len(data)}, key=key, subject=subject) + opened = await call( + app, 'transfer.open', {'direction': 'upload', 'size': len(data)}, key=key, subject=subject + ) assert opened.status == 'ok', wire(opened) tid = opened.data['transfer_id'] - uploaded = await call(app, 'transfer.part_put', {'transfer_id': tid, 'offset': 0, - 'data': b64(data), 'digest': digest(data)}, key=key, subject=subject) + uploaded = await call( + app, + 'transfer.part_put', + {'transfer_id': tid, 'offset': 0, 'data': b64(data), 'digest': digest(data)}, + key=key, + subject=subject, + ) assert uploaded.status == 'ok', wire(uploaded) - sealed = await call(app, 'transfer.seal', {'transfer_id': tid, 'final_size': len(data), - 'final_digest': digest(data)}, key=key, subject=subject) + sealed = await call( + app, + 'transfer.seal', + {'transfer_id': tid, 'final_size': len(data), 'final_digest': digest(data)}, + key=key, + subject=subject, + ) assert sealed.status == 'ok', wire(sealed) async with app.metadata.transaction(write=True) as tx: revision = await tx.revision(sealed.resources[0]) resource = await tx.resource(sealed.resources[0].id) - await maintenance.purge_revisions(app, tx, resource, actor=subject, - request_id='test-explicit-purge', reason='retention_expired') + await maintenance.purge_revisions( + app, + tx, + resource, + actor=subject, + request_id='test-explicit-purge', + reason='retention_expired', + ) assert await app.contents.pinned(revision.content, tid + ':sealed') set_clock(app, NOW + timedelta(seconds=app.settings.transfer_ttl + 1)) await maintenance.run_maintenance(app, 'cleanup_expired', scheduled=True) diff --git a/tests/test_security_topic_events.py b/tests/test_security_topic_events.py index acdc34ca..6d812ab1 100644 --- a/tests/test_security_topic_events.py +++ b/tests/test_security_topic_events.py @@ -1,4 +1,5 @@ """Topic history work is bounded by the requested page, not global activity.""" + import pytest from test_service import call, register @@ -10,26 +11,43 @@ async def test_topic_history_decodes_only_one_page(installed, monkeypatch): app, _ = installed key, subject, _ = await register(app, 'security-events') - made = await call(app, 'content.topic_create', {'parent': '/main', 'name': 'indexed-events'}, - key=key, subject=subject) + made = await call( + app, + 'content.topic_create', + {'parent': '/main', 'name': 'indexed-events'}, + key=key, + subject=subject, + ) assert made.status == 'ok', wire(made) topic = made.resources[0].id for policy in ('approval', 'open', 'invite', 'closed'): - result = await call(app, 'content.topic_policy_set', {'id': topic, 'membership_policy': policy}, - key=key, subject=subject) + result = await call( + app, + 'content.topic_policy_set', + {'id': topic, 'membership_policy': policy}, + key=key, + subject=subject, + ) assert result.status == 'ok', wire(result) for i in range(12): - posted = await call(app, 'content.post_create', {'parent': '/main', 'body': f'Unrelated {i}'}, - key=key, subject=subject) + posted = await call( + app, + 'content.post_create', + {'parent': '/main', 'body': f'Unrelated {i}'}, + key=key, + subject=subject, + ) assert posted.status == 'ok', wire(posted) decoded = 0 original = content.loads + def observed(raw): nonlocal decoded value = original(raw) if isinstance(value, dict) and 'request_id' in value and 'type' in value: decoded += 1 return value + monkeypatch.setattr(content, 'loads', observed) cursor = None seen = [] @@ -53,31 +71,52 @@ async def test_backfill_is_atomic_and_not_repeated(installed, tmp_path, backend) from msg.core.codec import canonical from msg.storage.sqlite import SqliteMetadataStore from msg.storage.topic_event_migration import EVENT_TYPES, MIGRATION_KEY, migrate_topic_events + app, _ = installed store = app.metadata if backend == 'postgres' else SqliteMetadataStore(tmp_path / 'events.db') assert set(EVENT_TYPES) == set(content.TOPIC_EVENT_CODES) async with store.transaction(write=True) as tx: tx.execute('DELETE FROM settings WHERE key=?', (MIGRATION_KEY,), write=True) - for identifier, kind, topic in [('historical', 'topic.create', 't_index'), - ('unrelated', 'content.post_create', 't_index'), - ('missing-topic', 'topic.create', None)]: + for identifier, kind, topic in [ + ('historical', 'topic.create', 't_index'), + ('unrelated', 'content.post_create', 't_index'), + ('missing-topic', 'topic.create', None), + ]: body = canonical({'type': kind, 'data': {'topic_id': topic}}).decode() tx.execute('INSERT INTO events (id,body) VALUES (?,?)', (identifier, body), write=True) # Constructor migrations run on the already-held physical schema lock. migrate_topic_events(tx._connection, postgres=backend == 'postgres') - assert tx.one('SELECT COUNT(*) FROM topic_event_projection WHERE topic=?', ('t_index',))[0] == 1 + assert ( + tx.one('SELECT COUNT(*) FROM topic_event_projection WHERE topic=?', ('t_index',))[0] + == 1 + ) assert tx.setting(MIGRATION_KEY) is True - tx.execute('INSERT INTO events (id,body) VALUES (?,?)', ('not-a-normal-write', - canonical({'type': 'topic.create', 'data': {'topic_id': 't_index'}}).decode()), write=True) + tx.execute( + 'INSERT INTO events (id,body) VALUES (?,?)', + ( + 'not-a-normal-write', + canonical({'type': 'topic.create', 'data': {'topic_id': 't_index'}}).decode(), + ), + write=True, + ) migrate_topic_events(tx._connection, postgres=backend == 'postgres') - assert tx.one('SELECT COUNT(*) FROM topic_event_projection WHERE topic=?', ('t_index',))[0] == 1 + assert ( + tx.one('SELECT COUNT(*) FROM topic_event_projection WHERE topic=?', ('t_index',))[0] + == 1 + ) # Both range shapes used by the rotating todo worker have supporting indexes. if backend == 'sqlite': - plan = tx.rows("EXPLAIN QUERY PLAN SELECT id,body FROM resources " - "WHERE type='todo' AND state='active' AND id>? ORDER BY id LIMIT ?", ('', 3)) + plan = tx.rows( + 'EXPLAIN QUERY PLAN SELECT id,body FROM resources ' + "WHERE type='todo' AND state='active' AND id>? ORDER BY id LIMIT ?", + ('', 3), + ) assert 'resources_type_state' in str(plan) else: tx.execute('SET LOCAL enable_seqscan=off') - plan = tx.rows("EXPLAIN SELECT id,body FROM resources WHERE type='todo' " - "AND state='active' AND id>? ORDER BY id LIMIT ?", ('', 3)) + plan = tx.rows( + "EXPLAIN SELECT id,body FROM resources WHERE type='todo' " + "AND state='active' AND id>? ORDER BY id LIMIT ?", + ('', 3), + ) assert 'resources_type_state' in str(plan) diff --git a/tests/test_security_transfer_rollback.py b/tests/test_security_transfer_rollback.py index b94832c8..81db17b7 100644 --- a/tests/test_security_transfer_rollback.py +++ b/tests/test_security_transfer_rollback.py @@ -1,4 +1,5 @@ """Authenticated upload failures must not leave permanent chunk/output pins.""" + import asyncio import pytest @@ -11,42 +12,59 @@ async def pins(app): - binary = sorted(str(p.relative_to(app.contents.path)) - for p in (app.contents.path / 'pins').glob('*/*') if p.is_file()) - git = await asyncio.to_thread(app.contents._run, 'for-each-ref', - '--format=%(refname)', 'refs/pins') + binary = sorted( + str(p.relative_to(app.contents.path)) + for p in (app.contents.path / 'pins').glob('*/*') + if p.is_file() + ) + git = await asyncio.to_thread( + app.contents._run, 'for-each-ref', '--format=%(refname)', 'refs/pins' + ) return set(binary) | set(git.splitlines()) @pytest.mark.parametrize('media_type', ['application/octet-stream', 'text/plain']) @pytest.mark.parametrize('operation', ['transfer.part_put', 'transfer.seal']) @pytest.mark.parametrize('failure', ['projection', 'partial_pin']) -async def test_failed_upload_removes_only_its_new_pins(installed, monkeypatch, operation, failure, media_type): +async def test_failed_upload_removes_only_its_new_pins( + installed, monkeypatch, operation, failure, media_type +): app, _ = installed key, subject, _ = await register(app, 'rollback-transfer') # The sealed digest differs from either chunk, so no shared chunk can hide a leak. data = b'first chunk' + b'second chunk' - opened = await call(app, 'transfer.open', {'direction': 'upload', 'size': len(data), 'media_type': media_type}, - key=key, subject=subject) + opened = await call( + app, + 'transfer.open', + {'direction': 'upload', 'size': len(data), 'media_type': media_type}, + key=key, + subject=subject, + ) assert opened.status == 'ok', wire(opened) tid = opened.data['transfer_id'] args = {'transfer_id': tid, 'offset': 0, 'data': b64(data), 'digest': digest(data)} lease = tid + ':0' if operation == 'transfer.seal': for offset, chunk in ((0, b'first chunk'), (11, b'second chunk')): - part = await call(app, 'transfer.part_put', {'transfer_id': tid, 'offset': offset, - 'data': b64(chunk), 'digest': digest(chunk)}, - key=key, subject=subject) + part = await call( + app, + 'transfer.part_put', + {'transfer_id': tid, 'offset': offset, 'data': b64(chunk), 'digest': digest(chunk)}, + key=key, + subject=subject, + ) assert part.status == 'ok', wire(part) args = {'transfer_id': tid, 'final_size': len(data), 'final_digest': digest(data)} lease = tid + ':sealed' before = await pins(app) async with app.metadata.transaction(write=False) as tx: transfer_before = await tx.transfer(tid) - counts_before = tuple(tx.one(f'SELECT COUNT(*) FROM {table}')[0] - for table in ('resources', 'revisions')) - chunks_before = tx.rows('SELECT offset,body FROM chunks WHERE transfer_id=? ORDER BY offset', - (tid,)) + counts_before = tuple( + tx.one(f'SELECT COUNT(*) FROM {table}')[0] for table in ('resources', 'revisions') + ) + chunks_before = tx.rows( + 'SELECT offset,body FROM chunks WHERE transfer_id=? ORDER BY offset', (tid,) + ) original_pin = app.contents.pin async def partial_pin(blob, lease_id): @@ -58,11 +76,20 @@ async def partial_pin(blob, lease_id): if failure == 'partial_pin': patch.setattr(app.contents, 'pin', partial_pin) for n in range(2): - result = await fresh_call(app, operation, args, key=key, subject=subject, - rid=f'failed-upload-{n}', - return_fields=('invalid_field',) if failure == 'projection' else ()) - projection_error = ('projection_unavailable' if operation == 'transfer.part_put' - else 'unknown_projection_field') + result = await fresh_call( + app, + operation, + args, + key=key, + subject=subject, + rid=f'failed-upload-{n}', + return_fields=('invalid_field',) if failure == 'projection' else (), + ) + projection_error = ( + 'projection_unavailable' + if operation == 'transfer.part_put' + else 'unknown_projection_field' + ) error = projection_error if failure == 'projection' else 'injected_pin_failure' assert result.status == 'error' and result.error.code == error, wire(result) after = await pins(app) @@ -70,11 +97,22 @@ async def partial_pin(blob, lease_id): assert not any(changes.values()), changes async with app.metadata.transaction(write=False) as tx: assert await tx.transfer(tid) == transfer_before - assert tuple(tx.one(f'SELECT COUNT(*) FROM {table}')[0] - for table in ('resources', 'revisions')) == counts_before - assert tx.rows('SELECT offset,body FROM chunks WHERE transfer_id=? ORDER BY offset', - (tid,)) == chunks_before - accepted = await fresh_call(app, operation, args, key=key, subject=subject, rid='accepted-upload') + assert ( + tuple( + tx.one(f'SELECT COUNT(*) FROM {table}')[0] + for table in ('resources', 'revisions') + ) + == counts_before + ) + assert ( + tx.rows( + 'SELECT offset,body FROM chunks WHERE transfer_id=? ORDER BY offset', (tid,) + ) + == chunks_before + ) + accepted = await fresh_call( + app, operation, args, key=key, subject=subject, rid='accepted-upload' + ) assert accepted.status == 'ok', wire(accepted) if operation == 'transfer.part_put': blob = decode(BlobRef, accepted.data['chunk']['content']) diff --git a/tests/test_service.py b/tests/test_service.py index 3dc43e30..d1c775e8 100644 --- a/tests/test_service.py +++ b/tests/test_service.py @@ -1,56 +1,97 @@ """End-to-end requests go through real signature verification and SQLite commits.""" -from dataclasses import replace -from datetime import UTC, datetime, timedelta + import hashlib import os +from datetime import UTC, datetime, timedelta from uuid import uuid4 import pytest -from msg.application import Application -from msg.admin.root import _provision, _approve_csr -from msg.config import write_example from msg.constants import ROOT_SUBJECT from msg.core.codec import b64, canonical, unb64, wire -from msg.core.requests import request_for, receipt_bytes -from msg.security.crypto import Ed25519Signer, subject_id, verify +from msg.core.requests import receipt_bytes, request_for from msg.security.age_keys import generate_age_key +from msg.security.crypto import Ed25519Signer, subject_id, verify NOW = datetime(2026, 9, 27, tzinfo=UTC) -def temporary_v3_args(*, nonce=None, recovery_secret=None, request_id=None, - signer=None, recipient=None): - nonce=nonce or b64(os.urandom(32)) - recovery_secret=recovery_secret or b64(os.urandom(32)) - request_id=request_id or uuid4().hex - signer=signer or Ed25519Signer.generate() - recipient=recipient or generate_age_key()[1] - subject='u_tmp_'+hashlib.sha256(unb64(nonce)).hexdigest()[:32] - proof=signer.sign(canonical({'subject_id':subject,'nonce':nonce, - 'public_key':b64(signer.public_key),'encryption_recipient':recipient, - 'request_id':request_id}),purpose='temporary-key-possession-v1') - return ({'nonce':nonce,'recovery_secret':recovery_secret, - 'public_key':b64(signer.public_key),'encryption_recipient':recipient, - 'possession_proof':wire(proof)},request_id,signer,recipient) - - -async def call(app, op, args, *, key=None, subject=None, certs=(), expected=(), rid=None, token=None, - contract_version=1): - packet = request_for(op, args, app.settings.service_url, - signer=key, subject=subject, certificates=certs, expected=expected, - request_id=rid, token=token, expires_at=NOW+timedelta(seconds=120), - contract_version=contract_version) +def temporary_v3_args( + *, nonce=None, recovery_secret=None, request_id=None, signer=None, recipient=None +): + nonce = nonce or b64(os.urandom(32)) + recovery_secret = recovery_secret or b64(os.urandom(32)) + request_id = request_id or uuid4().hex + signer = signer or Ed25519Signer.generate() + recipient = recipient or generate_age_key()[1] + subject = 'u_tmp_' + hashlib.sha256(unb64(nonce)).hexdigest()[:32] + proof = signer.sign( + canonical({ + 'subject_id': subject, + 'nonce': nonce, + 'public_key': b64(signer.public_key), + 'encryption_recipient': recipient, + 'request_id': request_id, + }), + purpose='temporary-key-possession-v1', + ) + return ( + { + 'nonce': nonce, + 'recovery_secret': recovery_secret, + 'public_key': b64(signer.public_key), + 'encryption_recipient': recipient, + 'possession_proof': wire(proof), + }, + request_id, + signer, + recipient, + ) + + +async def call( + app, + op, + args, + *, + key=None, + subject=None, + certs=(), + expected=(), + rid=None, + token=None, + contract_version=1, +): + packet = request_for( + op, + args, + app.settings.service_url, + signer=key, + subject=subject, + certificates=certs, + expected=expected, + request_id=rid, + token=token, + expires_at=NOW + timedelta(seconds=120), + contract_version=contract_version, + ) return await app.executor.execute(packet) async def register(app, handle): from msg.security.age_keys import generate_age_key + key = Ed25519Signer.generate() uid = subject_id(key.public_key) _, recipient = generate_age_key() - result = await call(app, 'identity.register', {'handle':handle,'public_key':b64(key.public_key), - 'encryption_recipient':recipient}, key=key, subject=uid, contract_version=2) + result = await call( + app, + 'identity.register', + {'handle': handle, 'public_key': b64(key.public_key), 'encryption_recipient': recipient}, + key=key, + subject=uid, + contract_version=2, + ) assert result.status == 'ok', wire(result) return key, uid, result.data['certificate_id'] @@ -59,25 +100,33 @@ async def register(app, handle): async def test_service_register_post_read_reply_receipt(installed): app, root = installed key, uid, cert = await register(app, 'alice') + async def invoke(op, args, **kw): return await call(app, op, args, key=key, subject=uid, certs=(cert,), **kw) - result = await invoke('content.post_create', {'parent':'/main','body':'A short message.'}, rid='post-once') + + result = await invoke( + 'content.post_create', {'parent': '/main', 'body': 'A short message.'}, rid='post-once' + ) assert result.status == 'ok', wire(result) verify(app.receipt_signer.public_key, receipt_bytes(result), result.receipt, purpose='receipt') - repeated = await invoke('content.post_create', {'parent':'/main','body':'A short message.'}, rid='post-once') + repeated = await invoke( + 'content.post_create', {'parent': '/main', 'body': 'A short message.'}, rid='post-once' + ) assert repeated.status == 'ok' and repeated.replayed - conflict = await invoke('content.post_create', {'parent':'/main','body':'Different.'}, rid='post-once') + conflict = await invoke( + 'content.post_create', {'parent': '/main', 'body': 'Different.'}, rid='post-once' + ) assert conflict.error.code == 'idempotency_conflict' rid = result.resources[0].id - read = await call(app, 'discovery.get', {'id':rid}) + read = await call(app, 'discovery.get', {'id': rid}) assert read.status == 'ok', wire(read) assert 'A short message.' in str(read.data) - reply = await invoke('discussion.reply', {'target':{'id':rid},'body':'Received.'}) + reply = await invoke('discussion.reply', {'target': {'id': rid}, 'body': 'Received.'}) assert reply.status == 'ok', wire(reply) async with app.metadata.transaction(write=False) as tx: assert (await tx.resource(reply.resources[0].id)).type == 'post' before = tx.one('SELECT COUNT(*) FROM events')[0] - await call(app, 'discovery.get', {'id':rid}) + await call(app, 'discovery.get', {'id': rid}) async with app.metadata.transaction(write=False) as tx: assert tx.one('SELECT COUNT(*) FROM events')[0] == before @@ -85,7 +134,7 @@ async def invoke(op, args, **kw): @pytest.mark.asyncio async def test_root_network_forbidden_even_cli_source(installed): app, root = installed - result = await call(app, 'discovery.get', {'id':'/private'}, key=root, subject=ROOT_SUBJECT) + result = await call(app, 'discovery.get', {'id': '/private'}, key=root, subject=ROOT_SUBJECT) assert result.status == 'error' and result.error.code == 'local_only', wire(result) assert not hasattr(app, 'root_signer') @@ -94,45 +143,88 @@ async def test_root_network_forbidden_even_cli_source(installed): async def test_default_deny_and_certgate(installed): app, _ = installed key, uid, cert = await register(app, 'bob') - for path in ('/private','/admins','/tools/curl'): - result = await call(app,'discovery.get',{'id':path},key=key,subject=uid,certs=(cert,)) - assert result.status == 'error', (path,wire(result)) - result = await call(app,'content.post_create',{'parent':'/certified','body':'gate'},key=key,subject=uid,certs=(cert,)) + for path in ('/private', '/admins', '/tools/curl'): + result = await call(app, 'discovery.get', {'id': path}, key=key, subject=uid, certs=(cert,)) + assert result.status == 'error', (path, wire(result)) + result = await call( + app, + 'content.post_create', + {'parent': '/certified', 'body': 'gate'}, + key=key, + subject=uid, + certs=(cert,), + ) assert result.error.code == 'certificate_gate', wire(result) @pytest.mark.asyncio async def test_temporary_rotation_and_upgrade_preserve_subject(installed): app, _ = installed - temporary_args,temporary_rid,new_key,recipient=temporary_v3_args() - result = await call(app,'identity.temporary',temporary_args, - rid=temporary_rid,contract_version=3) + temporary_args, temporary_rid, new_key, recipient = temporary_v3_args() + result = await call( + app, 'identity.temporary', temporary_args, rid=temporary_rid, contract_version=3 + ) assert result.status == 'ok', wire(result) uid = result.data['subject_id'] from msg.core.codec import unb64 + token = (result.data['credential_id'], unb64(result.data['token'])) - post = await call(app,'content.post_create',{'parent':'/tmp','body':'temporary'},subject=uid,token=token) + post = await call( + app, + 'content.post_create', + {'parent': '/tmp', 'body': 'temporary'}, + subject=uid, + token=token, + ) assert post.status == 'ok', wire(post) - forbidden = await call(app,'content.chmod',{'id':post.resources[0].id,'mode':'0777'},subject=uid,token=token, - expected=((post.resources[0].id,post.data['generation']),)) + forbidden = await call( + app, + 'content.chmod', + {'id': post.resources[0].id, 'mode': '0777'}, + subject=uid, + token=token, + expected=((post.resources[0].id, post.data['generation']),), + ) assert forbidden.error.code == 'credential_ceiling' - args = {'subject_id':uid,'public_key':b64(new_key.public_key),'handle':'promoted', - 'encryption_recipient':recipient} - upgrade = await call(app,'identity.upgrade',{'handle':'promoted','public_key':args['public_key'], - 'encryption_recipient':recipient, - 'possession_proof':wire(new_key.sign(canonical(args),purpose='upgrade'))}, - subject=uid,token=token,contract_version=2) + args = { + 'subject_id': uid, + 'public_key': b64(new_key.public_key), + 'handle': 'promoted', + 'encryption_recipient': recipient, + } + upgrade = await call( + app, + 'identity.upgrade', + { + 'handle': 'promoted', + 'public_key': args['public_key'], + 'encryption_recipient': recipient, + 'possession_proof': wire(new_key.sign(canonical(args), purpose='upgrade')), + }, + subject=uid, + token=token, + contract_version=2, + ) assert upgrade.status == 'ok' and upgrade.data['subject_id'] == uid, wire(upgrade) async with app.metadata.transaction(write=False) as tx: assert (await tx.resource(post.resources[0].id)).owner == uid + @pytest.mark.asyncio async def test_requested_write_projection_is_returned_without_readback(installed): - app,_=installed - key,uid,_=await register(app,'projection-author') + app, _ = installed + key, uid, _ = await register(app, 'projection-author') from msg.core.requests import request_for - packet=request_for('content.post_create',{'parent':'/main','body':'project me'},app.settings.service_url, - subject=uid,signer=key,expires_at=NOW+timedelta(seconds=120),return_fields=('id','revision','size')) - result=await app.executor.execute(packet) - assert result.status=='ok',result - assert result.data['projection'][0]['size']==10 + + packet = request_for( + 'content.post_create', + {'parent': '/main', 'body': 'project me'}, + app.settings.service_url, + subject=uid, + signer=key, + expires_at=NOW + timedelta(seconds=120), + return_fields=('id', 'revision', 'size'), + ) + result = await app.executor.execute(packet) + assert result.status == 'ok', result + assert result.data['projection'][0]['size'] == 10 diff --git a/tests/test_session_boundary.py b/tests/test_session_boundary.py index e6230fec..9ee1b9b8 100644 --- a/tests/test_session_boundary.py +++ b/tests/test_session_boundary.py @@ -1,32 +1,39 @@ """The same bounded query and domain contract on real SQLite and PostgreSQL.""" + import asyncio import importlib import importlib.util -from pathlib import Path import subprocess import sys +from pathlib import Path from typing import get_type_hints import pytest from msg.core.codec import canonical, digest -from msg.core.errors import Failure from msg.core.contracts import MetadataSession +from msg.core.errors import Failure from msg.market.policy import DEFAULT_POLICY, load_policy -from msg.storage.postgres import PostgresMetadataStore, PostgresSession, _SCHEMA as POSTGRES_SCHEMA +from msg.storage.postgres import _SCHEMA as POSTGRES_SCHEMA, PostgresMetadataStore, PostgresSession from msg.storage.sqlite import SqliteMetadataStore, SqliteSession @pytest.fixture(params=['sqlite', 'postgres']) async def store(request, tmp_path): - value = (PostgresMetadataStore(request.getfixturevalue('pg_dsn')) - if request.param == 'postgres' else SqliteMetadataStore(tmp_path / 'metadata.db')) + value = ( + PostgresMetadataStore(request.getfixturevalue('pg_dsn')) + if request.param == 'postgres' + else SqliteMetadataStore(tmp_path / 'metadata.db') + ) if request.param == 'sqlite': # SQLite is the small test backend, not a complete production bootstrap. # Install the real policy table DDL for this real consumer contract test; # do not pretend the test is a production schema migration or parity claim. - statement = next(sql for sql in POSTGRES_SCHEMA.split(';') - if sql.strip().startswith('CREATE TABLE IF NOT EXISTS arbitration_policies (')) + statement = next( + sql + for sql in POSTGRES_SCHEMA.split(';') + if sql.strip().startswith('CREATE TABLE IF NOT EXISTS arbitration_policies (') + ) async with value.transaction(write=True) as tx: tx.execute(statement, write=True) yield value @@ -63,8 +70,11 @@ def test_declared_session_covers_real_query_consumer_without_commit_rights(): assert hasattr(MetadataSession, name), name assert hasattr(query.QuerySession, name), name common = importlib.import_module('msg.storage.session').RelationalSession - domain_methods = {name for name, member in vars(common).items() - if callable(member) and not name.startswith('_')} + domain_methods = { + name + for name, member in vars(common).items() + if callable(member) and not name.startswith('_') + } # These two methods are storage lifecycle mechanics, not consumer ports. domain_methods -= {'check', 'run_rollback_effects'} assert all(hasattr(MetadataSession, name) for name in domain_methods) @@ -90,12 +100,15 @@ async def test_query_result_is_a_guarded_view_not_a_driver_cursor(store): async def test_query_result_and_session_refuse_cross_task_and_read_only_writes(store): async with store.transaction(write=False) as tx: result = tx.execute('SELECT 1') + async def misuse_result(): with pytest.raises(Failure, match='transaction_cross_task'): result.fetchone() + async def misuse_session(): with pytest.raises(Failure, match='transaction_cross_task'): tx.one('SELECT 1') + await asyncio.gather(misuse_result(), misuse_session()) with pytest.raises(Failure, match='read_only_transaction'): tx.set_setting('forbidden', True) @@ -109,11 +122,17 @@ async def test_declared_policy_consumer_and_nested_rollback_work_on_both_backend assert isinstance(tx, query.QuerySession) assert load_policy(tx, DEFAULT_POLICY['id']) == DEFAULT_POLICY policy = {**DEFAULT_POLICY, 'id': 'query-contract-policy'} - tx.execute('INSERT INTO arbitration_policies VALUES (?,?,?)', - (policy['id'], canonical(policy).decode(), digest(policy)), write=True) + tx.execute( + 'INSERT INTO arbitration_policies VALUES (?,?,?)', + (policy['id'], canonical(policy).decode(), digest(policy)), + write=True, + ) assert load_policy(tx, policy['id']) == policy - tx.execute('INSERT INTO arbitration_policies VALUES (?,?,?)', - ('corrupt-contract-policy', canonical(policy).decode(), 'wrong-digest'), write=True) + tx.execute( + 'INSERT INTO arbitration_policies VALUES (?,?,?)', + ('corrupt-contract-policy', canonical(policy).decode(), 'wrong-digest'), + write=True, + ) with pytest.raises(Failure, match='arbitration_policy_corrupt'): load_policy(tx, 'corrupt-contract-policy') tx.set_setting('outer', 'keep') diff --git a/tests/test_share_grants.py b/tests/test_share_grants.py index e2dccde2..8581f93b 100644 --- a/tests/test_share_grants.py +++ b/tests/test_share_grants.py @@ -1,10 +1,11 @@ """Direct sharing is a current read source for one leaf, never a directory right.""" + from datetime import timedelta import pytest +from test_service import NOW, call, register from msg.core.codec import wire -from test_service import NOW, call, register @pytest.mark.asyncio @@ -12,53 +13,86 @@ async def test_private_ancestor_exact_read_and_immediate_revocation(installed): app, _ = installed alice_key, alice, _ = await register(app, 'share-alice') bob_key, bob, _ = await register(app, 'share-bob') - topic = await call(app, 'content.topic_create', {'parent': '/main', 'name': 'share-private'}, - key=alice_key, subject=alice) + topic = await call( + app, + 'content.topic_create', + {'parent': '/main', 'name': 'share-private'}, + key=alice_key, + subject=alice, + ) assert topic.status == 'ok', wire(topic) folder = topic.resources[0].id - secret = await call(app, 'content.post_create', {'parent': folder, 'body': 'shared exact text'}, - key=alice_key, subject=alice) - sibling = await call(app, 'content.post_create', {'parent': folder, 'body': 'unshared sibling'}, - key=alice_key, subject=alice) + secret = await call( + app, + 'content.post_create', + {'parent': folder, 'body': 'shared exact text'}, + key=alice_key, + subject=alice, + ) + sibling = await call( + app, + 'content.post_create', + {'parent': folder, 'body': 'unshared sibling'}, + key=alice_key, + subject=alice, + ) assert secret.status == sibling.status == 'ok' leaf = secret.resources[0].id - chmod = await call(app, 'content.chmod', {'id': folder, 'mode': '0700'}, - key=alice_key, subject=alice, - expected=((folder, topic.data['generation']),)) + chmod = await call( + app, + 'content.chmod', + {'id': folder, 'mode': '0700'}, + key=alice_key, + subject=alice, + expected=((folder, topic.data['generation']),), + ) assert chmod.status == 'ok', wire(chmod) before = await call(app, 'discovery.get', {'id': leaf}, key=bob_key, subject=bob) assert before.status == 'error' and before.error.code == 'permission_denied' async with app.metadata.transaction(write=False) as tx: - epoch_before=tx.setting('authorization_epoch',0) - granted = await call(app, 'sharing.grant', {'resource': leaf, 'grantee': bob, - 'expires_at': wire(NOW + timedelta(days=1))}, key=alice_key, subject=alice) + epoch_before = tx.setting('authorization_epoch', 0) + granted = await call( + app, + 'sharing.grant', + {'resource': leaf, 'grantee': bob, 'expires_at': wire(NOW + timedelta(days=1))}, + key=alice_key, + subject=alice, + ) assert granted.status == 'ok', wire(granted) async with app.metadata.transaction(write=False) as tx: - assert tx.setting('authorization_epoch',0)==epoch_before+1 + assert tx.setting('authorization_epoch', 0) == epoch_before + 1 gid = granted.data['grant']['id'] for view in ({}, {'view': 'meta'}, {'view': 'history'}): read = await call(app, 'discovery.get', {'id': leaf, **view}, key=bob_key, subject=bob) assert read.status == 'ok', wire(read) raw = await call(app, 'discovery.raw', {'id': leaf}, key=bob_key, subject=bob) assert raw.status == 'ok', wire(raw) - for op, args in (('discovery.get', {'id': folder}), - ('discovery.list', {'parent': folder}), - ('discovery.get', {'id': sibling.resources[0].id}), - ('sharing.grant', {'resource': leaf, 'grantee': alice, - 'expires_at': wire(NOW + timedelta(days=1))})): + for op, args in ( + ('discovery.get', {'id': folder}), + ('discovery.list', {'parent': folder}), + ('discovery.get', {'id': sibling.resources[0].id}), + ( + 'sharing.grant', + {'resource': leaf, 'grantee': alice, 'expires_at': wire(NOW + timedelta(days=1))}, + ), + ): denied = await call(app, op, args, key=bob_key, subject=bob) assert denied.status == 'error', (op, wire(denied)) - revoked = await call(app, 'sharing.revoke', {'grant_id': gid}, - key=alice_key, subject=alice) + revoked = await call(app, 'sharing.revoke', {'grant_id': gid}, key=alice_key, subject=alice) assert revoked.status == 'ok', wire(revoked) async with app.metadata.transaction(write=False) as tx: - assert tx.setting('authorization_epoch',0)==epoch_before+2 + assert tx.setting('authorization_epoch', 0) == epoch_before + 2 after = await call(app, 'discovery.get', {'id': leaf}, key=bob_key, subject=bob) assert after.status == 'error' and after.error.code == 'permission_denied', wire(after) # Revoking one source does not disable ordinary mode access. - traversable = await call(app, 'content.chmod', {'id': folder, 'mode': '0711'}, - key=alice_key, subject=alice, - expected=((folder, chmod.data['generation']),)) + traversable = await call( + app, + 'content.chmod', + {'id': folder, 'mode': '0711'}, + key=alice_key, + subject=alice, + expected=((folder, chmod.data['generation']),), + ) assert traversable.status == 'ok', wire(traversable) mode_read = await call(app, 'discovery.get', {'id': leaf}, key=bob_key, subject=bob) assert mode_read.status == 'ok', wire(mode_read) @@ -72,47 +106,99 @@ async def test_share_rejects_containers_and_managed_or_dm_resources(installed): alice_key, alice, _ = await register(app, 'share-boundary-a') bob_key, bob, _ = await register(app, 'share-boundary-b') expiry = wire(NOW + timedelta(hours=1)) - topic = await call(app, 'content.topic_create', {'parent': '/main', 'name': 'share-container'}, - key=alice_key, subject=alice) + topic = await call( + app, + 'content.topic_create', + {'parent': '/main', 'name': 'share-container'}, + key=alice_key, + subject=alice, + ) assert topic.status == 'ok' - container = await call(app, 'sharing.grant', {'resource': topic.resources[0].id, - 'grantee': bob, 'expires_at': expiry}, key=alice_key, subject=alice) + container = await call( + app, + 'sharing.grant', + {'resource': topic.resources[0].id, 'grantee': bob, 'expires_at': expiry}, + key=alice_key, + subject=alice, + ) assert container.status == 'error' and container.error.code == 'share_container_forbidden' - personal = await call(app, 'identity.personal_put', {'kind': 'soul', 'body': 'private soul'}, - key=alice_key, subject=alice) + personal = await call( + app, + 'identity.personal_put', + {'kind': 'soul', 'body': 'private soul'}, + key=alice_key, + subject=alice, + ) assert personal.status == 'ok', wire(personal) soul = personal.resources[0].id - denied = await call(app, 'sharing.grant', {'resource': soul, 'grantee': bob, - 'expires_at': expiry}, key=alice_key, subject=alice) + denied = await call( + app, + 'sharing.grant', + {'resource': soul, 'grantee': bob, 'expires_at': expiry}, + key=alice_key, + subject=alice, + ) assert denied.status == 'error', wire(denied) - note = await call(app, 'identity.note_put', {'name': 'only-this.md', 'body': 'one note'}, - key=alice_key, subject=alice) + note = await call( + app, + 'identity.note_put', + {'name': 'only-this.md', 'body': 'one note'}, + key=alice_key, + subject=alice, + ) assert note.status == 'ok', wire(note) note_id = note.resources[0].id - shared_note = await call(app, 'sharing.grant', {'resource': note_id, 'grantee': bob, - 'expires_at': expiry}, key=alice_key, subject=alice) + shared_note = await call( + app, + 'sharing.grant', + {'resource': note_id, 'grantee': bob, 'expires_at': expiry}, + key=alice_key, + subject=alice, + ) assert shared_note.status == 'ok', wire(shared_note) read_note = await call(app, 'discovery.get', {'id': note_id}, key=bob_key, subject=bob) assert read_note.status == 'ok', wire(read_note) async with app.metadata.transaction(write=False) as tx: notes_folder = (await tx.resource(note_id)).parent - list_notes = await call(app, 'discovery.list', {'parent': notes_folder}, - key=bob_key, subject=bob) + list_notes = await call( + app, 'discovery.list', {'parent': notes_folder}, key=bob_key, subject=bob + ) assert list_notes.status == 'error', wire(list_notes) - todo = await call(app, 'identity.todo_put', {'name': 'do-it.md', 'title': 'one task'}, - key=alice_key, subject=alice) + todo = await call( + app, + 'identity.todo_put', + {'name': 'do-it.md', 'title': 'one task'}, + key=alice_key, + subject=alice, + ) assert todo.status == 'ok', wire(todo) - shared_todo = await call(app, 'sharing.grant', {'resource': todo.resources[0].id, - 'grantee': bob, 'expires_at': expiry}, key=alice_key, subject=alice) + shared_todo = await call( + app, + 'sharing.grant', + {'resource': todo.resources[0].id, 'grantee': bob, 'expires_at': expiry}, + key=alice_key, + subject=alice, + ) assert shared_todo.status == 'error', wire(shared_todo) - conversation = await call(app, 'communication.dm_request', {'recipient': bob}, - key=alice_key, subject=alice) + conversation = await call( + app, 'communication.dm_request', {'recipient': bob}, key=alice_key, subject=alice + ) assert conversation.status == 'ok', wire(conversation) - dm = await call(app, 'sharing.grant', {'resource': conversation.data['conversation_id'], - 'grantee': bob, 'expires_at': expiry}, key=alice_key, subject=alice) + dm = await call( + app, + 'sharing.grant', + {'resource': conversation.data['conversation_id'], 'grantee': bob, 'expires_at': expiry}, + key=alice_key, + subject=alice, + ) assert dm.status == 'error', wire(dm) - system = await call(app, 'sharing.grant', {'resource': 'r_rules', 'grantee': bob, - 'expires_at': expiry}, key=alice_key, subject=alice) + system = await call( + app, + 'sharing.grant', + {'resource': 'r_rules', 'grantee': bob, 'expires_at': expiry}, + key=alice_key, + subject=alice, + ) assert system.status == 'error', wire(system) @@ -122,37 +208,87 @@ async def test_owner_revocation_while_archived_survives_restore(installed): alice_key, alice, _ = await register(app, 'share-dormant-a') bob_key, bob, _ = await register(app, 'share-dormant-b') carol_key, carol, _ = await register(app, 'share-dormant-c') - post = await call(app, 'content.post_create', {'parent': '/main', 'body': 'dormant'}, - key=alice_key, subject=alice) + post = await call( + app, + 'content.post_create', + {'parent': '/main', 'body': 'dormant'}, + key=alice_key, + subject=alice, + ) rid = post.resources[0].id - private = await call(app, 'content.chmod', {'id': rid, 'mode': '0600'}, - key=alice_key, subject=alice, - expected=((rid, post.data['generation']),)) + private = await call( + app, + 'content.chmod', + {'id': rid, 'mode': '0600'}, + key=alice_key, + subject=alice, + expected=((rid, post.data['generation']),), + ) assert private.status == 'ok', wire(private) expiry = wire(NOW + timedelta(days=1)) - v1 = await call(app, 'sharing.grant', {'resource': rid, 'grantee': bob, - 'expires_at': expiry}, key=alice_key, subject=alice) - v2 = await call(app, 'sharing.grant', {'resource': rid, 'grantee': carol, - 'grantee_kind': 'user', 'operations': ['read'], 'expires_at': expiry}, - key=alice_key, subject=alice, contract_version=2) + v1 = await call( + app, + 'sharing.grant', + {'resource': rid, 'grantee': bob, 'expires_at': expiry}, + key=alice_key, + subject=alice, + ) + v2 = await call( + app, + 'sharing.grant', + { + 'resource': rid, + 'grantee': carol, + 'grantee_kind': 'user', + 'operations': ['read'], + 'expires_at': expiry, + }, + key=alice_key, + subject=alice, + contract_version=2, + ) assert v1.status == v2.status == 'ok', (wire(v1), wire(v2)) for key, subject in ((bob_key, bob), (carol_key, carol)): read = await call(app, 'discovery.get', {'id': rid}, key=key, subject=subject) assert read.status == 'ok', wire(read) - archived = await call(app, 'content.archive', {'id': rid}, key=alice_key, subject=alice, - expected=((rid, private.data['generation']),)) + archived = await call( + app, + 'content.archive', + {'id': rid}, + key=alice_key, + subject=alice, + expected=((rid, private.data['generation']),), + ) assert archived.status == 'ok', wire(archived) for grant_id, version in ((v1.data['grant']['id'], 1), (v2.data['grant']['id'], 2)): for key, subject in ((bob_key, bob), (carol_key, carol)): - denied = await call(app, 'sharing.revoke', {'grant_id': grant_id}, - key=key, subject=subject, contract_version=version) + denied = await call( + app, + 'sharing.revoke', + {'grant_id': grant_id}, + key=key, + subject=subject, + contract_version=version, + ) assert denied.status == 'error', wire(denied) - revoked = await call(app, 'sharing.revoke', {'grant_id': grant_id}, - key=alice_key, subject=alice, contract_version=version) + revoked = await call( + app, + 'sharing.revoke', + {'grant_id': grant_id}, + key=alice_key, + subject=alice, + contract_version=version, + ) assert revoked.status == 'ok', wire(revoked) assert revoked.data['grant']['revoked_at'] is not None - restored = await call(app, 'content.restore', {'id': rid}, key=alice_key, subject=alice, - expected=((rid, archived.data['generation']),)) + restored = await call( + app, + 'content.restore', + {'id': rid}, + key=alice_key, + subject=alice, + expected=((rid, archived.data['generation']),), + ) assert restored.status == 'ok', wire(restored) for key, subject in ((bob_key, bob), (carol_key, carol)): after = await call(app, 'discovery.get', {'id': rid}, key=key, subject=subject) @@ -164,21 +300,41 @@ async def test_expired_grant_does_not_authorize_and_can_be_replaced(installed): app, _ = installed alice_key, alice, _ = await register(app, 'share-expire-a') bob_key, bob, _ = await register(app, 'share-expire-b') - post = await call(app, 'content.post_create', {'parent': '/main', 'body': 'expiring'}, - key=alice_key, subject=alice) + post = await call( + app, + 'content.post_create', + {'parent': '/main', 'body': 'expiring'}, + key=alice_key, + subject=alice, + ) rid = post.resources[0].id - private = await call(app, 'content.chmod', {'id': rid, 'mode': '0600'}, - key=alice_key, subject=alice, - expected=((rid, post.data['generation']),)) + private = await call( + app, + 'content.chmod', + {'id': rid, 'mode': '0600'}, + key=alice_key, + subject=alice, + expected=((rid, post.data['generation']),), + ) assert private.status == 'ok', wire(private) - first = await call(app, 'sharing.grant', {'resource': rid, 'grantee': bob, - 'expires_at': wire(NOW + timedelta(seconds=1))}, key=alice_key, subject=alice) + first = await call( + app, + 'sharing.grant', + {'resource': rid, 'grantee': bob, 'expires_at': wire(NOW + timedelta(seconds=1))}, + key=alice_key, + subject=alice, + ) assert first.status == 'ok', wire(first) app.executor.clock = lambda: NOW + timedelta(seconds=2) expired = await call(app, 'discovery.get', {'id': rid}, key=bob_key, subject=bob) assert expired.status == 'error' and expired.error.code == 'permission_denied', wire(expired) - replacement = await call(app, 'sharing.grant', {'resource': rid, 'grantee': bob, - 'expires_at': wire(NOW + timedelta(days=1))}, key=alice_key, subject=alice) + replacement = await call( + app, + 'sharing.grant', + {'resource': rid, 'grantee': bob, 'expires_at': wire(NOW + timedelta(days=1))}, + key=alice_key, + subject=alice, + ) assert replacement.status == 'ok', wire(replacement) restored = await call(app, 'discovery.get', {'id': rid}, key=bob_key, subject=bob) assert restored.status == 'ok', wire(restored) diff --git a/tests/test_share_grants_v2.py b/tests/test_share_grants_v2.py index c2cceaf4..776a65c3 100644 --- a/tests/test_share_grants_v2.py +++ b/tests/test_share_grants_v2.py @@ -1,137 +1,294 @@ """Share sources are independent and delegated grants track live group membership.""" + from datetime import timedelta import pytest +from test_service import NOW, call, register from msg.core.codec import wire -from test_service import NOW,call,register -async def invoke(app,identity,operation,args,*,version=1,expected=()): - return await call(app,operation,args,key=identity[0],subject=identity[1], - contract_version=version,expected=expected) +async def invoke(app, identity, operation, args, *, version=1, expected=()): + return await call( + app, + operation, + args, + key=identity[0], + subject=identity[1], + contract_version=version, + expected=expected, + ) def ok(result): - assert result.status=='ok',(result.error.code if result.error else None,wire(result)) + assert result.status == 'ok', (result.error.code if result.error else None, wire(result)) return result @pytest.mark.asyncio async def test_group_reshare_tracks_source_membership_and_independent_grants(installed): - app,_=installed - owner=await register(app,'share2-owner') - member=await register(app,'share2-member') - recipient=await register(app,'share2-recipient') - group=ok(await invoke(app,owner,'group.create',{'name':'share2-group'})).resources[0].id - ok(await invoke(app,owner,'group.invite',{'group':group,'subject':member[1]})) - ok(await invoke(app,member,'group.join',{'group':group})) - post=ok(await invoke(app,owner,'content.post_create', - {'parent':'/main','body':'v2 private'})) - rid=post.resources[0].id - ok(await invoke(app,owner,'content.chmod',{'id':rid,'mode':'0600'}, - expected=((rid,post.data['generation']),))) - expiry=wire(NOW+timedelta(days=2)) - group_grant=ok(await invoke(app,owner,'sharing.grant', - {'resource':rid,'grantee':group,'grantee_kind':'group','operations':['read'], - 'expires_at':expiry,'allow_reshare':True},version=2)).data['grant']['id'] - ok(await invoke(app,member,'discovery.get',{'id':rid})) - missing_source=await invoke(app,member,'sharing.grant', - {'resource':rid,'grantee':recipient[1],'grantee_kind':'user', - 'operations':['read'],'expires_at':expiry},version=2) - assert missing_source.error.code=='share_source_required' - overlong=await invoke(app,member,'sharing.grant', - {'resource':rid,'grantee':recipient[1],'grantee_kind':'user', - 'operations':['read'],'expires_at':wire(NOW+timedelta(days=3)), - 'source_grant_id':group_grant},version=2) - assert overlong.error.code=='share_scope_exceeded' - delegated=ok(await invoke(app,member,'sharing.grant', - {'resource':rid,'grantee':recipient[1],'grantee_kind':'user', - 'operations':['read'],'expires_at':wire(NOW+timedelta(days=1)), - 'source_grant_id':group_grant},version=2)).data['grant'] + app, _ = installed + owner = await register(app, 'share2-owner') + member = await register(app, 'share2-member') + recipient = await register(app, 'share2-recipient') + group = ok(await invoke(app, owner, 'group.create', {'name': 'share2-group'})).resources[0].id + ok(await invoke(app, owner, 'group.invite', {'group': group, 'subject': member[1]})) + ok(await invoke(app, member, 'group.join', {'group': group})) + post = ok( + await invoke(app, owner, 'content.post_create', {'parent': '/main', 'body': 'v2 private'}) + ) + rid = post.resources[0].id + ok( + await invoke( + app, + owner, + 'content.chmod', + {'id': rid, 'mode': '0600'}, + expected=((rid, post.data['generation']),), + ) + ) + expiry = wire(NOW + timedelta(days=2)) + group_grant = ok( + await invoke( + app, + owner, + 'sharing.grant', + { + 'resource': rid, + 'grantee': group, + 'grantee_kind': 'group', + 'operations': ['read'], + 'expires_at': expiry, + 'allow_reshare': True, + }, + version=2, + ) + ).data['grant']['id'] + ok(await invoke(app, member, 'discovery.get', {'id': rid})) + missing_source = await invoke( + app, + member, + 'sharing.grant', + { + 'resource': rid, + 'grantee': recipient[1], + 'grantee_kind': 'user', + 'operations': ['read'], + 'expires_at': expiry, + }, + version=2, + ) + assert missing_source.error.code == 'share_source_required' + overlong = await invoke( + app, + member, + 'sharing.grant', + { + 'resource': rid, + 'grantee': recipient[1], + 'grantee_kind': 'user', + 'operations': ['read'], + 'expires_at': wire(NOW + timedelta(days=3)), + 'source_grant_id': group_grant, + }, + version=2, + ) + assert overlong.error.code == 'share_scope_exceeded' + delegated = ok( + await invoke( + app, + member, + 'sharing.grant', + { + 'resource': rid, + 'grantee': recipient[1], + 'grantee_kind': 'user', + 'operations': ['read'], + 'expires_at': wire(NOW + timedelta(days=1)), + 'source_grant_id': group_grant, + }, + version=2, + ) + ).data['grant'] assert delegated['allow_reshare'] is False - ok(await invoke(app,recipient,'discovery.get',{'id':rid})) - denied=await invoke(app,recipient,'sharing.grant', - {'resource':rid,'grantee':member[1],'grantee_kind':'user', - 'operations':['read'],'expires_at':wire(NOW+timedelta(hours=1)), - 'source_grant_id':delegated['id']},version=2) - assert denied.error.code=='share_source_unavailable' - ok(await invoke(app,member,'group.leave',{'group':group})) - for identity in (member,recipient): - denied=await invoke(app,identity,'discovery.get',{'id':rid}) - assert denied.error.code=='permission_denied',wire(denied) - direct=ok(await invoke(app,owner,'sharing.grant', - {'resource':rid,'grantee':recipient[1],'grantee_kind':'user', - 'operations':['read'],'expires_at':expiry},version=2)).data['grant']['id'] - ok(await invoke(app,recipient,'discovery.get',{'id':rid})) - ok(await invoke(app,owner,'sharing.revoke',{'grant_id':group_grant},version=2)) - ok(await invoke(app,recipient,'discovery.get',{'id':rid})) - ok(await invoke(app,owner,'sharing.revoke',{'grant_id':direct},version=2)) - denied=await invoke(app,recipient,'discovery.get',{'id':rid}) - assert denied.error.code=='permission_denied' + ok(await invoke(app, recipient, 'discovery.get', {'id': rid})) + denied = await invoke( + app, + recipient, + 'sharing.grant', + { + 'resource': rid, + 'grantee': member[1], + 'grantee_kind': 'user', + 'operations': ['read'], + 'expires_at': wire(NOW + timedelta(hours=1)), + 'source_grant_id': delegated['id'], + }, + version=2, + ) + assert denied.error.code == 'share_source_unavailable' + ok(await invoke(app, member, 'group.leave', {'group': group})) + for identity in (member, recipient): + denied = await invoke(app, identity, 'discovery.get', {'id': rid}) + assert denied.error.code == 'permission_denied', wire(denied) + direct = ok( + await invoke( + app, + owner, + 'sharing.grant', + { + 'resource': rid, + 'grantee': recipient[1], + 'grantee_kind': 'user', + 'operations': ['read'], + 'expires_at': expiry, + }, + version=2, + ) + ).data['grant']['id'] + ok(await invoke(app, recipient, 'discovery.get', {'id': rid})) + ok(await invoke(app, owner, 'sharing.revoke', {'grant_id': group_grant}, version=2)) + ok(await invoke(app, recipient, 'discovery.get', {'id': rid})) + ok(await invoke(app, owner, 'sharing.revoke', {'grant_id': direct}, version=2)) + denied = await invoke(app, recipient, 'discovery.get', {'id': rid}) + assert denied.error.code == 'permission_denied' @pytest.mark.asyncio async def test_reshare_revocation_and_expiry_do_not_remove_other_sources(installed): - app,_=installed - owner=await register(app,'share2-chain-owner') - intermediary=await register(app,'share2-chain-middle') - recipient=await register(app,'share2-chain-end') - post=ok(await invoke(app,owner,'content.post_create', - {'parent':'/main','body':'chain private'})) - rid=post.resources[0].id - ok(await invoke(app,owner,'content.chmod',{'id':rid,'mode':'0600'}, - expected=((rid,post.data['generation']),))) - source=ok(await invoke(app,owner,'sharing.grant', - {'resource':rid,'grantee':intermediary[1],'grantee_kind':'user', - 'operations':['read'],'expires_at':wire(NOW+timedelta(days=2)), - 'allow_reshare':True},version=2)).data['grant']['id'] - child=ok(await invoke(app,intermediary,'sharing.grant', - {'resource':rid,'grantee':recipient[1],'grantee_kind':'user', - 'operations':['read'],'expires_at':wire(NOW+timedelta(days=1)), - 'source_grant_id':source},version=2)).data['grant']['id'] - independent=ok(await invoke(app,owner,'sharing.grant', - {'resource':rid,'grantee':recipient[1],'grantee_kind':'user', - 'operations':['read'],'expires_at':wire(NOW+timedelta(hours=1))}, - version=2)).data['grant']['id'] - ok(await invoke(app,recipient,'discovery.get',{'id':rid})) - ok(await invoke(app,owner,'sharing.revoke',{'grant_id':source},version=2)) - ok(await invoke(app,recipient,'discovery.get',{'id':rid})) - ok(await invoke(app,owner,'sharing.revoke',{'grant_id':independent},version=2)) - denied=await invoke(app,recipient,'discovery.get',{'id':rid}) - assert denied.error.code=='permission_denied' - listed=ok(await invoke(app,intermediary,'sharing.list',{'resource':rid},version=2)) - assert listed.data['grants'][0]['id']==child - app.executor.clock=lambda:NOW+timedelta(days=3) - denied=await invoke(app,intermediary,'discovery.get',{'id':rid}) - assert denied.error.code=='permission_denied' + app, _ = installed + owner = await register(app, 'share2-chain-owner') + intermediary = await register(app, 'share2-chain-middle') + recipient = await register(app, 'share2-chain-end') + post = ok( + await invoke( + app, owner, 'content.post_create', {'parent': '/main', 'body': 'chain private'} + ) + ) + rid = post.resources[0].id + ok( + await invoke( + app, + owner, + 'content.chmod', + {'id': rid, 'mode': '0600'}, + expected=((rid, post.data['generation']),), + ) + ) + source = ok( + await invoke( + app, + owner, + 'sharing.grant', + { + 'resource': rid, + 'grantee': intermediary[1], + 'grantee_kind': 'user', + 'operations': ['read'], + 'expires_at': wire(NOW + timedelta(days=2)), + 'allow_reshare': True, + }, + version=2, + ) + ).data['grant']['id'] + child = ok( + await invoke( + app, + intermediary, + 'sharing.grant', + { + 'resource': rid, + 'grantee': recipient[1], + 'grantee_kind': 'user', + 'operations': ['read'], + 'expires_at': wire(NOW + timedelta(days=1)), + 'source_grant_id': source, + }, + version=2, + ) + ).data['grant']['id'] + independent = ok( + await invoke( + app, + owner, + 'sharing.grant', + { + 'resource': rid, + 'grantee': recipient[1], + 'grantee_kind': 'user', + 'operations': ['read'], + 'expires_at': wire(NOW + timedelta(hours=1)), + }, + version=2, + ) + ).data['grant']['id'] + ok(await invoke(app, recipient, 'discovery.get', {'id': rid})) + ok(await invoke(app, owner, 'sharing.revoke', {'grant_id': source}, version=2)) + ok(await invoke(app, recipient, 'discovery.get', {'id': rid})) + ok(await invoke(app, owner, 'sharing.revoke', {'grant_id': independent}, version=2)) + denied = await invoke(app, recipient, 'discovery.get', {'id': rid}) + assert denied.error.code == 'permission_denied' + listed = ok(await invoke(app, intermediary, 'sharing.list', {'resource': rid}, version=2)) + assert listed.data['grants'][0]['id'] == child + app.executor.clock = lambda: NOW + timedelta(days=3) + denied = await invoke(app, intermediary, 'discovery.get', {'id': rid}) + assert denied.error.code == 'permission_denied' @pytest.mark.asyncio async def test_v2_group_share_does_not_open_parent_or_sensitive_resource(installed): - app,_=installed - owner=await register(app,'share2-private-owner') - member=await register(app,'share2-private-member') - group=ok(await invoke(app,owner,'group.create',{'name':'share2-private-group'})).resources[0].id - ok(await invoke(app,owner,'group.invite',{'group':group,'subject':member[1]})) - ok(await invoke(app,member,'group.join',{'group':group})) - note=ok(await invoke(app,owner,'identity.note_put', - {'name':'alone.md','body':'single private note'})) - rid=note.resources[0].id - grant=ok(await invoke(app,owner,'sharing.grant', - {'resource':rid,'grantee':group,'grantee_kind':'group', - 'operations':['read'],'expires_at':wire(NOW+timedelta(days=1))}, - version=2)) - assert grant.data['grant']['status']=='active' - ok(await invoke(app,member,'discovery.get',{'id':rid})) + app, _ = installed + owner = await register(app, 'share2-private-owner') + member = await register(app, 'share2-private-member') + group = ( + ok(await invoke(app, owner, 'group.create', {'name': 'share2-private-group'})) + .resources[0] + .id + ) + ok(await invoke(app, owner, 'group.invite', {'group': group, 'subject': member[1]})) + ok(await invoke(app, member, 'group.join', {'group': group})) + note = ok( + await invoke( + app, owner, 'identity.note_put', {'name': 'alone.md', 'body': 'single private note'} + ) + ) + rid = note.resources[0].id + grant = ok( + await invoke( + app, + owner, + 'sharing.grant', + { + 'resource': rid, + 'grantee': group, + 'grantee_kind': 'group', + 'operations': ['read'], + 'expires_at': wire(NOW + timedelta(days=1)), + }, + version=2, + ) + ) + assert grant.data['grant']['status'] == 'active' + ok(await invoke(app, member, 'discovery.get', {'id': rid})) async with app.metadata.transaction(write=False) as tx: - parent=(await tx.resource(rid)).parent - denied=await invoke(app,member,'discovery.list',{'parent':parent}) - assert denied.error.code=='permission_denied' - for kind in ('soul','agents'): - personal=ok(await invoke(app,owner,'identity.personal_put', - {'kind':kind,'body':'private'})) - result=await invoke(app,owner,'sharing.grant', - {'resource':personal.resources[0].id,'grantee':group,'grantee_kind':'group', - 'operations':['read'],'expires_at':wire(NOW+timedelta(days=1))},version=2) - assert result.status=='error',wire(result) + parent = (await tx.resource(rid)).parent + denied = await invoke(app, member, 'discovery.list', {'parent': parent}) + assert denied.error.code == 'permission_denied' + for kind in ('soul', 'agents'): + personal = ok( + await invoke(app, owner, 'identity.personal_put', {'kind': kind, 'body': 'private'}) + ) + result = await invoke( + app, + owner, + 'sharing.grant', + { + 'resource': personal.resources[0].id, + 'grantee': group, + 'grantee_kind': 'group', + 'operations': ['read'], + 'expires_at': wire(NOW + timedelta(days=1)), + }, + version=2, + ) + assert result.status == 'error', wire(result) diff --git a/tests/test_share_links.py b/tests/test_share_links.py index 74807a31..aba9f699 100644 --- a/tests/test_share_links.py +++ b/tests/test_share_links.py @@ -1,54 +1,71 @@ """Bearer links stay off by default and authorize exactly one bounded read.""" -from datetime import timedelta + import hashlib import os +from datetime import timedelta import httpx import pytest +from test_authorization import approve, scoped +from test_service import NOW, call, register from msg.core.codec import b64, canonical, unb64, wire from msg.core.requests import request_for from msg.transports.dictionary import build_dictionary from msg.transports.http import create_app -from test_service import NOW, call, register -from test_authorization import approve, scoped def proof(): token = b64(os.urandom(32)) - verifier = 'sha256:' + hashlib.sha256(b'share-link-v1\0'+unb64(token)).hexdigest() + verifier = 'sha256:' + hashlib.sha256(b'share-link-v1\0' + unb64(token)).hexdigest() return token, verifier @pytest.mark.asyncio async def test_bearer_link_rejects_get_packet_and_short_paths(installed): - app,_=installed - token,_=proof() - packet=request_for('sharing.link_read',{'link_id':'link_missing','token':token}, - app.settings.service_url,expires_at=NOW+timedelta(seconds=120)) - encoded=b64(canonical(wire(packet))) - code=build_dictionary(app.registry).code_for('operation','sharing.link_read@1') - async with httpx.AsyncClient(transport=httpx.ASGITransport(app=create_app(app)), - base_url=app.settings.service_url) as http: - named=await http.get('/-/g/sharing.link_read/j/'+encoded) - short=await http.get('/-/g/'+code) - assert named.status_code==short.status_code==405 - posted=await http.post('/-/p/sharing.link_read',content=canonical(wire(packet))) - assert posted.status_code==400 - assert posted.json()['error']['code']=='share_links_disabled' + app, _ = installed + token, _ = proof() + packet = request_for( + 'sharing.link_read', + {'link_id': 'link_missing', 'token': token}, + app.settings.service_url, + expires_at=NOW + timedelta(seconds=120), + ) + encoded = b64(canonical(wire(packet))) + code = build_dictionary(app.registry).code_for('operation', 'sharing.link_read@1') + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=create_app(app)), base_url=app.settings.service_url + ) as http: + named = await http.get('/-/g/sharing.link_read/j/' + encoded) + short = await http.get('/-/g/' + code) + assert named.status_code == short.status_code == 405 + posted = await http.post('/-/p/sharing.link_read', content=canonical(wire(packet))) + assert posted.status_code == 400 + assert posted.json()['error']['code'] == 'share_links_disabled' @pytest.mark.asyncio async def test_share_links_require_explicit_system_enable(installed): - app,root=installed - key,subject,_=await register(app,'link-operator') - denied=await call(app,'system.share_links_set',{'enabled':True},key=key,subject=subject) - assert denied.status=='error' and denied.error.code=='capability_required' - cert=await approve(app,root,subject,key,( - scoped(app,'system.config','r_root',('system.share_links_set@1',)),)) - enabled=await call(app,'system.share_links_set',{'enabled':True}, - key=key,subject=subject,certs=(cert.resource_id,)) - assert enabled.status=='ok' and enabled.data['enabled'] is True + app, root = installed + key, subject, _ = await register(app, 'link-operator') + denied = await call(app, 'system.share_links_set', {'enabled': True}, key=key, subject=subject) + assert denied.status == 'error' and denied.error.code == 'capability_required' + cert = await approve( + app, + root, + subject, + key, + (scoped(app, 'system.config', 'r_root', ('system.share_links_set@1',)),), + ) + enabled = await call( + app, + 'system.share_links_set', + {'enabled': True}, + key=key, + subject=subject, + certs=(cert.resource_id,), + ) + assert enabled.status == 'ok' and enabled.data['enabled'] is True async with app.metadata.transaction(write=False) as tx: assert tx.setting('share_links_enabled') is True @@ -57,18 +74,39 @@ async def test_share_links_require_explicit_system_enable(installed): async def test_share_link_is_explicit_bounded_and_revocable(installed): app, _ = installed key, owner, _ = await register(app, 'link-owner') - topic = await call(app, 'content.topic_create', {'parent':'/main','name':'link-folder'}, - key=key, subject=owner) + topic = await call( + app, + 'content.topic_create', + {'parent': '/main', 'name': 'link-folder'}, + key=key, + subject=owner, + ) folder = topic.resources[0].id - post = await call(app, 'content.post_create', {'parent':folder,'body':'one private leaf'}, - key=key, subject=owner) - sibling = await call(app, 'content.post_create', {'parent':folder,'body':'hidden sibling'}, - key=key, subject=owner) + post = await call( + app, + 'content.post_create', + {'parent': folder, 'body': 'one private leaf'}, + key=key, + subject=owner, + ) + sibling = await call( + app, + 'content.post_create', + {'parent': folder, 'body': 'hidden sibling'}, + key=key, + subject=owner, + ) rid = post.resources[0].id - await call(app, 'content.chmod', {'id':folder,'mode':'0700'}, key=key, subject=owner, - expected=((folder,topic.data['generation']),)) + await call( + app, + 'content.chmod', + {'id': folder, 'mode': '0700'}, + key=key, + subject=owner, + expected=((folder, topic.data['generation']),), + ) token, verifier = proof() - args = {'resource':rid,'verifier':verifier,'expires_at':wire(NOW+timedelta(hours=1))} + args = {'resource': rid, 'verifier': verifier, 'expires_at': wire(NOW + timedelta(hours=1))} off = await call(app, 'sharing.link_create', args, key=key, subject=owner) assert off.error.code == 'share_links_disabled' async with app.metadata.transaction(write=True) as tx: @@ -79,62 +117,91 @@ async def test_share_link_is_explicit_bounded_and_revocable(installed): assert token not in str(created.data) and verifier not in str(created.data) assert created.data['endpoint'] == '/-/p/sharing.link_read' async with app.metadata.transaction(write=False) as tx: - before = (tx.one('SELECT COUNT(*) FROM events')[0], - tx.one('SELECT COUNT(*) FROM audit')[0], - tx.setting('authorization_epoch',0)) - assert tx.one('SELECT verifier FROM share_links WHERE id=?',(link['id'],))[0] == verifier - wrong = await call(app, 'sharing.link_read', {'link_id':link['id'], - 'token':b64(os.urandom(32))}) - missing = await call(app, 'sharing.link_read', {'link_id':'link_missing','token':token}) - for denied in (wrong,missing): - assert denied.status=='error' and denied.error.code=='share_link_unavailable' - read = await call(app, 'sharing.link_read', {'link_id':link['id'],'token':token}) - assert read.status=='ok', (read.error.code if read.error else None, wire(read)) - assert read.data['id']==rid and 'hidden sibling' not in str(read.data) - assert set(read.data)=={'id','revision','media_type','data'} + before = ( + tx.one('SELECT COUNT(*) FROM events')[0], + tx.one('SELECT COUNT(*) FROM audit')[0], + tx.setting('authorization_epoch', 0), + ) + assert tx.one('SELECT verifier FROM share_links WHERE id=?', (link['id'],))[0] == verifier + wrong = await call( + app, 'sharing.link_read', {'link_id': link['id'], 'token': b64(os.urandom(32))} + ) + missing = await call(app, 'sharing.link_read', {'link_id': 'link_missing', 'token': token}) + for denied in (wrong, missing): + assert denied.status == 'error' and denied.error.code == 'share_link_unavailable' + read = await call(app, 'sharing.link_read', {'link_id': link['id'], 'token': token}) + assert read.status == 'ok', (read.error.code if read.error else None, wire(read)) + assert read.data['id'] == rid and 'hidden sibling' not in str(read.data) + assert set(read.data) == {'id', 'revision', 'media_type', 'data'} assert 'one private leaf' in unb64(read.data['data']).decode() - anonymous_folder = await call(app, 'discovery.list', {'parent':folder}) - anonymous_sibling = await call(app, 'discovery.get', {'id':sibling.resources[0].id}) - assert anonymous_folder.status==anonymous_sibling.status=='error' + anonymous_folder = await call(app, 'discovery.list', {'parent': folder}) + anonymous_sibling = await call(app, 'discovery.get', {'id': sibling.resources[0].id}) + assert anonymous_folder.status == anonymous_sibling.status == 'error' async with app.metadata.transaction(write=False) as tx: - assert (tx.one('SELECT COUNT(*) FROM events')[0], - tx.one('SELECT COUNT(*) FROM audit')[0], - tx.setting('authorization_epoch',0)) == before - revoked = await call(app, 'sharing.link_revoke', {'link_id':link['id']}, - key=key, subject=owner) - assert revoked.status=='ok' - after = await call(app, 'sharing.link_read', {'link_id':link['id'],'token':token}) - assert after.error.code=='share_link_unavailable' + assert ( + tx.one('SELECT COUNT(*) FROM events')[0], + tx.one('SELECT COUNT(*) FROM audit')[0], + tx.setting('authorization_epoch', 0), + ) == before + revoked = await call( + app, 'sharing.link_revoke', {'link_id': link['id']}, key=key, subject=owner + ) + assert revoked.status == 'ok' + after = await call(app, 'sharing.link_read', {'link_id': link['id'], 'token': token}) + assert after.error.code == 'share_link_unavailable' @pytest.mark.asyncio async def test_share_link_rechecks_owner_acl_expiry_and_global_switch(installed): app, _ = installed key, owner, _ = await register(app, 'link-policy-owner') - post = await call(app, 'content.post_create', {'parent':'/main','body':'policy body'}, - key=key, subject=owner) - rid=post.resources[0].id + post = await call( + app, + 'content.post_create', + {'parent': '/main', 'body': 'policy body'}, + key=key, + subject=owner, + ) + rid = post.resources[0].id async with app.metadata.transaction(write=True) as tx: tx.set_setting('share_links_enabled', True) - token, verifier=proof() - created=await call(app,'sharing.link_create',{'resource':rid,'verifier':verifier, - 'expires_at':wire(NOW+timedelta(seconds=2))},key=key,subject=owner) - assert created.status=='ok',wire(created) - lid=created.data['link']['id'] - read=lambda:call(app,'sharing.link_read',{'link_id':lid,'token':token}) - first=await read() - assert first.status=='ok', (first.error.code if first.error else None,wire(first)) - hidden=await call(app,'content.chmod',{'id':rid,'mode':'0000'},key=key,subject=owner, - expected=((rid,post.data['generation']),)) - assert hidden.status=='ok',wire(hidden) - assert (await read()).error.code=='share_link_unavailable' - restored=await call(app,'content.chmod',{'id':rid,'mode':'0600'},key=key,subject=owner, - expected=((rid,hidden.data['generation']),)) - assert restored.status=='ok',wire(restored) - assert (await read()).status=='ok' - app.executor.clock=lambda:NOW+timedelta(seconds=3) - assert (await read()).error.code=='share_link_unavailable' - app.executor.clock=lambda:NOW + token, verifier = proof() + created = await call( + app, + 'sharing.link_create', + {'resource': rid, 'verifier': verifier, 'expires_at': wire(NOW + timedelta(seconds=2))}, + key=key, + subject=owner, + ) + assert created.status == 'ok', wire(created) + lid = created.data['link']['id'] + def read(): + return call(app, 'sharing.link_read', {'link_id': lid, 'token': token}) + first = await read() + assert first.status == 'ok', (first.error.code if first.error else None, wire(first)) + hidden = await call( + app, + 'content.chmod', + {'id': rid, 'mode': '0000'}, + key=key, + subject=owner, + expected=((rid, post.data['generation']),), + ) + assert hidden.status == 'ok', wire(hidden) + assert (await read()).error.code == 'share_link_unavailable' + restored = await call( + app, + 'content.chmod', + {'id': rid, 'mode': '0600'}, + key=key, + subject=owner, + expected=((rid, hidden.data['generation']),), + ) + assert restored.status == 'ok', wire(restored) + assert (await read()).status == 'ok' + app.executor.clock = lambda: NOW + timedelta(seconds=3) + assert (await read()).error.code == 'share_link_unavailable' + app.executor.clock = lambda: NOW async with app.metadata.transaction(write=True) as tx: tx.set_setting('share_links_enabled', False) - assert (await read()).error.code=='share_links_disabled' + assert (await read()).error.code == 'share_links_disabled' diff --git a/tests/test_share_source_boundaries.py b/tests/test_share_source_boundaries.py index b8befb3f..3fcab2c1 100644 --- a/tests/test_share_source_boundaries.py +++ b/tests/test_share_source_boundaries.py @@ -1,28 +1,53 @@ """Every consumer of a ShareGrant must recheck the same current target boundary.""" + from dataclasses import replace from datetime import timedelta import pytest - -from msg.core.codec import canonical, wire from test_service import NOW, register from test_share_grants_v2 import invoke, ok +from msg.core.codec import canonical, wire + @pytest.mark.asyncio async def test_named_source_revalidates_target_and_persisted_constraints(installed): app, _ = installed owner = await register(app, 'boundary-owner') reader = await register(app, 'boundary-reader') - post = ok(await invoke(app, owner, 'content.post_create', - {'parent': '/main', 'body': 'private source evidence'})) + post = ok( + await invoke( + app, + owner, + 'content.post_create', + {'parent': '/main', 'body': 'private source evidence'}, + ) + ) rid = post.resources[0].id - ok(await invoke(app, owner, 'content.chmod', {'id': rid, 'mode': '0600'}, - expected=((rid, post.data['generation']),))) - grant = ok(await invoke(app, owner, 'sharing.grant', - {'resource': rid, 'grantee': reader[1], 'grantee_kind': 'user', - 'operations': ['read'], 'expires_at': wire(NOW + timedelta(days=2))}, - version=2)).data['grant']['id'] + ok( + await invoke( + app, + owner, + 'content.chmod', + {'id': rid, 'mode': '0600'}, + expected=((rid, post.data['generation']),), + ) + ) + grant = ok( + await invoke( + app, + owner, + 'sharing.grant', + { + 'resource': rid, + 'grantee': reader[1], + 'grantee_kind': 'user', + 'operations': ['read'], + 'expires_at': wire(NOW + timedelta(days=2)), + }, + version=2, + ) + ).data['grant']['id'] async with app.metadata.transaction(write=True) as tx: resource = await tx.resource(rid) assert await app.authorizer.share_source_active(resource, grant, reader[1], NOW, tx) @@ -36,11 +61,18 @@ async def test_named_source_revalidates_target_and_persisted_constraints(install ] for column, value in cases: original = tx.one(f'SELECT {column} FROM share_grants_v2 WHERE id=?', (grant,))[0] - tx.execute(f'UPDATE share_grants_v2 SET {column}=? WHERE id=?', (value, grant), write=True) - assert not await app.authorizer.share_source_active(resource, grant, reader[1], NOW, tx), column - tx.execute(f'UPDATE share_grants_v2 SET {column}=? WHERE id=?', (original, grant), write=True) + tx.execute( + f'UPDATE share_grants_v2 SET {column}=? WHERE id=?', (value, grant), write=True + ) + assert not await app.authorizer.share_source_active( + resource, grant, reader[1], NOW, tx + ), column + tx.execute( + f'UPDATE share_grants_v2 SET {column}=? WHERE id=?', (original, grant), write=True + ) assert not await app.authorizer.share_source_active( - replace(resource, state='archived'), grant, reader[1], NOW, tx) + replace(resource, state='archived'), grant, reader[1], NOW, tx + ) tx.set_setting('hosting_preview_file:' + rid, {'owner': owner[1]}) assert not await app.authorizer.share_source_active(resource, grant, reader[1], NOW, tx) @@ -51,25 +83,65 @@ async def test_child_cannot_outlive_source_even_after_restore(installed): owner = await register(app, 'duration-owner') middle = await register(app, 'duration-middle') reader = await register(app, 'duration-reader') - post = ok(await invoke(app, owner, 'content.post_create', - {'parent': '/main', 'body': 'duration'})) + post = ok( + await invoke(app, owner, 'content.post_create', {'parent': '/main', 'body': 'duration'}) + ) rid = post.resources[0].id - source = ok(await invoke(app, owner, 'sharing.grant', - {'resource': rid, 'grantee': middle[1], 'grantee_kind': 'user', - 'operations': ['read'], 'expires_at': wire(NOW + timedelta(days=2)), - 'allow_reshare': True}, version=2)).data['grant']['id'] - child = ok(await invoke(app, middle, 'sharing.grant', - {'resource': rid, 'grantee': reader[1], 'grantee_kind': 'user', - 'operations': ['read'], 'expires_at': wire(NOW + timedelta(days=1)), - 'source_grant_id': source}, version=2)).data['grant']['id'] + source = ok( + await invoke( + app, + owner, + 'sharing.grant', + { + 'resource': rid, + 'grantee': middle[1], + 'grantee_kind': 'user', + 'operations': ['read'], + 'expires_at': wire(NOW + timedelta(days=2)), + 'allow_reshare': True, + }, + version=2, + ) + ).data['grant']['id'] + child = ok( + await invoke( + app, + middle, + 'sharing.grant', + { + 'resource': rid, + 'grantee': reader[1], + 'grantee_kind': 'user', + 'operations': ['read'], + 'expires_at': wire(NOW + timedelta(days=1)), + 'source_grant_id': source, + }, + version=2, + ) + ).data['grant']['id'] async with app.metadata.transaction(write=True) as tx: resource = await tx.resource(rid) assert await app.authorizer.share_source_active(resource, child, reader[1], NOW, tx) - tx.execute('UPDATE share_grants_v2 SET expires_at=? WHERE id=?', - (wire(NOW + timedelta(days=3)), child), write=True) + tx.execute( + 'UPDATE share_grants_v2 SET expires_at=? WHERE id=?', + (wire(NOW + timedelta(days=3)), child), + write=True, + ) assert not await app.authorizer.share_source_active(resource, child, reader[1], NOW, tx) # An independent owner grant is not vetoed by this malformed source. - ok(await invoke(app, owner, 'sharing.grant', - {'resource': rid, 'grantee': reader[1], 'grantee_kind': 'user', - 'operations': ['read'], 'expires_at': wire(NOW + timedelta(hours=1))}, version=2)) + ok( + await invoke( + app, + owner, + 'sharing.grant', + { + 'resource': rid, + 'grantee': reader[1], + 'grantee_kind': 'user', + 'operations': ['read'], + 'expires_at': wire(NOW + timedelta(hours=1)), + }, + version=2, + ) + ) ok(await invoke(app, reader, 'discovery.get', {'id': rid})) diff --git a/tests/test_share_transport_matrix.py b/tests/test_share_transport_matrix.py index caecde78..d1b75b81 100644 --- a/tests/test_share_transport_matrix.py +++ b/tests/test_share_transport_matrix.py @@ -1,11 +1,12 @@ """Live share revocation crosses real adapters, historical reads, search and Sync.""" + import asyncio -from contextlib import asynccontextmanager -from datetime import timedelta import os -from pathlib import Path import socket import sys +from contextlib import asynccontextmanager +from datetime import timedelta +from pathlib import Path from uuid import uuid4 import pytest @@ -17,7 +18,7 @@ from msg.config import write_example from msg.core.codec import canonical, digest, loads, wire from msg.core.executor import result_wire -from msg.transports.client import HTTPTransport, TRANSPORTS +from msg.transports.client import TRANSPORTS, HTTPTransport from msg.transports.http import create_app @@ -33,21 +34,21 @@ async def live_adapters(tmp_path, pg_dsn, mode): sock.bind(('127.0.0.1', 0)) sock.setblocking(False) url = 'http://127.0.0.1:' + str(sock.getsockname()[1]) - app = Application(write_example(tmp_path / 'etc', tmp_path / 'data', url, - postgres_dsn=pg_dsn)) + app = Application(write_example(tmp_path / 'etc', tmp_path / 'data', url, postgres_dsn=pg_dsn)) server = None serving = process = None transports = [] try: csr, root = await _provision(app, 'share-matrix-test-passphrase') await _approve_csr(app, csr, root, expected_digest=None, operator='matrix-fixture') - server = uvicorn.Server(uvicorn.Config(create_app(app), log_level='critical', - access_log=False, ws='none')) + server = uvicorn.Server( + uvicorn.Config(create_app(app), log_level='critical', access_log=False, ws='none') + ) serving = asyncio.create_task(server.serve(sockets=[sock])) for _ in range(100): if server.started: break - await asyncio.sleep(.02) + await asyncio.sleep(0.02) assert server.started peers = [] for name in ('owner', 'middle', 'reader'): @@ -61,23 +62,50 @@ async def live_adapters(tmp_path, pg_dsn, mode): transports.append(transport) client = MsgClient(reader.state, transport) environment = {**os.environ, 'PYTHONPATH': str(Path(__file__).parents[1] / 'src')} - command = [sys.executable, '-m', 'msg.cli', '--config-dir', - str(reader.state.directory), '--server', url] + command = [ + sys.executable, + '-m', + 'msg.cli', + '--config-dir', + str(reader.state.directory), + '--server', + url, + ] if mode == 'mcp_stdio': - process = await asyncio.create_subprocess_exec(*command, 'mcp', env=environment, - stdin=asyncio.subprocess.PIPE, stdout=asyncio.subprocess.PIPE, - stderr=asyncio.subprocess.PIPE) - process.stdin.write(canonical({'jsonrpc': '2.0', 'id': 'init', - 'method': 'initialize', 'params': {'protocolVersion': '2025-11-25'}}) + b'\n') + process = await asyncio.create_subprocess_exec( + *command, + 'mcp', + env=environment, + stdin=asyncio.subprocess.PIPE, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + process.stdin.write( + canonical({ + 'jsonrpc': '2.0', + 'id': 'init', + 'method': 'initialize', + 'params': {'protocolVersion': '2025-11-25'}, + }) + + b'\n' + ) await process.stdin.drain() assert 'result' in loads(await asyncio.wait_for(process.stdout.readline(), 20)) async def invoke(operation, arguments): request_id = uuid4().hex if mode == 'cli': - child = await asyncio.create_subprocess_exec(*command, 'call', operation, - canonical(arguments).decode(), '--request-id', request_id, env=environment, - stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE) + child = await asyncio.create_subprocess_exec( + *command, + 'call', + operation, + canonical(arguments).decode(), + '--request-id', + request_id, + env=environment, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) try: stdout, stderr = await asyncio.wait_for(child.communicate(), 30) except BaseException: @@ -88,9 +116,19 @@ async def invoke(operation, arguments): assert child.returncode in {0, 1}, stderr.decode() return loads(stdout) if mode == 'mcp_stdio': - process.stdin.write(canonical({'jsonrpc': '2.0', 'id': request_id, - 'method': 'tools/call', 'params': {'name': operation, 'arguments': arguments, - '_meta': {'msg/request_id': request_id}}}) + b'\n') + process.stdin.write( + canonical({ + 'jsonrpc': '2.0', + 'id': request_id, + 'method': 'tools/call', + 'params': { + 'name': operation, + 'arguments': arguments, + '_meta': {'msg/request_id': request_id}, + }, + }) + + b'\n' + ) await process.stdin.drain() response = loads(await asyncio.wait_for(process.stdout.readline(), 20)) assert 'result' in response, response @@ -126,16 +164,41 @@ async def test_live_group_reshare_revocation_across_read_surfaces(tmp_path, pg_d ok(await middle.call('group.join', {'group': group})) post = ok(await owner.call('content.post_create', {'parent': '/main', 'body': marker})) rid, original = post['resources'][0]['id'], post['resources'][0]['revision'] - private = ok(await owner.call('content.chmod', {'id': rid, 'mode': '0600'}, - expected=((rid, post['data']['generation']),))) - ok(await owner.call('content.post_edit', {'id': rid, 'expected_revision': original, - 'body': marker + '\nnew revision'}, expected=((rid, private['data']['generation']),))) + private = ok( + await owner.call( + 'content.chmod', + {'id': rid, 'mode': '0600'}, + expected=((rid, post['data']['generation']),), + ) + ) + ok( + await owner.call( + 'content.post_edit', + {'id': rid, 'expected_revision': original, 'body': marker + '\nnew revision'}, + expected=((rid, private['data']['generation']),), + ) + ) expiry = wire(app.clock() + timedelta(minutes=30)) terms = {'resource': rid, 'operations': ['read'], 'expires_at': expiry} - parent = ok(await owner.call('sharing.grant', {**terms, 'grantee': group, - 'grantee_kind': 'group', 'allow_reshare': True}, contract_version=2))['data']['grant']['id'] - ok(await middle.call('sharing.grant', {**terms, 'grantee': reader.state.subject, - 'grantee_kind': 'user', 'source_grant_id': parent}, contract_version=2)) + parent = ok( + await owner.call( + 'sharing.grant', + {**terms, 'grantee': group, 'grantee_kind': 'group', 'allow_reshare': True}, + contract_version=2, + ) + )['data']['grant']['id'] + ok( + await middle.call( + 'sharing.grant', + { + **terms, + 'grantee': reader.state.subject, + 'grantee_kind': 'user', + 'source_grant_id': parent, + }, + contract_version=2, + ) + ) current = ok(await invoke('discovery.get', {'id': rid})) assert marker in canonical(current).decode() historical = ok(await invoke('discovery.get', {'id': rid, 'revision': original})) @@ -158,22 +221,38 @@ async def test_live_group_reshare_revocation_across_read_surfaces(tmp_path, pg_d # Reuse the same clients, credential, cursor and cache digest after the # source member leaves. Child grants are not independent authority. ok(await middle.call('group.leave', {'group': group})) - for query in ({'id': rid}, {'id': rid, 'revision': original}, - {'id': rid, 'view': 'history'}, cached): + for query in ( + {'id': rid}, + {'id': rid, 'revision': original}, + {'id': rid, 'view': 'history'}, + cached, + ): denied = await invoke('discovery.get', query) - assert denied['status'] == 'error' and denied['error']['code'] == 'permission_denied', denied + assert denied['status'] == 'error' and denied['error']['code'] == 'permission_denied', ( + denied + ) assert marker not in canonical(denied).decode() assert not denied.get('data', {}).get('not_modified', False) hidden = ok(await invoke('discovery.lexical_search', search)) - assert rid not in canonical(hidden).decode() and marker not in canonical(hidden['data']).decode() + assert ( + rid not in canonical(hidden).decode() + and marker not in canonical(hidden['data']).decode() + ) delta = ok(await invoke('communication.sync', {'cursor': cursor, 'limit': 100}))['data'] assert delta['resync_required'] is True - assert any(item['kind'] == 'revoked' and item['ref']['id'] == rid for item in delta['items']) + assert any( + item['kind'] == 'revoked' and item['ref']['id'] == rid for item in delta['items'] + ) assert marker not in canonical(delta).decode() # An invalid old source must not veto an independent live owner grant. - direct = ok(await owner.call('sharing.grant', {**terms, 'grantee': reader.state.subject, - 'grantee_kind': 'user'}, contract_version=2))['data']['grant']['id'] + direct = ok( + await owner.call( + 'sharing.grant', + {**terms, 'grantee': reader.state.subject, 'grantee_kind': 'user'}, + contract_version=2, + ) + )['data']['grant']['id'] ok(await invoke('discovery.get', {'id': rid, 'revision': original})) ok(await owner.call('sharing.revoke', {'grant_id': parent}, contract_version=2)) ok(await invoke('discovery.get', {'id': rid})) diff --git a/tests/test_single_mailbox_delivery.py b/tests/test_single_mailbox_delivery.py index 5274187d..e717ff9b 100644 --- a/tests/test_single_mailbox_delivery.py +++ b/tests/test_single_mailbox_delivery.py @@ -3,10 +3,11 @@ SMTP uses the real stdlib send_message parser and serializer with only sendmail replaced by a recorder. No socket, SMTP server or external address is contacted. """ + +import smtplib from datetime import UTC, datetime from email import policy from types import SimpleNamespace -import smtplib import pytest @@ -16,20 +17,32 @@ from msg.market.delivery_targets import email_binding, validate_address from msg.workers import mail - AMBIGUOUS = ( - 'alias,buyer@example.test', 'buyer@example.test,alias', - ',buyer@example.test', 'buyer@example.test,', - 'buyer@example.test;', 'group:buyer@example.test;', - '', 'buyer(comment)@example.test', - 'buyer@example.test(comment)', 'buyer@exam(comment)ple.test', - '"buyer"@example.test', 'buyer..alias@example.test', - 'buyer@example..test', 'buyer@example.test\x7f', + 'alias,buyer@example.test', + 'buyer@example.test,alias', + ',buyer@example.test', + 'buyer@example.test,', + 'buyer@example.test;', + 'group:buyer@example.test;', + '', + 'buyer(comment)@example.test', + 'buyer@example.test(comment)', + 'buyer@exam(comment)ple.test', + '"buyer"@example.test', + 'buyer..alias@example.test', + 'buyer@example..test', + 'buyer@example.test\x7f', ) VALID = ( - 'buyer@example.test', 'Buyer+tag@Example.TEST', 'first.last@example.test', - 'a_b-c@example.test', "o'hara@example.test", 'buyer@xn--bcher-kva.test', - 'buyer@例子.测试', '用户@example.test', '"a,b"@example.test', + 'buyer@example.test', + 'Buyer+tag@Example.TEST', + 'first.last@example.test', + 'a_b-c@example.test', + "o'hara@example.test", + 'buyer@xn--bcher-kva.test', + 'buyer@例子.测试', + '用户@example.test', + '"a,b"@example.test', ) @@ -37,8 +50,11 @@ def test_market_rejects_non_exact_single_mailbox_without_leaking_it(address): with pytest.raises(Failure) as caught: validate_address(address) - assert caught.value.as_dict() == {'code': 'invalid_email', 'retryable': False, - 'message': 'Enter one valid email address.'} + assert caught.value.as_dict() == { + 'code': 'invalid_email', + 'retryable': False, + 'message': 'Enter one valid email address.', + } assert address not in str(caught.value) @@ -48,8 +64,10 @@ def test_market_rejects_controls_and_whitespace(codepoint): validate_address('buyer' + chr(codepoint) + '@example.test') -@pytest.mark.parametrize('address', [None, 1, [], {}, '', 'buyer', '@example.test', - 'buyer@localhost', 'x' * 243 + '@example.test']) +@pytest.mark.parametrize( + 'address', + [None, 1, [], {}, '', 'buyer', '@example.test', 'buyer@localhost', 'x' * 243 + '@example.test'], +) def test_market_rejects_non_addresses(address): with pytest.raises(Failure, match='invalid_email'): validate_address(address) @@ -65,8 +83,9 @@ def test_valid_mailbox_bytes_are_not_rewritten(address): @pytest.mark.parametrize('address', AMBIGUOUS) def test_legacy_verified_row_cannot_authorize_an_ambiguous_recipient(address): - email = EmailSettings(subject_id='u_buyer', address=address, - verified_at=datetime(2026, 1, 1, tzinfo=UTC)) + email = EmailSettings( + subject_id='u_buyer', address=address, verified_at=datetime(2026, 1, 1, tzinfo=UTC) + ) tx = SimpleNamespace(one=lambda *_: (3, canonical(email))) with pytest.raises(Failure, match='invalid_email'): email_binding(tx, 'u_buyer', address) @@ -74,8 +93,9 @@ def test_legacy_verified_row_cannot_authorize_an_ambiguous_recipient(address): @pytest.fixture def smtp(monkeypatch): - seen = SimpleNamespace(connections=0, envelopes=[], messages=[], closed=0, - fail_after_data=False, refused={}) + seen = SimpleNamespace( + connections=0, envelopes=[], messages=[], closed=0, fail_after_data=False, refused={} + ) real_smtp = smtplib.SMTP class RecordingSMTP(real_smtp): @@ -92,8 +112,9 @@ def ehlo(self, name=''): def starttls(self, *, context=None): return 220, b'test' - def send_message(self, msg, from_addr=None, to_addrs=None, - mail_options=(), rcpt_options=()): + def send_message( + self, msg, from_addr=None, to_addrs=None, mail_options=(), rcpt_options=() + ): seen.envelopes.append(to_addrs) return super().send_message(msg, from_addr, to_addrs, mail_options, rcpt_options) @@ -112,16 +133,29 @@ def close(self): def sender(tls='starttls'): - return mail.SmtpSender(SimpleNamespace(enabled=True, host='smtp.invalid', port=587, - tls=tls, sender='msg@example.test', credential_file=None)) + return mail.SmtpSender( + SimpleNamespace( + enabled=True, + host='smtp.invalid', + port=587, + tls=tls, + sender='msg@example.test', + credential_file=None, + ) + ) def job(recipient, verification=False): - return SimpleNamespace(id='job_mailbox', arguments={ - 'recipient': recipient, 'recipient_subject': 'u_buyer', 'verification': verification, - 'subject': 'Verify msg email' if verification else 'msg order ready', - 'text': 'test-only notification', - }) + return SimpleNamespace( + id='job_mailbox', + arguments={ + 'recipient': recipient, + 'recipient_subject': 'u_buyer', + 'verification': verification, + 'subject': 'Verify msg email' if verification else 'msg order ready', + 'text': 'test-only notification', + }, + ) @pytest.mark.parametrize('address', AMBIGUOUS) diff --git a/tests/test_smtp_socket_acceptance.py b/tests/test_smtp_socket_acceptance.py index 4c1a2e47..dc294e8b 100644 --- a/tests/test_smtp_socket_acceptance.py +++ b/tests/test_smtp_socket_acceptance.py @@ -1,4 +1,5 @@ """Exercise SMTP over a real, certificate-verified loopback TLS socket.""" + import socket import ssl import subprocess @@ -17,10 +18,28 @@ @pytest.fixture def tls_contexts(tmp_path): cert, key = tmp_path / 'cert.pem', tmp_path / 'key.pem' - subprocess.run(['openssl', 'req', '-x509', '-newkey', 'rsa:2048', '-nodes', - '-keyout', str(key), '-out', str(cert), '-days', '1', - '-subj', '/CN=localhost', '-addext', 'subjectAltName=IP:127.0.0.1'], - check=True, capture_output=True) + subprocess.run( + [ + 'openssl', + 'req', + '-x509', + '-newkey', + 'rsa:2048', + '-nodes', + '-keyout', + str(key), + '-out', + str(cert), + '-days', + '1', + '-subj', + '/CN=localhost', + '-addext', + 'subjectAltName=IP:127.0.0.1', + ], + check=True, + capture_output=True, + ) server = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) server.load_cert_chain(cert, key) client = ssl.create_default_context(cafile=str(cert)) @@ -41,9 +60,11 @@ def serve(): with context.wrap_socket(conn, server_side=True) as conn: conn.settimeout(5) with conn.makefile('rwb') as wire: + def reply(value): wire.write(value + b'\r\n') wire.flush() + reply(b'220 localhost isolated test sink') while line := wire.readline(): commands.append(line.strip()) @@ -53,7 +74,11 @@ def reply(value): elif verb == b'MAIL': reply(b'250 sender accepted') elif verb == b'RCPT': - reply(b'550 recipient refused' if outcome == 'refused' else b'250 recipient accepted') + reply( + b'550 recipient refused' + if outcome == 'refused' + else b'250 recipient accepted' + ) elif verb == b'RSET': reply(b'250 reset') elif verb == b'DATA': @@ -83,20 +108,37 @@ def reply(value): def sender(port): - return SmtpSender(SimpleNamespace(enabled=True, host='127.0.0.1', port=port, - sender='sender@example.test', tls='tls', credential_file=None)) + return SmtpSender( + SimpleNamespace( + enabled=True, + host='127.0.0.1', + port=port, + sender='sender@example.test', + tls='tls', + credential_file=None, + ) + ) def job(): - return SimpleNamespace(id='isolated-smtp-acceptance', arguments={ - 'recipient': 'reader@example.test', 'subject': 'minimal notice', - 'text': 'Open your inbox.\n.dot-stuffed line'}) + return SimpleNamespace( + id='isolated-smtp-acceptance', + arguments={ + 'recipient': 'reader@example.test', + 'subject': 'minimal notice', + 'text': 'Open your inbox.\n.dot-stuffed line', + }, + ) @pytest.mark.asyncio -@pytest.mark.parametrize(('outcome', 'expected'), [ - ('accepted', 'sent'), ('disconnect', 'uncertain'), ('refused', 'uncertain')]) -async def test_real_tls_smtp_success_refusal_and_lost_ack(tls_contexts, monkeypatch, outcome, expected): +@pytest.mark.parametrize( + ('outcome', 'expected'), + [('accepted', 'sent'), ('disconnect', 'uncertain'), ('refused', 'uncertain')], +) +async def test_real_tls_smtp_success_refusal_and_lost_ack( + tls_contexts, monkeypatch, outcome, expected +): server, client = tls_contexts monkeypatch.setattr(ssl, 'create_default_context', lambda: client) # Keep EHLO hostname discovery independent of the host machine's DNS. @@ -105,7 +147,8 @@ async def test_real_tls_smtp_success_refusal_and_lost_ack(tls_contexts, monkeypa assert await sender(port).send(job()) == expected assert sum(command.upper().startswith(b'MAIL ') for command in commands) == 1 assert [command.lower() for command in commands if command.upper().startswith(b'RCPT ')] == [ - b'rcpt to:'] + b'rcpt to:' + ] assert len(messages) == (0 if outcome == 'refused' else 1) if messages: message = BytesParser(policy=policy.default).parsebytes(messages[0]) diff --git a/tests/test_ssh_process_identity.py b/tests/test_ssh_process_identity.py index fc44905c..72e61e8e 100644 --- a/tests/test_ssh_process_identity.py +++ b/tests/test_ssh_process_identity.py @@ -1,8 +1,9 @@ """Process names and root-owned interpreters are not an SSH authentication channel.""" + import os -from pathlib import Path import subprocess import sys +from pathlib import Path from types import SimpleNamespace import pytest @@ -32,8 +33,14 @@ def test_real_unprivileged_process_cannot_spoof_an_sshd_ancestor(name): # normally already runs as an unprivileged account; no service/user changes. identity = {'user': 65534, 'group': 65534, 'extra_groups': []} if os.geteuid() == 0 else {} environment = {**os.environ, 'PYTHONPATH': str(Path(__file__).resolve().parents[1] / 'src')} - result = subprocess.run([sys.executable, '-c', parent], env=environment, - capture_output=True, text=True, timeout=15, **identity) + result = subprocess.run( + [sys.executable, '-c', parent], + env=environment, + capture_output=True, + text=True, + timeout=15, + **identity, + ) assert result.returncode == 0, result.stdout + result.stderr assert result.stdout.strip() == 'ssh_os_isolation_required' @@ -43,15 +50,18 @@ def proc_tree(monkeypatch, tmp_path, *, owner, real_uid, effective_uid, parent=1 directory.mkdir() (directory / 'comm').write_text('sshd\n') (directory / 'status').write_text( - f'Uid:\t{real_uid}\t{effective_uid}\t{effective_uid}\t{effective_uid}\nPPid:\t{parent}\n') + f'Uid:\t{real_uid}\t{effective_uid}\t{effective_uid}\t{effective_uid}\nPPid:\t{parent}\n' + ) monkeypatch.setattr(ssh, 'Path', lambda root: tmp_path) monkeypatch.setattr(ssh.os, 'geteuid', lambda: 1000) monkeypatch.setattr(ssh.os, 'getppid', lambda: 123) original_stat = Path.stat + def stat(path, *args, **kwargs): if path == directory: return SimpleNamespace(st_uid=owner) return original_stat(path, *args, **kwargs) + monkeypatch.setattr(Path, 'stat', stat) return directory @@ -62,8 +72,9 @@ def test_privileged_sshd_ancestor_does_not_require_readable_proc_exe(monkeypatch @pytest.mark.parametrize('owner,real_uid,effective_uid', [(1000, 0, 0), (0, 1000, 0), (0, 0, 1000)]) -def test_inconsistent_or_unprivileged_sshd_identity_fails_closed(monkeypatch, tmp_path, - owner, real_uid, effective_uid): +def test_inconsistent_or_unprivileged_sshd_identity_fails_closed( + monkeypatch, tmp_path, owner, real_uid, effective_uid +): proc_tree(monkeypatch, tmp_path, owner=owner, real_uid=real_uid, effective_uid=effective_uid) with pytest.raises(Failure, match='ssh_os_isolation_required'): ssh.require_sshd_process() diff --git a/tests/test_ssh_rss.py b/tests/test_ssh_rss.py index b124bc78..6545b891 100644 --- a/tests/test_ssh_rss.py +++ b/tests/test_ssh_rss.py @@ -1,133 +1,256 @@ +from datetime import timedelta + import pytest from cryptography.hazmat.primitives import serialization from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey -from msg.core.codec import canonical,wire +from test_service import NOW, call, register + +from msg.core.codec import canonical, wire from msg.core.errors import Failure from msg.core.executor import OperationExecutor from msg.core.requests import request_for -from msg.extensions.ssh import SSHAuthenticator,parse_command,public_key +from msg.extensions.ssh import SSHAuthenticator, parse_command from msg.security.crypto import Ed25519Signer -from test_service import register,call,NOW -from datetime import timedelta def test_ssh_command_allowlist_has_no_shell_or_root_proxy(): - assert parse_command("git-upload-pack '/@alice/code.git'")==('git-upload-pack','/@alice/code.git') - assert parse_command("msg call discovery.get '{\"id\":\"/main\"}'")[0]=='call' - for command in ('','sh','bash -c id','msgd cert issue a','git-upload-pack /etc/passwd', - 'git-upload-pack /@a/x.git;id','msg call identity.register {} extra','scp -t .'): - with pytest.raises(Failure):parse_command(command) + assert parse_command("git-upload-pack '/@alice/code.git'") == ( + 'git-upload-pack', + '/@alice/code.git', + ) + assert parse_command('msg call discovery.get \'{"id":"/main"}\'')[0] == 'call' + for command in ( + '', + 'sh', + 'bash -c id', + 'msgd cert issue a', + 'git-upload-pack /etc/passwd', + 'git-upload-pack /@a/x.git;id', + 'msg call identity.register {} extra', + 'scp -t .', + ): + with pytest.raises(Failure): + parse_command(command) @pytest.mark.asyncio async def test_registered_ssh_key_binds_identity_and_ceiling(installed): - app,_=installed - key,uid,_=await register(app,'ssh-agent') - ssh=Ed25519Signer.generate() - line=Ed25519PrivateKey.from_private_bytes(ssh.private_bytes()).public_key().public_bytes( - serialization.Encoding.OpenSSH,serialization.PublicFormat.OpenSSH).decode() - pop=ssh.sign(canonical({'subject_id':uid,'public_key':line}),purpose='ssh-key-add') - grant=next(g for g in app.base_grants() if g.capability=='discovery.basic') - added=await call(app,'identity.ssh_key_add',{'public_key':line,'proof':wire(pop),'ceiling':[wire(grant)]},key=key,subject=uid) - assert added.status=='ok',wire(added) - authenticator=SSHAuthenticator(app,ssh.key_id) - executor=OperationExecutor(app.registry,app.metadata,app.contents,authenticator,app.authorizer,app.clock,app.receipt_signer) - packet=request_for('discovery.get',{'id':'/main'},app.settings.service_url,subject=uid,expires_at=NOW+timedelta(seconds=60)) - read=await executor.execute(packet) - assert read.status=='ok' and read.actor==uid,wire(read) - bad=await executor.execute(request_for('content.post_create',{'parent':'/main','body':'not authorized'}, - app.settings.service_url,subject=uid,expires_at=NOW+timedelta(seconds=60))) - assert bad.status=='error' - revoked=await call(app,'identity.ssh_key_revoke',{'key_id':ssh.key_id},key=key,subject=uid) - assert revoked.status=='ok',wire(revoked) - assert (await executor.execute(packet)).error.code=='credential_revoked' + app, _ = installed + key, uid, _ = await register(app, 'ssh-agent') + ssh = Ed25519Signer.generate() + line = ( + Ed25519PrivateKey + .from_private_bytes(ssh.private_bytes()) + .public_key() + .public_bytes(serialization.Encoding.OpenSSH, serialization.PublicFormat.OpenSSH) + .decode() + ) + pop = ssh.sign(canonical({'subject_id': uid, 'public_key': line}), purpose='ssh-key-add') + grant = next(g for g in app.base_grants() if g.capability == 'discovery.basic') + added = await call( + app, + 'identity.ssh_key_add', + {'public_key': line, 'proof': wire(pop), 'ceiling': [wire(grant)]}, + key=key, + subject=uid, + ) + assert added.status == 'ok', wire(added) + authenticator = SSHAuthenticator(app, ssh.key_id) + executor = OperationExecutor( + app.registry, + app.metadata, + app.contents, + authenticator, + app.authorizer, + app.clock, + app.receipt_signer, + ) + packet = request_for( + 'discovery.get', + {'id': '/main'}, + app.settings.service_url, + subject=uid, + expires_at=NOW + timedelta(seconds=60), + ) + read = await executor.execute(packet) + assert read.status == 'ok' and read.actor == uid, wire(read) + bad = await executor.execute( + request_for( + 'content.post_create', + {'parent': '/main', 'body': 'not authorized'}, + app.settings.service_url, + subject=uid, + expires_at=NOW + timedelta(seconds=60), + ) + ) + assert bad.status == 'error' + revoked = await call( + app, 'identity.ssh_key_revoke', {'key_id': ssh.key_id}, key=key, subject=uid + ) + assert revoked.status == 'ok', wire(revoked) + assert (await executor.execute(packet)).error.code == 'credential_revoked' + @pytest.mark.asyncio -async def test_git_reference_guard_real_commit_and_revoked_key(installed,tmp_path): - from msg.extensions.ssh_git import ReferenceGuard,guarded_command +async def test_git_reference_guard_real_commit_and_revoked_key(installed, tmp_path): from msg.extensions.repositories import NativeGitStore - app,_=installed - key,uid,_=await register(app,'ssh-git-owner') - ssh=Ed25519Signer.generate() - line=Ed25519PrivateKey.from_private_bytes(ssh.private_bytes()).public_key().public_bytes( - serialization.Encoding.OpenSSH,serialization.PublicFormat.OpenSSH).decode() - pop=ssh.sign(canonical({'subject_id':uid,'public_key':line}),purpose='ssh-key-add') - grant=next(g for g in app.base_grants() if g.capability=='git.basic') - added=await call(app,'identity.ssh_key_add',{'public_key':line,'proof':wire(pop),'ceiling':[wire(grant)]},key=key,subject=uid) - assert added.status=='ok',wire(added) - repo=await call(app,'git.create',{'parent':'/@ssh-git-owner','name':'code.git'},key=key,subject=uid) - rid=repo.resources[0].id - store=NativeGitStore(app) - import asyncio,subprocess + from msg.extensions.ssh_git import ReferenceGuard, guarded_command + + app, _ = installed + key, uid, _ = await register(app, 'ssh-git-owner') + ssh = Ed25519Signer.generate() + line = ( + Ed25519PrivateKey + .from_private_bytes(ssh.private_bytes()) + .public_key() + .public_bytes(serialization.Encoding.OpenSSH, serialization.PublicFormat.OpenSSH) + .decode() + ) + pop = ssh.sign(canonical({'subject_id': uid, 'public_key': line}), purpose='ssh-key-add') + grant = next(g for g in app.base_grants() if g.capability == 'git.basic') + added = await call( + app, + 'identity.ssh_key_add', + {'public_key': line, 'proof': wire(pop), 'ceiling': [wire(grant)]}, + key=key, + subject=uid, + ) + assert added.status == 'ok', wire(added) + repo = await call( + app, 'git.create', {'parent': '/@ssh-git-owner', 'name': 'code.git'}, key=key, subject=uid + ) + rid = repo.resources[0].id + store = NativeGitStore(app) + import asyncio + import subprocess + # Real Git objects with an empty tree; neither arbitrary shell nor fake refs. - tree=(await asyncio.to_thread(store._run,rid,'mktree',input=b'')).decode().strip() - env={**store.env,'GIT_AUTHOR_NAME':'Test','GIT_AUTHOR_EMAIL':'test@example.invalid'} - commit=subprocess.run(['git','--git-dir',str(store.path(rid)),'commit-tree',tree],input=b'initial\n',capture_output=True,env=env,check=True).stdout.decode().strip() - auth=SSHAuthenticator(app,ssh.key_id) - executor=OperationExecutor(app.registry,app.metadata,app.contents,auth,app.authorizer,app.clock,app.receipt_signer) + tree = (await asyncio.to_thread(store._run, rid, 'mktree', input=b'')).decode().strip() + env = {**store.env, 'GIT_AUTHOR_NAME': 'Test', 'GIT_AUTHOR_EMAIL': 'test@example.invalid'} + commit = ( + subprocess + .run( + ['git', '--git-dir', str(store.path(rid)), 'commit-tree', tree], + input=b'initial\n', + capture_output=True, + env=env, + check=True, + ) + .stdout.decode() + .strip() + ) + auth = SSHAuthenticator(app, ssh.key_id) + executor = OperationExecutor( + app.registry, + app.metadata, + app.contents, + auth, + app.authorizer, + app.clock, + app.receipt_signer, + ) + async def job(): - packet=request_for('git.receive',{'id':rid},app.settings.service_url,subject=uid,expires_at=NOW+timedelta(seconds=60)) - accepted=await executor.execute(packet) - assert accepted.status=='accepted',wire(accepted) - async with app.metadata.transaction(write=False) as tx:return await tx.job(accepted.data['job_id']) - initial=await job() + packet = request_for( + 'git.receive', + {'id': rid}, + app.settings.service_url, + subject=uid, + expires_at=NOW + timedelta(seconds=60), + ) + accepted = await executor.execute(packet) + assert accepted.status == 'accepted', wire(accepted) + async with app.metadata.transaction(write=False) as tx: + return await tx.job(accepted.data['job_id']) + + initial = await job() # Git 2.54+ invokes reference-transaction with a pre-lock `preparing` # phase. Exercise it explicitly so older local Git versions do not hide # compatibility regressions. - probe=ReferenceGuard(app,initial) - probe_task=asyncio.create_task(probe.run()) - changes=f"{'0'*40} {commit} refs/heads/main" - assert (await probe.message({'state':'preparing','changes':changes}))['ok'] - await probe.message({'state':'stop'}) + probe = ReferenceGuard(app, initial) + probe_task = asyncio.create_task(probe.run()) + changes = f'{"0" * 40} {commit} refs/heads/main' + assert (await probe.message({'state': 'preparing', 'changes': changes}))['ok'] + await probe.message({'state': 'stop'}) await probe_task - code=await guarded_command(app,initial,lambda _:['update-ref','refs/heads/main',commit,'0'*40]) - assert code==0 + code = await guarded_command( + app, initial, lambda _: ['update-ref', 'refs/heads/main', commit, '0' * 40] + ) + assert code == 0 async with app.metadata.transaction(write=False) as tx: - assert (await tx.job(initial.id)).state=='done' - assert (await tx.resource(rid)).generation==repo.data['generation']+1 - assert tx.one('SELECT COUNT(*) FROM audit')[0]>0 + assert (await tx.job(initial.id)).state == 'done' + assert (await tx.resource(rid)).generation == repo.data['generation'] + 1 + assert tx.one('SELECT COUNT(*) FROM audit')[0] > 0 # A real Git ref transaction that loses stdin after prepare must abort # every staged ref. This covers EOF, not host power loss or disk durability. - interrupted=await job() - before_refs,_=await store.refs(rid) + interrupted = await job() + before_refs, _ = await store.refs(rid) async with app.metadata.transaction(write=False) as tx: - before_generation=(await tx.resource(rid)).generation - commands=(f'start\ncreate refs/heads/second {commit}\n' - f'create refs/heads/third {commit}\nprepare\n').encode() - code,output,changed=await guarded_command(app,interrupted, - lambda _:['update-ref','--stdin'],input_data=commands,capture_output=True) - assert code==0 and b'prepare: ok' in output and not changed - assert (await store.refs(rid))[0]==before_refs + before_generation = (await tx.resource(rid)).generation + commands = ( + f'start\ncreate refs/heads/second {commit}\ncreate refs/heads/third {commit}\nprepare\n' + ).encode() + code, output, changed = await guarded_command( + app, + interrupted, + lambda _: ['update-ref', '--stdin'], + input_data=commands, + capture_output=True, + ) + assert code == 0 and b'prepare: ok' in output and not changed + assert (await store.refs(rid))[0] == before_refs async with app.metadata.transaction(write=False) as tx: - assert (await tx.resource(rid)).generation==before_generation - pending=await job() - await call(app,'identity.ssh_key_revoke',{'key_id':ssh.key_id},key=key,subject=uid) - denied=await guarded_command(app,pending,lambda _:['update-ref','refs/heads/other',commit,'0'*40]) - assert denied!=0 - refs,_=await store.refs(rid) - assert [r['name'] for r in refs]==['refs/heads/main'] + assert (await tx.resource(rid)).generation == before_generation + pending = await job() + await call(app, 'identity.ssh_key_revoke', {'key_id': ssh.key_id}, key=key, subject=uid) + denied = await guarded_command( + app, pending, lambda _: ['update-ref', 'refs/heads/other', commit, '0' * 40] + ) + assert denied != 0 + refs, _ = await store.refs(rid) + assert [r['name'] for r in refs] == ['refs/heads/main'] async with app.metadata.transaction(write=False) as tx: - assert (await tx.job(pending.id)).state=='failed' + assert (await tx.job(pending.id)).state == 'failed' + @pytest.mark.asyncio async def test_rss_is_read_only_acl_filtered_and_escapes_titles(installed): - import httpx from xml.etree import ElementTree as ET + + import httpx + from msg.transports.http import create_app - app,_=installed - key,uid,_=await register(app,'feed-author') - post=await call(app,'content.post_create',{'parent':'/main','body':'public'},key=key,subject=uid) - secret=await call(app,'content.post_create',{'parent':'/@feed-author/files','body':'private'},key=key,subject=uid) - async with app.metadata.transaction(write=False) as tx:before=tx.one('SELECT COUNT(*) FROM events')[0] - async with httpx.AsyncClient(transport=httpx.ASGITransport(app=create_app(app)),base_url=app.settings.service_url) as http: - feed=await http.get('/rss') - assert feed.status_code==200 - ids=[x.text for x in ET.fromstring(feed.content).findall('./channel/item/guid')] + + app, _ = installed + key, uid, _ = await register(app, 'feed-author') + post = await call( + app, 'content.post_create', {'parent': '/main', 'body': 'public'}, key=key, subject=uid + ) + secret = await call( + app, + 'content.post_create', + {'parent': '/@feed-author/files', 'body': 'private'}, + key=key, + subject=uid, + ) + async with app.metadata.transaction(write=False) as tx: + before = tx.one('SELECT COUNT(*) FROM events')[0] + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=create_app(app)), base_url=app.settings.service_url + ) as http: + feed = await http.get('/rss') + assert feed.status_code == 200 + ids = [x.text for x in ET.fromstring(feed.content).findall('./channel/item/guid')] assert post.resources[0].id in ids and secret.resources[0].id not in ids - latest=await http.get('/latest/post') - assert latest.status_code==200 - raw=await http.get('/_id/'+post.resources[0].id+'/raw',headers={'Range':'bytes=1-3'}) - assert raw.status_code==206 and raw.content==b'ubl' - bad=await http.get('/_id/'+post.resources[0].id+'/raw',headers={'Range':'bytes=100-105'}) - assert bad.status_code==416 - async with app.metadata.transaction(write=False) as tx:assert tx.one('SELECT COUNT(*) FROM events')[0]==before \ No newline at end of file + latest = await http.get('/latest/post') + assert latest.status_code == 200 + raw = await http.get( + '/_id/' + post.resources[0].id + '/raw', headers={'Range': 'bytes=1-3'} + ) + assert raw.status_code == 206 and raw.content == b'ubl' + bad = await http.get( + '/_id/' + post.resources[0].id + '/raw', headers={'Range': 'bytes=100-105'} + ) + assert bad.status_code == 416 + async with app.metadata.transaction(write=False) as tx: + assert tx.one('SELECT COUNT(*) FROM events')[0] == before diff --git a/tests/test_stable_resource_views.py b/tests/test_stable_resource_views.py index 90e9bd51..10325f50 100644 --- a/tests/test_stable_resource_views.py +++ b/tests/test_stable_resource_views.py @@ -1,4 +1,5 @@ """Stable resource-ID projections keep authorization and canonical paths separate.""" + from datetime import timedelta import httpx @@ -14,22 +15,34 @@ async def test_stable_id_views_survive_move_and_reuse_existing_projections(installed): app, _ = installed key, uid, cert = await register(app, 'stable-view-owner') - created = await call(app, 'content.post_create', {'parent': '/main', 'body': 'stable body'}, - key=key, subject=uid, certs=(cert,)) + created = await call( + app, + 'content.post_create', + {'parent': '/main', 'body': 'stable body'}, + key=key, + subject=uid, + certs=(cert,), + ) rid, revision = created.resources[0].id, created.resources[0].revision base = f'/_r/{rid}' - async with httpx.AsyncClient(transport=httpx.ASGITransport(app=create_app(app)), - base_url='http://testserver') as http: + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=create_app(app)), base_url='http://testserver' + ) as http: first = await http.get(base + '/json') assert first.status_code == 200, first.text assert first.json()['id'] == rid assert (await http.get(base + '/raw')).content == b'stable body' assert (await http.get(base + '/rev/' + revision)).status_code == 200 - moved = await call(app, 'content.move', {'id': rid, 'parent': '/intro', - 'name': 'renamed.md'}, - key=key, subject=uid, certs=(cert,), - expected=((rid, created.data['generation']),)) + moved = await call( + app, + 'content.move', + {'id': rid, 'parent': '/intro', 'name': 'renamed.md'}, + key=key, + subject=uid, + certs=(cert,), + expected=((rid, created.data['generation']),), + ) assert moved.status == 'ok' async with app.metadata.transaction(write=False) as tx: events_before = tx.one('SELECT COUNT(*) FROM events')[0] @@ -57,14 +70,28 @@ async def test_stable_id_views_survive_move_and_reuse_existing_projections(insta async def test_private_stable_views_do_not_disclose_via_head_etag_or_status(installed): app, _ = installed key, uid, cert = await register(app, 'stable-private-owner') - created = await call(app, 'content.post_create', {'parent': '/main', 'body': 'hidden body'}, - key=key, subject=uid, certs=(cert,)) + created = await call( + app, + 'content.post_create', + {'parent': '/main', 'body': 'hidden body'}, + key=key, + subject=uid, + certs=(cert,), + ) rid, revision = created.resources[0].id, created.resources[0].revision - await call(app, 'content.chmod', {'id': rid, 'mode': '0600'}, key=key, subject=uid, - certs=(cert,), expected=((rid, created.data['generation']),)) + await call( + app, + 'content.chmod', + {'id': rid, 'mode': '0600'}, + key=key, + subject=uid, + certs=(cert,), + expected=((rid, created.data['generation']),), + ) base = f'/_r/{rid}' - async with httpx.AsyncClient(transport=httpx.ASGITransport(app=create_app(app)), - base_url='http://testserver') as http: + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=create_app(app)), base_url='http://testserver' + ) as http: for suffix in ('/json', '/meta', '/raw', '/history', '/rev/' + revision): for method in (http.get, http.head): response = await method(base + suffix, headers={'If-None-Match': '*'}) @@ -75,9 +102,14 @@ async def test_private_stable_views_do_not_disclose_via_head_etag_or_status(inst assert missing.status_code == 404 assert 'etag' not in missing.headers - signed = request_for('discovery.get', {'id': rid, 'view': 'meta'}, - app.settings.service_url, signer=key, subject=uid, - expires_at=NOW + timedelta(seconds=120)) + signed = request_for( + 'discovery.get', + {'id': rid, 'view': 'meta'}, + app.settings.service_url, + signer=key, + subject=uid, + expires_at=NOW + timedelta(seconds=120), + ) headers = {'X-Msg-Request': b64(canonical(signed))} permitted = await http.get(base + '/meta', headers=headers) assert permitted.status_code == 200, permitted.text diff --git a/tests/test_stdio_cli.py b/tests/test_stdio_cli.py index 6cf19971..37de9bdb 100644 --- a/tests/test_stdio_cli.py +++ b/tests/test_stdio_cli.py @@ -3,106 +3,171 @@ import os import socket import sys -import tempfile from pathlib import Path import httpx import pytest import uvicorn +from test_service import NOW +from msg.admin.root import _approve_csr, _provision from msg.application import Application -from msg.admin.root import _provision,_approve_csr -from msg.client import ClientState,MsgClient +from msg.client import ClientState, MsgClient from msg.config import write_example -from msg.core.codec import canonical,loads +from msg.core.codec import canonical, loads from msg.transports.client import HTTPTransport from msg.transports.http import create_app from msg.transports.mcp import MCPServer from msg.transports.stdio import serve_stdio -from test_service import NOW @pytest.mark.asyncio -async def test_local_stdio_signs_tools_and_never_exposes_root(installed,tmp_path): - app,_=installed - async with httpx.AsyncClient(transport=httpx.ASGITransport(app=create_app(app)),base_url=app.settings.service_url) as http: - client=MsgClient(ClientState(tmp_path/'cli',server=app.settings.service_url), - HTTPTransport(app.settings.service_url,http=http),clock=lambda:NOW) +async def test_local_stdio_signs_tools_and_never_exposes_root(installed, tmp_path): + app, _ = installed + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=create_app(app)), base_url=app.settings.service_url + ) as http: + client = MsgClient( + ClientState(tmp_path / 'cli', server=app.settings.service_url), + HTTPTransport(app.settings.service_url, http=http), + clock=lambda: NOW, + ) await client.register('stdio-agent') - registry=await client.contract_registry() - assert registry.operation('identity.register',1).version==1 - newer=registry.operation('identity.register',2) + registry = await client.contract_registry() + assert registry.operation('identity.register', 1).version == 1 + newer = registry.operation('identity.register', 2) await registry.load_schemas((newer,)) - assert tuple(registry.schema(newer.input_schema)['required'])==( - 'handle','public_key','encryption_recipient') - mcp=MCPServer(app) - listed=[] - cursor=None + assert tuple(registry.schema(newer.input_schema)['required']) == ( + 'handle', + 'public_key', + 'encryption_recipient', + ) + mcp = MCPServer(app) + listed = [] + cursor = None while True: - page=await mcp.tools(cursor) + page = await mcp.tools(cursor) listed.extend(tool['name'] for tool in page['tools']) - cursor=page.get('nextCursor') + cursor = page.get('nextCursor') if cursor is None: break assert 'identity.register' in listed and 'identity.register@2' in listed - assert len(listed)==len(set(listed)) - request={'jsonrpc':'2.0','id':3,'method':'tools/call','params':{ - 'name':'content.post_create','arguments':{'parent':'/main','body':'stdio message'}}} - messages=[{'jsonrpc':'2.0','id':1,'method':'initialize','params':{'protocolVersion':'2025-11-25'}}, - {'jsonrpc':'2.0','id':2,'method':'tools/list'},request,request, - {'jsonrpc':'2.0','id':4,'method':'tools/call','params':{'name':'root.cert.issue','arguments':{}}}] - source=io.BytesIO(b''.join(canonical(m)+b'\n' for m in messages));target=io.BytesIO() - await serve_stdio(client,source,target) - replies=[loads(line) for line in target.getvalue().splitlines()] - assert len(replies)==5 - assert replies[2]['result']['structuredContent']['status']=='ok' + assert len(listed) == len(set(listed)) + request = { + 'jsonrpc': '2.0', + 'id': 3, + 'method': 'tools/call', + 'params': { + 'name': 'content.post_create', + 'arguments': {'parent': '/main', 'body': 'stdio message'}, + }, + } + messages = [ + { + 'jsonrpc': '2.0', + 'id': 1, + 'method': 'initialize', + 'params': {'protocolVersion': '2025-11-25'}, + }, + {'jsonrpc': '2.0', 'id': 2, 'method': 'tools/list'}, + request, + request, + { + 'jsonrpc': '2.0', + 'id': 4, + 'method': 'tools/call', + 'params': {'name': 'root.cert.issue', 'arguments': {}}, + }, + ] + source = io.BytesIO(b''.join(canonical(m) + b'\n' for m in messages)) + target = io.BytesIO() + await serve_stdio(client, source, target) + replies = [loads(line) for line in target.getvalue().splitlines()] + assert len(replies) == 5 + assert replies[2]['result']['structuredContent']['status'] == 'ok' assert replies[3]['result']['structuredContent']['replayed'] is True assert 'error' in replies[4] for tool in replies[1]['result']['tools']: assert 'packet' not in tool['inputSchema']['properties'] async with app.metadata.transaction(write=False) as tx: - assert tx.one("SELECT COUNT(*) FROM resources WHERE type='post'")[0]==1 + assert tx.one("SELECT COUNT(*) FROM resources WHERE type='post'")[0] == 1 @pytest.mark.asyncio -async def test_real_cli_process_and_stdio_over_loopback(tmp_path,pg_dsn): - sock=socket.socket();sock.bind(('127.0.0.1',0));port=sock.getsockname()[1] +async def test_real_cli_process_and_stdio_over_loopback(tmp_path, pg_dsn): + sock = socket.socket() + sock.bind(('127.0.0.1', 0)) + port = sock.getsockname()[1] sock.setblocking(False) - url=f'http://127.0.0.1:{port}' - app=Application(write_example(tmp_path/'server-etc',tmp_path/'server-data',url, - postgres_dsn=pg_dsn)) - csr,root=await _provision(app,'a-long-test-passphrase') - await _approve_csr(app,csr,root,expected_digest=None,operator='test') - server=uvicorn.Server(uvicorn.Config(create_app(app),log_level='critical',access_log=False)) - task=asyncio.create_task(server.serve(sockets=[sock])) + url = f'http://127.0.0.1:{port}' + app = Application( + write_example(tmp_path / 'server-etc', tmp_path / 'server-data', url, postgres_dsn=pg_dsn) + ) + csr, root = await _provision(app, 'a-long-test-passphrase') + await _approve_csr(app, csr, root, expected_digest=None, operator='test') + server = uvicorn.Server(uvicorn.Config(create_app(app), log_level='critical', access_log=False)) + task = asyncio.create_task(server.serve(sockets=[sock])) for _ in range(100): - if server.started:break - await asyncio.sleep(.02) + if server.started: + break + await asyncio.sleep(0.02) assert server.started - env={**os.environ,'PYTHONPATH':str(Path(__file__).parents[1]/'src')} - base=[sys.executable,'-m','msg.cli','--config-dir',str(tmp_path/'client'),'--server',url] - async def cli(*args,input=None): - process=await asyncio.create_subprocess_exec(*base,*args,env=env, - stdin=asyncio.subprocess.PIPE,stdout=asyncio.subprocess.PIPE,stderr=asyncio.subprocess.PIPE) - out,err=await asyncio.wait_for(process.communicate(input),20) - assert process.returncode==0,err.decode()+out.decode() + env = {**os.environ, 'PYTHONPATH': str(Path(__file__).parents[1] / 'src')} + base = [ + sys.executable, + '-m', + 'msg.cli', + '--config-dir', + str(tmp_path / 'client'), + '--server', + url, + ] + + async def cli(*args, input=None): + process = await asyncio.create_subprocess_exec( + *base, + *args, + env=env, + stdin=asyncio.subprocess.PIPE, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + out, err = await asyncio.wait_for(process.communicate(input), 20) + assert process.returncode == 0, err.decode() + out.decode() return [loads(line) for line in out.splitlines()] + try: - registration=(await cli('identity','new','real-cli'))[0] - assert registration['status']=='ok' - posted=(await cli('--transport','path_get','post','/main','--text','from subprocess'))[0] - assert posted['status']=='ok' - rid=posted['resources'][0]['id'] - reply=(await cli('reply',rid,'--text','reply subprocess'))[0] - assert reply['status']=='ok' - read=(await cli('read',rid,'--ack'))[0] - assert read['read']['status']=='ok' and read['ack']['status']=='ok',read - messages=[{'jsonrpc':'2.0','id':1,'method':'initialize','params':{'protocolVersion':'2025-11-25'}}, - {'jsonrpc':'2.0','id':2,'method':'tools/call','params':{'name':'content.post_create', - 'arguments':{'parent':'/main','body':'real stdio'}}}] - responses=await cli('mcp',input=b''.join(canonical(m)+b'\n' for m in messages)) - assert responses[1]['result']['structuredContent']['status']=='ok' + registration = (await cli('identity', 'new', 'real-cli'))[0] + assert registration['status'] == 'ok' + posted = ( + await cli('--transport', 'path_get', 'post', '/main', '--text', 'from subprocess') + )[0] + assert posted['status'] == 'ok' + rid = posted['resources'][0]['id'] + reply = (await cli('reply', rid, '--text', 'reply subprocess'))[0] + assert reply['status'] == 'ok' + read = (await cli('read', rid, '--ack'))[0] + assert read['read']['status'] == 'ok' and read['ack']['status'] == 'ok', read + messages = [ + { + 'jsonrpc': '2.0', + 'id': 1, + 'method': 'initialize', + 'params': {'protocolVersion': '2025-11-25'}, + }, + { + 'jsonrpc': '2.0', + 'id': 2, + 'method': 'tools/call', + 'params': { + 'name': 'content.post_create', + 'arguments': {'parent': '/main', 'body': 'real stdio'}, + }, + }, + ] + responses = await cli('mcp', input=b''.join(canonical(m) + b'\n' for m in messages)) + assert responses[1]['result']['structuredContent']['status'] == 'ok' finally: - server.should_exit=True + server.should_exit = True await task sock.close() diff --git a/tests/test_storage_commit_faults.py b/tests/test_storage_commit_faults.py index a21ae46d..e09dde8f 100644 --- a/tests/test_storage_commit_faults.py +++ b/tests/test_storage_commit_faults.py @@ -1,4 +1,5 @@ """Real PostgreSQL COMMIT rejection after durable Git/CAS publication.""" + import pytest from test_service import call, register @@ -9,12 +10,23 @@ @pytest.mark.asyncio @pytest.mark.parametrize('media', ['text/plain', 'application/octet-stream']) -async def test_deferred_commit_failure_leaves_collectable_orphan_and_retry(installed, monkeypatch, media): +async def test_deferred_commit_failure_leaves_collectable_orphan_and_retry( + installed, monkeypatch, media +): app, _ = installed key, subject, _ = await register(app, 'commit-fault') - initial = await call(app, 'file.create', {'parent': '/@commit-fault/files', - 'name': 'retained.dat', 'data': b64(b'retained bytes'), 'media_type': media}, - key=key, subject=subject) + initial = await call( + app, + 'file.create', + { + 'parent': '/@commit-fault/files', + 'name': 'retained.dat', + 'data': b64(b'retained bytes'), + 'media_type': media, + }, + key=key, + subject=subject, + ) assert initial.status == 'ok', initial.error rid = initial.resources[0].id tables = ('resources', 'revisions', 'events', 'results', 'audit', 'jobs') @@ -23,24 +35,39 @@ async def test_deferred_commit_failure_leaves_collectable_orphan_and_retry(insta before = tuple(tx.one('SELECT COUNT(*) FROM ' + table)[0] for table in tables) captured = [] commit_revision = app.contents.commit_revision + async def observe(topic, revision): result = await commit_revision(topic, revision) captured.append(revision) return result + monkeypatch.setattr(app.contents, 'commit_revision', observe) # A real deferred PostgreSQL trigger fails inside connection.commit(), after # handler, Event and idempotency rows have been prepared. No mocked principal # or fabricated connection failure can short-circuit that boundary. async with app.metadata.transaction(write=True) as tx: - tx.execute("""CREATE FUNCTION reject_test_revision_commit() RETURNS trigger AS $$ + tx.execute( + """CREATE FUNCTION reject_test_revision_commit() RETURNS trigger AS $$ BEGIN RAISE EXCEPTION 'injected_deferred_commit_failure'; END; - $$ LANGUAGE plpgsql""", write=True) - tx.execute('''CREATE CONSTRAINT TRIGGER reject_test_revision_commit + $$ LANGUAGE plpgsql""", + write=True, + ) + tx.execute( + """CREATE CONSTRAINT TRIGGER reject_test_revision_commit AFTER INSERT ON revisions DEFERRABLE INITIALLY DEFERRED - FOR EACH ROW EXECUTE FUNCTION reject_test_revision_commit()''', write=True) + FOR EACH ROW EXECUTE FUNCTION reject_test_revision_commit()""", + write=True, + ) args = {'id': rid, 'base_revision': old.id, 'data': b64(b'uncommitted bytes')} - failed = await call(app, 'file.write', args, key=key, subject=subject, - expected=((rid, initial.data['generation']),), rid='commit-fault-request') + failed = await call( + app, + 'file.write', + args, + key=key, + subject=subject, + expected=((rid, initial.data['generation']),), + rid='commit-fault-request', + ) assert failed.status == 'error', failed assert len(captured) == 1 # durable publication ran before COMMIT rejected it orphan = captured[0] @@ -61,12 +88,25 @@ async def observe(topic, revision): assert not (app.contents.binary / orphan.content.digest[7:]).exists() assert await app.contents.read_bytes(old.content) == b'retained bytes' assert await app.contents.pinned(old.content, old.id) - retry = await call(app, 'file.write', args, key=key, subject=subject, - expected=((rid, initial.data['generation']),), rid='commit-fault-request') + retry = await call( + app, + 'file.write', + args, + key=key, + subject=subject, + expected=((rid, initial.data['generation']),), + rid='commit-fault-request', + ) assert retry.status == 'ok' and not retry.replayed, retry.error # Old immutable bytes remain retained after a new revision and ordinary archive. - archived = await call(app, 'file.delete', {'id': rid}, key=key, subject=subject, - expected=((rid, retry.data['generation']),)) + archived = await call( + app, + 'file.delete', + {'id': rid}, + key=key, + subject=subject, + expected=((rid, retry.data['generation']),), + ) assert archived.status == 'ok', archived.error async with app.metadata.transaction(write=True) as tx: assert (await _collect(app, tx, grace_seconds=0))['unreferenced_contents_removed'] == 0 diff --git a/tests/test_storage_layout.py b/tests/test_storage_layout.py index dbbebe49..dd3ce0b4 100644 --- a/tests/test_storage_layout.py +++ b/tests/test_storage_layout.py @@ -17,8 +17,9 @@ async def test_new_installation_uses_documented_persistent_layout(tmp_path): assert server.staging_dir == data / 'transfers' / 'staging' assert server.service_keys_dir == data / 'service' - store = GitContentStore(server.content_dir, binary_dir=server.blob_dir, - staging_dir=server.staging_dir) + store = GitContentStore( + server.content_dir, binary_dir=server.blob_dir, staging_dir=server.staging_dir + ) binary = await store.put_bytes(b'\x00\x01', 'application/octet-stream') assert (server.blob_dir / binary.digest[7:]).read_bytes() == b'\x00\x01' assert not (server.content_dir / 'binary').exists() @@ -30,9 +31,14 @@ async def test_new_installation_uses_documented_persistent_layout(tmp_path): assert NativeGitStore(app).path('example') == server.repositories_dir / 'example.git' config = tmp_path / 'etc' / 'msgd.toml' - config.write_text('\n'.join(line for line in config.read_text().splitlines() - if not line.startswith(('repositories =', 'blobs =', - 'service_keys ='))) + '\n') + config.write_text( + '\n'.join( + line + for line in config.read_text().splitlines() + if not line.startswith(('repositories =', 'blobs =', 'service_keys =')) + ) + + '\n' + ) defaults = load_settings(tmp_path / 'etc').server assert defaults.repositories_dir == server.repositories_dir assert defaults.blob_dir == server.blob_dir diff --git a/tests/test_store_foundation.py b/tests/test_store_foundation.py index 66521a5d..202724c7 100644 --- a/tests/test_store_foundation.py +++ b/tests/test_store_foundation.py @@ -1,173 +1,299 @@ """The catalog has immutable facts, while buying and settlement remain absent.""" -import pytest + from types import SimpleNamespace +import pytest +from test_service import call, register + from msg.core.codec import b64, canonical, wire from msg.plugins.store import _body -from test_service import call, register def listing_args(**changes): - return {'name':'sample.bin', 'item_kind':'file', 'price_minor':1250000, - 'currency_id':'primary', 'quantity':2, 'delivery_mode':'managed_instant', - 'escrow_policy':'escrow-v1', 'dispute_policy':'dispute-v1', - 'terms':'The file is delivered after settlement.', **changes} + return { + 'name': 'sample.bin', + 'item_kind': 'file', + 'price_minor': 1250000, + 'currency_id': 'primary', + 'quantity': 2, + 'delivery_mode': 'managed_instant', + 'escrow_policy': 'escrow-v1', + 'dispute_policy': 'dispute-v1', + 'terms': 'The file is delivered after settlement.', + **changes, + } @pytest.mark.asyncio async def test_store_starts_empty_and_listing_requires_seller_signature(installed): app, _ = installed - anonymous=await call(app,'discovery.get',{'id':'/store'}) - assert anonymous.status=='ok',wire(anonymous) + anonymous = await call(app, 'discovery.get', {'id': '/store'}) + assert anonymous.status == 'ok', wire(anonymous) async with app.metadata.transaction(write=False) as tx: - assert tx.one("SELECT COUNT(*) FROM resources WHERE parent='t_store'")[0]==0 - assert tx.one('SELECT COUNT(*) FROM store_packages')[0]==0 - unsigned=await call(app,'store.listing_create',listing_args()) - assert unsigned.status=='error' - key,seller,_=await register(app,'catalog-seller') - created=await call(app,'store.listing_create',listing_args(),key=key,subject=seller) - assert created.status=='ok',wire(created) - listing=created.data['listing'] - assert listing['seller']==seller and listing['state']=='draft' and listing['mode']=='sale' + assert tx.one("SELECT COUNT(*) FROM resources WHERE parent='t_store'")[0] == 0 + assert tx.one('SELECT COUNT(*) FROM store_packages')[0] == 0 + unsigned = await call(app, 'store.listing_create', listing_args()) + assert unsigned.status == 'error' + key, seller, _ = await register(app, 'catalog-seller') + created = await call(app, 'store.listing_create', listing_args(), key=key, subject=seller) + assert created.status == 'ok', wire(created) + listing = created.data['listing'] + assert listing['seller'] == seller and listing['state'] == 'draft' and listing['mode'] == 'sale' assert listing['package_ref'] is None - assert listing['listing_revision']==listing['terms_revision'] - assert (await call(app,'store.listing_get',{'id':listing['listing_id']})).status=='ok' - public_resource=await call(app,'discovery.get',{'id':listing['listing_id']}) - assert public_resource.status=='ok' and listing['listing_id'] in str(public_resource.data) + assert listing['listing_revision'] == listing['terms_revision'] + assert (await call(app, 'store.listing_get', {'id': listing['listing_id']})).status == 'ok' + public_resource = await call(app, 'discovery.get', {'id': listing['listing_id']}) + assert public_resource.status == 'ok' and listing['listing_id'] in str(public_resource.data) # A draft catalog entry is not a purchase API. - assert not any(s.name in {'store.buy','store.order_create','store.escrow_fund'} - for s in app.registry.operations()) - explicit=await call(app,'store.listing_create',listing_args(mode='sale',name='sale.bin'), - key=key,subject=seller) - assert explicit.status=='ok' and explicit.data['listing']['mode']=='sale' - unsupported=await call(app,'store.listing_create',listing_args(mode='bounty',name='bounty.bin'), - key=key,subject=seller) - assert unsupported.error.code=='listing_mode_unsupported' + assert not any( + s.name in {'store.buy', 'store.order_create', 'store.escrow_fund'} + for s in app.registry.operations() + ) + explicit = await call( + app, + 'store.listing_create', + listing_args(mode='sale', name='sale.bin'), + key=key, + subject=seller, + ) + assert explicit.status == 'ok' and explicit.data['listing']['mode'] == 'sale' + unsupported = await call( + app, + 'store.listing_create', + listing_args(mode='bounty', name='bounty.bin'), + key=key, + subject=seller, + ) + assert unsupported.error.code == 'listing_mode_unsupported' async with app.metadata.transaction(write=False) as tx: - revision=await tx.revision(created.resources[0]) - persisted=await app.contents.read_bytes(revision.content) + revision = await tx.revision(created.resources[0]) + persisted = await app.contents.read_bytes(revision.content) assert b'"mode":"sale"' in persisted - assert tx.one("SELECT COUNT(*) FROM resources WHERE parent='t_store'")[0]==2 + assert tx.one("SELECT COUNT(*) FROM resources WHERE parent='t_store'")[0] == 2 @pytest.mark.asyncio async def test_legacy_listing_body_without_mode_reads_as_sale(): - old={'currency_id':'primary','price_minor':1,'quantity':1,'item_kind':'file', - 'delivery_mode':'managed_instant','package_ref':None,'escrow_policy':'v1', - 'dispute_policy':'v1','terms':'old','expires_at':None,'state':'draft'} - revision=SimpleNamespace(id='v_legacy',content=object()) + old = { + 'currency_id': 'primary', + 'price_minor': 1, + 'quantity': 1, + 'item_kind': 'file', + 'delivery_mode': 'managed_instant', + 'package_ref': None, + 'escrow_policy': 'v1', + 'dispute_policy': 'v1', + 'terms': 'old', + 'expires_at': None, + 'state': 'draft', + } + revision = SimpleNamespace(id='v_legacy', content=object()) + class LegacyTx: async def revision(self, ref): - assert ref.id=='r_legacy' + assert ref.id == 'r_legacy' return revision + class LegacyContent: async def read_bytes(self, blob): assert blob is revision.content return canonical(old) - body,found=await _body(SimpleNamespace(contents=LegacyContent()),LegacyTx(), - SimpleNamespace(id='r_legacy')) - assert found is revision and body=={'mode':'sale',**old} + + body, found = await _body( + SimpleNamespace(contents=LegacyContent()), LegacyTx(), SimpleNamespace(id='r_legacy') + ) + assert found is revision and body == {'mode': 'sale', **old} @pytest.mark.asyncio async def test_deposit_is_immutable_and_only_seller_can_read_payload(installed): - app,_=installed - key,seller,_=await register(app,'deposit-seller') - other_key,other,_=await register(app,'deposit-other') - listing=await call(app,'store.listing_create',listing_args(),key=key,subject=seller) - rid=listing.resources[0].id - version=listing.resources[0].revision - file=await call(app,'content.file_put',{'parent':'/@deposit-seller/files', - 'name':'private.bin','data':b64(b'package secret'), - 'media_type':'application/octet-stream'},key=key,subject=seller) - assert file.status=='ok',wire(file) - payload={'id':file.resources[0].id,'revision':file.resources[0].revision} - deposit=await call(app,'store.package_deposit',{'listing_id':rid, - 'listing_revision':version,'manifest':{'format':1,'name':'private.bin'}, - 'payload_refs':[payload]},key=key,subject=seller,rid='deposit-once') - assert deposit.status=='ok',wire(deposit) - replay=await call(app,'store.package_deposit',{'listing_id':rid, - 'listing_revision':version,'manifest':{'format':1,'name':'private.bin'}, - 'payload_refs':[payload]},key=key,subject=seller,rid='deposit-once') - assert replay.status=='ok' and replay.replayed - package_id=deposit.data['package']['id'] - first=await call(app,'store.package_get',{'id':package_id},key=key,subject=seller) - assert first.status=='ok' and first.data['package']['payload_refs'][0]['id']==payload['id'] - hidden=await call(app,'store.package_get',{'id':package_id},key=other_key,subject=other) - absent=await call(app,'store.package_get',{'id':'pkg_does_not_exist'}, - key=other_key,subject=other) - assert hidden.error.code==absent.error.code=='package_not_found' + app, _ = installed + key, seller, _ = await register(app, 'deposit-seller') + other_key, other, _ = await register(app, 'deposit-other') + listing = await call(app, 'store.listing_create', listing_args(), key=key, subject=seller) + rid = listing.resources[0].id + version = listing.resources[0].revision + file = await call( + app, + 'content.file_put', + { + 'parent': '/@deposit-seller/files', + 'name': 'private.bin', + 'data': b64(b'package secret'), + 'media_type': 'application/octet-stream', + }, + key=key, + subject=seller, + ) + assert file.status == 'ok', wire(file) + payload = {'id': file.resources[0].id, 'revision': file.resources[0].revision} + deposit = await call( + app, + 'store.package_deposit', + { + 'listing_id': rid, + 'listing_revision': version, + 'manifest': {'format': 1, 'name': 'private.bin'}, + 'payload_refs': [payload], + }, + key=key, + subject=seller, + rid='deposit-once', + ) + assert deposit.status == 'ok', wire(deposit) + replay = await call( + app, + 'store.package_deposit', + { + 'listing_id': rid, + 'listing_revision': version, + 'manifest': {'format': 1, 'name': 'private.bin'}, + 'payload_refs': [payload], + }, + key=key, + subject=seller, + rid='deposit-once', + ) + assert replay.status == 'ok' and replay.replayed + package_id = deposit.data['package']['id'] + first = await call(app, 'store.package_get', {'id': package_id}, key=key, subject=seller) + assert first.status == 'ok' and first.data['package']['payload_refs'][0]['id'] == payload['id'] + hidden = await call(app, 'store.package_get', {'id': package_id}, key=other_key, subject=other) + absent = await call( + app, 'store.package_get', {'id': 'pkg_does_not_exist'}, key=other_key, subject=other + ) + assert hidden.error.code == absent.error.code == 'package_not_found' assert 'package secret' not in str(hidden.data) - update=await call(app,'store.listing_update',{'id':rid,'package_ref':package_id, - 'state':'active'},key=key,subject=seller, - expected=((rid,listing.data['generation']),)) - assert update.status=='ok',wire(update) - old=await call(app,'store.listing_get',{'id':rid,'revision':version}) - assert old.status=='ok' and old.data['listing']['package_ref'] is None - assert old.data['listing']['price_minor']==1250000 - newer=await call(app,'store.listing_update',{'id':rid,'price_minor':1500000, - 'terms':'Revised terms.'},key=key,subject=seller, - expected=((rid,update.data['generation']),)) - assert newer.status=='ok',wire(newer) - again=await call(app,'store.package_get',{'id':package_id},key=key,subject=seller) - assert again.data['package']['digest']==first.data['package']['digest'] - assert again.data['package']['listing_revision']==version + update = await call( + app, + 'store.listing_update', + {'id': rid, 'package_ref': package_id, 'state': 'active'}, + key=key, + subject=seller, + expected=((rid, listing.data['generation']),), + ) + assert update.status == 'ok', wire(update) + old = await call(app, 'store.listing_get', {'id': rid, 'revision': version}) + assert old.status == 'ok' and old.data['listing']['package_ref'] is None + assert old.data['listing']['price_minor'] == 1250000 + newer = await call( + app, + 'store.listing_update', + {'id': rid, 'price_minor': 1500000, 'terms': 'Revised terms.'}, + key=key, + subject=seller, + expected=((rid, update.data['generation']),), + ) + assert newer.status == 'ok', wire(newer) + again = await call(app, 'store.package_get', {'id': package_id}, key=key, subject=seller) + assert again.data['package']['digest'] == first.data['package']['digest'] + assert again.data['package']['listing_revision'] == version assert old.data['listing']['terms_digest'] != newer.data['listing']['terms_digest'] @pytest.mark.asyncio async def test_listing_cas_payload_ownership_and_no_secret_leak(installed): - app,_=installed - seller_key,seller,_=await register(app,'catalog-owner') - other_key,other,_=await register(app,'catalog-intruder') - listing=await call(app,'store.listing_create',listing_args(),key=seller_key,subject=seller) - rid=listing.resources[0].id - generation=listing.data['generation'] - stale=await call(app,'store.listing_update',{'id':rid,'price_minor':2}, - key=seller_key,subject=seller,expected=((rid,generation+1),)) - assert stale.error.code=='generation_conflict' - forbidden=await call(app,'store.listing_update',{'id':rid,'price_minor':2}, - key=other_key,subject=other,expected=((rid,generation),)) - missing=await call(app,'store.listing_update',{'id':'r_absent','price_minor':2}, - key=other_key,subject=other) - assert forbidden.error.code==missing.error.code=='listing_not_found' - foreign=await call(app,'content.file_put',{'parent':'/@catalog-intruder/files', - 'name':'private.txt','data':b64(b'not yours')},key=other_key,subject=other) - deposit=await call(app,'store.package_deposit',{'listing_id':rid, - 'listing_revision':listing.resources[0].revision,'manifest':{}, - 'payload_refs':[{'id':foreign.resources[0].id, - 'revision':foreign.resources[0].revision}]}, - key=seller_key,subject=seller) - assert deposit.error.code=='payload_not_owned' + app, _ = installed + seller_key, seller, _ = await register(app, 'catalog-owner') + other_key, other, _ = await register(app, 'catalog-intruder') + listing = await call( + app, 'store.listing_create', listing_args(), key=seller_key, subject=seller + ) + rid = listing.resources[0].id + generation = listing.data['generation'] + stale = await call( + app, + 'store.listing_update', + {'id': rid, 'price_minor': 2}, + key=seller_key, + subject=seller, + expected=((rid, generation + 1),), + ) + assert stale.error.code == 'generation_conflict' + forbidden = await call( + app, + 'store.listing_update', + {'id': rid, 'price_minor': 2}, + key=other_key, + subject=other, + expected=((rid, generation),), + ) + missing = await call( + app, + 'store.listing_update', + {'id': 'r_absent', 'price_minor': 2}, + key=other_key, + subject=other, + ) + assert forbidden.error.code == missing.error.code == 'listing_not_found' + foreign = await call( + app, + 'content.file_put', + {'parent': '/@catalog-intruder/files', 'name': 'private.txt', 'data': b64(b'not yours')}, + key=other_key, + subject=other, + ) + deposit = await call( + app, + 'store.package_deposit', + { + 'listing_id': rid, + 'listing_revision': listing.resources[0].revision, + 'manifest': {}, + 'payload_refs': [ + {'id': foreign.resources[0].id, 'revision': foreign.resources[0].revision} + ], + }, + key=seller_key, + subject=seller, + ) + assert deposit.error.code == 'payload_not_owned' async with app.metadata.transaction(write=False) as tx: - assert tx.one('SELECT COUNT(*) FROM store_packages')[0]==0 + assert tx.one('SELECT COUNT(*) FROM store_packages')[0] == 0 @pytest.mark.asyncio async def test_store_namespace_and_listing_metadata_need_store_operations(installed): - app,_=installed - key,seller,_=await register(app,'catalog-boundary') - for operation,args in ( - ('content.post_create',{'parent':'/store','body':'unrelated post'}), - ('content.file_put',{'parent':'/store','name':'unrelated.bin','data':b64(b'x')}), - ('content.topic_create',{'parent':'/store','name':'unrelated'}), + app, _ = installed + key, seller, _ = await register(app, 'catalog-boundary') + for operation, args in ( + ('content.post_create', {'parent': '/store', 'body': 'unrelated post'}), + ('content.file_put', {'parent': '/store', 'name': 'unrelated.bin', 'data': b64(b'x')}), + ('content.topic_create', {'parent': '/store', 'name': 'unrelated'}), ): - denied=await call(app,operation,args,key=key,subject=seller) - assert denied.error.code=='store_controlled_resource',wire(denied) - ordinary=await call(app,'content.post_create',{'parent':'/main','body':'ordinary'}, - key=key,subject=seller) - assert ordinary.status=='ok',wire(ordinary) - moved=await call(app,'content.move',{'id':ordinary.resources[0].id, - 'parent':'/store'},key=key,subject=seller, - expected=((ordinary.resources[0].id,ordinary.data['generation']),)) - assert moved.error.code=='store_controlled_resource' - listing=await call(app,'store.listing_create',listing_args(),key=key,subject=seller) - assert listing.status=='ok',wire(listing) - rid=listing.resources[0].id - changed=await call(app,'content.chmod',{'id':rid,'mode':'0777'}, - key=key,subject=seller, - expected=((rid,listing.data['generation']),)) - assert changed.error.code=='controlled_resource' - changed_group=await call(app,'content.chgrp',{'id':rid,'group':'/&public'}, - key=key,subject=seller,expected=((rid,listing.data['generation']),)) - assert changed_group.error.code=='controlled_resource' + denied = await call(app, operation, args, key=key, subject=seller) + assert denied.error.code == 'store_controlled_resource', wire(denied) + ordinary = await call( + app, 'content.post_create', {'parent': '/main', 'body': 'ordinary'}, key=key, subject=seller + ) + assert ordinary.status == 'ok', wire(ordinary) + moved = await call( + app, + 'content.move', + {'id': ordinary.resources[0].id, 'parent': '/store'}, + key=key, + subject=seller, + expected=((ordinary.resources[0].id, ordinary.data['generation']),), + ) + assert moved.error.code == 'store_controlled_resource' + listing = await call(app, 'store.listing_create', listing_args(), key=key, subject=seller) + assert listing.status == 'ok', wire(listing) + rid = listing.resources[0].id + changed = await call( + app, + 'content.chmod', + {'id': rid, 'mode': '0777'}, + key=key, + subject=seller, + expected=((rid, listing.data['generation']),), + ) + assert changed.error.code == 'controlled_resource' + changed_group = await call( + app, + 'content.chgrp', + {'id': rid, 'group': '/&public'}, + key=key, + subject=seller, + expected=((rid, listing.data['generation']),), + ) + assert changed_group.error.code == 'controlled_resource' diff --git a/tests/test_subject_read_aliases.py b/tests/test_subject_read_aliases.py index 87573064..529b6ce1 100644 --- a/tests/test_subject_read_aliases.py +++ b/tests/test_subject_read_aliases.py @@ -1,98 +1,137 @@ """Short and long subject paths share live read authorization and content.""" + from datetime import timedelta import httpx import pytest from cryptography.hazmat.primitives import serialization from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey +from test_service import NOW, call, register -from msg.core.codec import b64, canonical +from msg.core.codec import b64, canonical, wire from msg.core.requests import request_for -from msg.transports.http import create_app -from msg.security.crypto import Ed25519Signer from msg.security.capabilities import grant_for -from msg.core.codec import wire -from test_service import NOW, call, register +from msg.security.crypto import Ed25519Signer +from msg.transports.http import create_app @pytest.mark.asyncio async def test_resource_backed_subject_aliases_share_content_cache_and_permissions(installed): app, _ = installed key, user, certificate_id = await register(app, 'alias-owner') - ssh=Ed25519Signer.generate() - public=Ed25519PrivateKey.from_private_bytes(ssh.private_bytes()).public_key().public_bytes( - serialization.Encoding.OpenSSH,serialization.PublicFormat.OpenSSH).decode() - proof=ssh.sign(canonical({'subject_id':user,'public_key':public}),purpose='ssh-key-add') - added=await call(app,'identity.ssh_key_add',{'public_key':public,'proof':wire(proof), - 'ceiling':wire((grant_for(app.registry.capability('git.basic')),))},key=key,subject=user) - assert added.status=='ok',wire(added) - stored=await call(app,'keystore.put',{'name':'sealed','format':'age', - 'ciphertext':b64(b'age-encryption.org/v1\npayload')},key=key,subject=user) - assert stored.status=='ok',wire(stored) - before=None + ssh = Ed25519Signer.generate() + public = ( + Ed25519PrivateKey + .from_private_bytes(ssh.private_bytes()) + .public_key() + .public_bytes(serialization.Encoding.OpenSSH, serialization.PublicFormat.OpenSSH) + .decode() + ) + proof = ssh.sign(canonical({'subject_id': user, 'public_key': public}), purpose='ssh-key-add') + added = await call( + app, + 'identity.ssh_key_add', + { + 'public_key': public, + 'proof': wire(proof), + 'ceiling': wire((grant_for(app.registry.capability('git.basic')),)), + }, + key=key, + subject=user, + ) + assert added.status == 'ok', wire(added) + stored = await call( + app, + 'keystore.put', + {'name': 'sealed', 'format': 'age', 'ciphertext': b64(b'age-encryption.org/v1\npayload')}, + key=key, + subject=user, + ) + assert stored.status == 'ok', wire(stored) + before = None async with app.metadata.transaction(write=False) as tx: - before=(tx.one('SELECT COUNT(*) FROM events')[0], - tx.one('SELECT SUM(generation) FROM resources')[0]) - keystore=await tx.resolve('/@alias-owner/keystore') - signed=request_for('discovery.get',{'id':keystore},app.settings.service_url, - signer=key,subject=user,expires_at=NOW+timedelta(seconds=120)) - headers={'X-Msg-Request':b64(canonical(signed))} - async with httpx.AsyncClient(transport=httpx.ASGITransport(app=create_app(app)), - base_url='http://testserver') as http: - for short,long in (('cert','certificates'),('ssh','ssh-keys')): - left=await http.get('/@alias-owner/'+short+'/json') - right=await http.get('/@alias-owner/'+long+'/json') - assert left.status_code==right.status_code==200 - assert left.content==right.content - assert left.headers['etag']==right.headers['etag'] + before = ( + tx.one('SELECT COUNT(*) FROM events')[0], + tx.one('SELECT SUM(generation) FROM resources')[0], + ) + keystore = await tx.resolve('/@alias-owner/keystore') + signed = request_for( + 'discovery.get', + {'id': keystore}, + app.settings.service_url, + signer=key, + subject=user, + expires_at=NOW + timedelta(seconds=120), + ) + headers = {'X-Msg-Request': b64(canonical(signed))} + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=create_app(app)), base_url='http://testserver' + ) as http: + for short, long in (('cert', 'certificates'), ('ssh', 'ssh-keys')): + left = await http.get('/@alias-owner/' + short + '/json') + right = await http.get('/@alias-owner/' + long + '/json') + assert left.status_code == right.status_code == 200 + assert left.content == right.content + assert left.headers['etag'] == right.headers['etag'] assert not left.is_redirect and not right.is_redirect - cached_short=await http.get('/@alias-owner/'+short+'/json', - headers={'If-None-Match':left.headers['etag']}) - cached_long=await http.get('/@alias-owner/'+long+'/json', - headers={'If-None-Match':right.headers['etag']}) - assert cached_short.status_code==cached_long.status_code==304 - if short=='cert': - assert left.json()['path']=='/@alias-owner/'+short - detail=await http.get('/@alias-owner/cert/'+certificate_id) - legacy=await http.get('/@alias-owner/certificates/'+certificate_id) - assert detail.status_code==legacy.status_code==200 - assert detail.content==legacy.content + cached_short = await http.get( + '/@alias-owner/' + short + '/json', headers={'If-None-Match': left.headers['etag']} + ) + cached_long = await http.get( + '/@alias-owner/' + long + '/json', headers={'If-None-Match': right.headers['etag']} + ) + assert cached_short.status_code == cached_long.status_code == 304 + if short == 'cert': + assert left.json()['path'] == '/@alias-owner/' + short + detail = await http.get('/@alias-owner/cert/' + certificate_id) + legacy = await http.get('/@alias-owner/certificates/' + certificate_id) + assert detail.status_code == legacy.status_code == 200 + assert detail.content == legacy.content else: - assert [item['key_id'] for item in left.json()['keys']]==[added.data['key_id']] - detail=await http.get('/@alias-owner/ssh/'+added.data['key_id']) - legacy=await http.get('/@alias-owner/ssh-keys/'+added.data['key_id']) - assert detail.status_code==legacy.status_code==200 - assert detail.content==legacy.content - for short,long in (('ks','keystore'),): - denied_short=await http.get('/@alias-owner/'+short+'/json') - denied_long=await http.get('/@alias-owner/'+long+'/json') - assert denied_short.status_code==denied_long.status_code==403 - assert denied_short.json()['error']['code']==denied_long.json()['error']['code'] - left=await http.get('/@alias-owner/'+short+'/json',headers=headers) - right=await http.get('/@alias-owner/'+long+'/json',headers=headers) - assert left.status_code==right.status_code==200 - assert left.content==right.content - assert left.headers['etag']==right.headers['etag'] - assert left.json()['path']=='/@alias-owner/ks' - cached=await http.get('/@alias-owner/keystore/json', - headers={**headers,'If-None-Match':left.headers['etag']}) - assert cached.status_code==304 - head=await http.head('/@alias-owner/ks/json',headers=headers) - assert head.status_code==200 and head.headers['etag']==left.headers['etag'] - entry_packet=request_for('discovery.get',{'id':stored.resources[0].id}, - app.settings.service_url,signer=key,subject=user, - expires_at=NOW+timedelta(seconds=120)) - entry_headers={'X-Msg-Request':b64(canonical(entry_packet))} - entry_short=await http.get('/@alias-owner/ks/sealed/json',headers=entry_headers) - entry_long=await http.get('/@alias-owner/keystore/sealed/json',headers=entry_headers) - assert entry_short.status_code==entry_long.status_code==200 - assert entry_short.content==entry_long.content - assert entry_short.json()['path']=='/@alias-owner/ks/sealed' - anonymous=await http.get('/@alias-owner/ks/sealed/meta') - assert anonymous.status_code==403 + assert [item['key_id'] for item in left.json()['keys']] == [added.data['key_id']] + detail = await http.get('/@alias-owner/ssh/' + added.data['key_id']) + legacy = await http.get('/@alias-owner/ssh-keys/' + added.data['key_id']) + assert detail.status_code == legacy.status_code == 200 + assert detail.content == legacy.content + for short, long in (('ks', 'keystore'),): + denied_short = await http.get('/@alias-owner/' + short + '/json') + denied_long = await http.get('/@alias-owner/' + long + '/json') + assert denied_short.status_code == denied_long.status_code == 403 + assert denied_short.json()['error']['code'] == denied_long.json()['error']['code'] + left = await http.get('/@alias-owner/' + short + '/json', headers=headers) + right = await http.get('/@alias-owner/' + long + '/json', headers=headers) + assert left.status_code == right.status_code == 200 + assert left.content == right.content + assert left.headers['etag'] == right.headers['etag'] + assert left.json()['path'] == '/@alias-owner/ks' + cached = await http.get( + '/@alias-owner/keystore/json', + headers={**headers, 'If-None-Match': left.headers['etag']}, + ) + assert cached.status_code == 304 + head = await http.head('/@alias-owner/ks/json', headers=headers) + assert head.status_code == 200 and head.headers['etag'] == left.headers['etag'] + entry_packet = request_for( + 'discovery.get', + {'id': stored.resources[0].id}, + app.settings.service_url, + signer=key, + subject=user, + expires_at=NOW + timedelta(seconds=120), + ) + entry_headers = {'X-Msg-Request': b64(canonical(entry_packet))} + entry_short = await http.get('/@alias-owner/ks/sealed/json', headers=entry_headers) + entry_long = await http.get('/@alias-owner/keystore/sealed/json', headers=entry_headers) + assert entry_short.status_code == entry_long.status_code == 200 + assert entry_short.content == entry_long.content + assert entry_short.json()['path'] == '/@alias-owner/ks/sealed' + anonymous = await http.get('/@alias-owner/ks/sealed/meta') + assert anonymous.status_code == 403 async with app.metadata.transaction(write=False) as tx: - assert before==(tx.one('SELECT COUNT(*) FROM events')[0], - tx.one('SELECT SUM(generation) FROM resources')[0]) + assert before == ( + tx.one('SELECT COUNT(*) FROM events')[0], + tx.one('SELECT SUM(generation) FROM resources')[0], + ) @pytest.mark.asyncio @@ -101,77 +140,117 @@ async def test_operation_backed_subject_aliases_use_same_live_handler(installed) key, user, _ = await register(app, 'mailbox-alias') other_key, other, _ = await register(app, 'mailbox-other') for index in range(2): - post=await call(app,'content.post_create',{'parent':'/main','body':f'notice {index}'}, - key=other_key,subject=other) - sent=await call(app,'communication.send', - {'recipient':user,'resource':wire(post.resources[0])}, - key=other_key,subject=other) - assert sent.status=='ok',wire(sent) + post = await call( + app, + 'content.post_create', + {'parent': '/main', 'body': f'notice {index}'}, + key=other_key, + subject=other, + ) + sent = await call( + app, + 'communication.send', + {'recipient': user, 'resource': wire(post.resources[0])}, + key=other_key, + subject=other, + ) + assert sent.status == 'ok', wire(sent) async with app.metadata.transaction(write=False) as tx: - before=(tx.one('SELECT COUNT(*) FROM events')[0], - tx.one('SELECT COUNT(*) FROM jobs')[0], - tx.one('SELECT COUNT(*) FROM revisions')[0], - tx.one('SELECT SUM(generation) FROM resources')[0]) - async with httpx.AsyncClient(transport=httpx.ASGITransport(app=create_app(app)), - base_url='http://testserver') as http: - for short,long,operation in (('in','inbox','communication.inbox'), - ('out','outbox','communication.outbox'), - ('dm','dm','communication.dm_list')): - packet=request_for(operation,{},app.settings.service_url, - signer=key,subject=user,expires_at=NOW+timedelta(seconds=120)) - headers={'X-Msg-Request':b64(canonical(packet))} - left=await http.get('/@mailbox-alias/'+short,headers=headers) - right=await http.get('/@mailbox-alias/'+long,headers=headers) - assert left.status_code==right.status_code==200,(left.text,right.text) - assert left.content==right.content - assert left.headers['etag']==right.headers['etag'] + before = ( + tx.one('SELECT COUNT(*) FROM events')[0], + tx.one('SELECT COUNT(*) FROM jobs')[0], + tx.one('SELECT COUNT(*) FROM revisions')[0], + tx.one('SELECT SUM(generation) FROM resources')[0], + ) + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=create_app(app)), base_url='http://testserver' + ) as http: + for short, long, operation in ( + ('in', 'inbox', 'communication.inbox'), + ('out', 'outbox', 'communication.outbox'), + ('dm', 'dm', 'communication.dm_list'), + ): + packet = request_for( + operation, + {}, + app.settings.service_url, + signer=key, + subject=user, + expires_at=NOW + timedelta(seconds=120), + ) + headers = {'X-Msg-Request': b64(canonical(packet))} + left = await http.get('/@mailbox-alias/' + short, headers=headers) + right = await http.get('/@mailbox-alias/' + long, headers=headers) + assert left.status_code == right.status_code == 200, (left.text, right.text) + assert left.content == right.content + assert left.headers['etag'] == right.headers['etag'] assert not left.is_redirect and not right.is_redirect - cached=await http.get('/@mailbox-alias/'+long, - headers={**headers,'If-None-Match':left.headers['etag']}) - assert cached.status_code==304 - denied=await http.get('/@mailbox-other/'+short,headers=headers) - assert denied.status_code==403 - inbox=request_for('communication.inbox',{'limit':1},app.settings.service_url, - signer=key,subject=user,expires_at=NOW+timedelta(seconds=120)) - paged_headers={'X-Msg-Request':b64(canonical(inbox))} - first=await http.get('/@mailbox-alias/in?limit=1',headers=paged_headers) - old=await http.get('/@mailbox-alias/inbox?limit=1',headers=paged_headers) - assert first.status_code==old.status_code==200 - assert first.content==old.content and first.json().get('cursor') - for short,long in (('pk','pubkey'),('k','keys'),('ek','encryption-key'), - ('e','encryption-keys'),('ach','achievements')): - public_short=await http.get('/@mailbox-alias/'+short) - public_long=await http.get('/@mailbox-alias/'+long) - assert public_short.status_code==public_long.status_code==200 - assert public_short.content==public_long.content - assert public_short.headers['etag']==public_long.headers['etag'] + cached = await http.get( + '/@mailbox-alias/' + long, + headers={**headers, 'If-None-Match': left.headers['etag']}, + ) + assert cached.status_code == 304 + denied = await http.get('/@mailbox-other/' + short, headers=headers) + assert denied.status_code == 403 + inbox = request_for( + 'communication.inbox', + {'limit': 1}, + app.settings.service_url, + signer=key, + subject=user, + expires_at=NOW + timedelta(seconds=120), + ) + paged_headers = {'X-Msg-Request': b64(canonical(inbox))} + first = await http.get('/@mailbox-alias/in?limit=1', headers=paged_headers) + old = await http.get('/@mailbox-alias/inbox?limit=1', headers=paged_headers) + assert first.status_code == old.status_code == 200 + assert first.content == old.content and first.json().get('cursor') + for short, long in ( + ('pk', 'pubkey'), + ('k', 'keys'), + ('ek', 'encryption-key'), + ('e', 'encryption-keys'), + ('ach', 'achievements'), + ): + public_short = await http.get('/@mailbox-alias/' + short) + public_long = await http.get('/@mailbox-alias/' + long) + assert public_short.status_code == public_long.status_code == 200 + assert public_short.content == public_long.content + assert public_short.headers['etag'] == public_long.headers['etag'] assert not public_short.is_redirect and not public_long.is_redirect - assert public_short.json()['path']=='/@mailbox-alias/'+short - cached=await http.get('/@mailbox-alias/'+long, - headers={'If-None-Match':public_short.headers['etag']}) - assert cached.status_code==304 - if short in {'pk','ek','k','e'}: - key_id=(public_short.json()['key_id'] if short in {'pk','ek'} - else public_short.json()['keys'][0]['key_id']) - by_id=await http.get('/@mailbox-alias/'+short+'/'+key_id) - old_by_id=await http.get('/@mailbox-alias/'+long+'/'+key_id) - assert by_id.status_code==old_by_id.status_code==200 - assert by_id.content==old_by_id.content + assert public_short.json()['path'] == '/@mailbox-alias/' + short + cached = await http.get( + '/@mailbox-alias/' + long, headers={'If-None-Match': public_short.headers['etag']} + ) + assert cached.status_code == 304 + if short in {'pk', 'ek', 'k', 'e'}: + key_id = ( + public_short.json()['key_id'] + if short in {'pk', 'ek'} + else public_short.json()['keys'][0]['key_id'] + ) + by_id = await http.get('/@mailbox-alias/' + short + '/' + key_id) + old_by_id = await http.get('/@mailbox-alias/' + long + '/' + key_id) + assert by_id.status_code == old_by_id.status_code == 200 + assert by_id.content == old_by_id.content async with app.metadata.transaction(write=False) as tx: - assert before==(tx.one('SELECT COUNT(*) FROM events')[0], - tx.one('SELECT COUNT(*) FROM jobs')[0], - tx.one('SELECT COUNT(*) FROM revisions')[0], - tx.one('SELECT SUM(generation) FROM resources')[0]) + assert before == ( + tx.one('SELECT COUNT(*) FROM events')[0], + tx.one('SELECT COUNT(*) FROM jobs')[0], + tx.one('SELECT COUNT(*) FROM revisions')[0], + tx.one('SELECT SUM(generation) FROM resources')[0], + ) @pytest.mark.asyncio async def test_key_aliases_fail_closed_for_missing_key_id(installed): app, _ = installed await register(app, 'missing-key-view') - async with httpx.AsyncClient(transport=httpx.ASGITransport(app=create_app(app)), - base_url='http://testserver') as http: - for short,long in (('pk','pubkey'),('ek','encryption-key')): - left=await http.get('/@missing-key-view/'+short+'/k_missing') - right=await http.get('/@missing-key-view/'+long+'/k_missing') - assert left.status_code==right.status_code - assert left.json()['error']['code']==right.json()['error']['code'] + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=create_app(app)), base_url='http://testserver' + ) as http: + for short, long in (('pk', 'pubkey'), ('ek', 'encryption-key')): + left = await http.get('/@missing-key-view/' + short + '/k_missing') + right = await http.get('/@missing-key-view/' + long + '/k_missing') + assert left.status_code == right.status_code + assert left.json()['error']['code'] == right.json()['error']['code'] diff --git a/tests/test_subject_receipts.py b/tests/test_subject_receipts.py index 7d28c0fe..09cf4b1c 100644 --- a/tests/test_subject_receipts.py +++ b/tests/test_subject_receipts.py @@ -1,193 +1,263 @@ """/@user/receipts projects the caller's persisted OperationResults read-only.""" + from datetime import timedelta import httpx import pytest +from test_service import NOW, call, register from msg.core.codec import b64, canonical, digest, wire from msg.core.requests import request_for from msg.transports.http import create_app -from test_service import NOW, call, register PROJECTION = {'request_id', 'operation', 'status', 'committed_at', 'resources', 'signed'} def _headers(app, operation, args, key, subject): - packet=request_for(operation,args,app.settings.service_url,signer=key, - subject=subject,expires_at=NOW+timedelta(seconds=120)) - return {'X-Msg-Request':b64(canonical(packet))} + packet = request_for( + operation, + args, + app.settings.service_url, + signer=key, + subject=subject, + expires_at=NOW + timedelta(seconds=120), + ) + return {'X-Msg-Request': b64(canonical(packet))} def _state(tx): - counts=tuple(tx.one(f'SELECT COUNT(*) FROM {table}')[0] for table in - ('results','audit','jobs','events','messages','revisions')) - results=digest([list(row) for row in tx.rows( - 'SELECT subject,request_id,digest,body FROM results ORDER BY subject,request_id')]) - generations=tx.one('SELECT SUM(generation) FROM resources')[0] - return counts,results,generations + counts = tuple( + tx.one(f'SELECT COUNT(*) FROM {table}')[0] + for table in ('results', 'audit', 'jobs', 'events', 'messages', 'revisions') + ) + results = digest([ + list(row) + for row in tx.rows( + 'SELECT subject,request_id,digest,body FROM results ORDER BY subject,request_id' + ) + ]) + generations = tx.one('SELECT SUM(generation) FROM resources')[0] + return counts, results, generations @pytest.mark.asyncio async def test_subject_receipts_are_owner_only_read_only_projection(installed): - app,_=installed - alice_key,alice,_=await register(app,'receipt-alice') - bob_key,bob,_=await register(app,'receipt-bob') - written=[] + app, _ = installed + alice_key, alice, _ = await register(app, 'receipt-alice') + bob_key, bob, _ = await register(app, 'receipt-bob') + written = [] for index in range(3): - post=await call(app,'content.post_create',{'parent':'/main','body':f'receipt {index}'}, - key=alice_key,subject=alice) - assert post.status=='ok',wire(post) + post = await call( + app, + 'content.post_create', + {'parent': '/main', 'body': f'receipt {index}'}, + key=alice_key, + subject=alice, + ) + assert post.status == 'ok', wire(post) written.append(post) - bob_post=await call(app,'content.post_create',{'parent':'/main','body':'bob receipt'}, - key=bob_key,subject=bob) - assert bob_post.status=='ok',wire(bob_post) + bob_post = await call( + app, + 'content.post_create', + {'parent': '/main', 'body': 'bob receipt'}, + key=bob_key, + subject=bob, + ) + assert bob_post.status == 'ok', wire(bob_post) async with app.metadata.transaction(write=False) as tx: - expected=[row[0] for row in tx.rows( - 'SELECT request_id FROM results WHERE subject=? ORDER BY request_id',(alice,))] - before=_state(tx) - assert {post.request_id for post in written}<=set(expected) and len(expected)>=4 + expected = [ + row[0] + for row in tx.rows( + 'SELECT request_id FROM results WHERE subject=? ORDER BY request_id', (alice,) + ) + ] + before = _state(tx) + assert {post.request_id for post in written} <= set(expected) and len(expected) >= 4 assert bob_post.request_id not in expected - async with httpx.AsyncClient(transport=httpx.ASGITransport(app=create_app(app)), - base_url=app.settings.service_url) as http: - list_headers=_headers(app,'communication.receipt_list',{},alice_key,alice) - listed=await http.get('/@receipt-alice/receipts',headers=list_headers) - assert listed.status_code==200,listed.text - body=listed.json() - assert body['path']=='/@receipt-alice/receipts' - assert [item['request_id'] for item in body['items']]==expected + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=create_app(app)), base_url=app.settings.service_url + ) as http: + list_headers = _headers(app, 'communication.receipt_list', {}, alice_key, alice) + listed = await http.get('/@receipt-alice/receipts', headers=list_headers) + assert listed.status_code == 200, listed.text + body = listed.json() + assert body['path'] == '/@receipt-alice/receipts' + assert [item['request_id'] for item in body['items']] == expected assert 'cursor' not in body for item in body['items']: - assert set(item)==PROJECTION - assert item['signed'] is True and item['status']=='ok' - slashed=await http.get('/@receipt-alice/receipts/',headers=list_headers) - suffixed=await http.get('/@receipt-alice/receipts/json',headers=list_headers) - assert slashed.content==suffixed.content==listed.content - cached=await http.get('/@receipt-alice/receipts', - headers={**list_headers,'If-None-Match':listed.headers['etag']}) - assert cached.status_code==304 - - target=written[1] - item_headers=_headers(app,'communication.receipt_get', - {'request_id':target.request_id},alice_key,alice) - single=await http.get(f'/@receipt-alice/receipts/{target.request_id}',headers=item_headers) - assert single.status_code==200,single.text - receipt=single.json()['receipt'] - assert single.json()['path']==f'/@receipt-alice/receipts/{target.request_id}' - assert receipt==next(item for item in body['items'] - if item['request_id']==target.request_id) - assert receipt['operation']=='content.post_create' - assert receipt['resources']==[wire(ref) for ref in target.resources] - assert receipt['committed_at']==wire(target.committed_at) - for secret in ('data','output','receipt','cli_url','prefer_cli','actor','subject'): + assert set(item) == PROJECTION + assert item['signed'] is True and item['status'] == 'ok' + slashed = await http.get('/@receipt-alice/receipts/', headers=list_headers) + suffixed = await http.get('/@receipt-alice/receipts/json', headers=list_headers) + assert slashed.content == suffixed.content == listed.content + cached = await http.get( + '/@receipt-alice/receipts', + headers={**list_headers, 'If-None-Match': listed.headers['etag']}, + ) + assert cached.status_code == 304 + + target = written[1] + item_headers = _headers( + app, 'communication.receipt_get', {'request_id': target.request_id}, alice_key, alice + ) + single = await http.get( + f'/@receipt-alice/receipts/{target.request_id}', headers=item_headers + ) + assert single.status_code == 200, single.text + receipt = single.json()['receipt'] + assert single.json()['path'] == f'/@receipt-alice/receipts/{target.request_id}' + assert receipt == next( + item for item in body['items'] if item['request_id'] == target.request_id + ) + assert receipt['operation'] == 'content.post_create' + assert receipt['resources'] == [wire(ref) for ref in target.resources] + assert receipt['committed_at'] == wire(target.committed_at) + for secret in ('data', 'output', 'receipt', 'cli_url', 'prefer_cli', 'actor', 'subject'): assert secret not in receipt - suffixed=await http.get(f'/@receipt-alice/receipts/{target.request_id}/json', - headers=item_headers) - assert suffixed.content==single.content - head=await http.head(f'/@receipt-alice/receipts/{target.request_id}',headers=item_headers) - assert head.status_code==200 and head.content==b'' - assert head.headers['etag']==single.headers['etag'] - - missing_headers=_headers(app,'communication.receipt_get', - {'request_id':'r_missing_receipt'},alice_key,alice) - missing=await http.get('/@receipt-alice/receipts/r_missing_receipt',headers=missing_headers) - assert missing.status_code==404 and missing.json()['error']['code']=='not_found' - - bob_list=_headers(app,'communication.receipt_list',{},bob_key,bob) - cross=await http.get('/@receipt-alice/receipts',headers=bob_list) - assert cross.status_code==403 and cross.json()['error']['code']=='permission_denied' + suffixed = await http.get( + f'/@receipt-alice/receipts/{target.request_id}/json', headers=item_headers + ) + assert suffixed.content == single.content + head = await http.head( + f'/@receipt-alice/receipts/{target.request_id}', headers=item_headers + ) + assert head.status_code == 200 and head.content == b'' + assert head.headers['etag'] == single.headers['etag'] + + missing_headers = _headers( + app, 'communication.receipt_get', {'request_id': 'r_missing_receipt'}, alice_key, alice + ) + missing = await http.get( + '/@receipt-alice/receipts/r_missing_receipt', headers=missing_headers + ) + assert missing.status_code == 404 and missing.json()['error']['code'] == 'not_found' + + bob_list = _headers(app, 'communication.receipt_list', {}, bob_key, bob) + cross = await http.get('/@receipt-alice/receipts', headers=bob_list) + assert cross.status_code == 403 and cross.json()['error']['code'] == 'permission_denied' assert target.request_id not in cross.text - for request_id in (target.request_id,bob_post.request_id,'r_missing_receipt'): - bob_get=_headers(app,'communication.receipt_get',{'request_id':request_id},bob_key,bob) - denied=await http.get(f'/@receipt-alice/receipts/{request_id}',headers=bob_get) - assert denied.status_code==403,denied.text - assert denied.json()['error']['code']=='permission_denied' - own=await http.get('/@receipt-bob/receipts',headers=bob_list) - assert own.status_code==200 + for request_id in (target.request_id, bob_post.request_id, 'r_missing_receipt'): + bob_get = _headers( + app, 'communication.receipt_get', {'request_id': request_id}, bob_key, bob + ) + denied = await http.get(f'/@receipt-alice/receipts/{request_id}', headers=bob_get) + assert denied.status_code == 403, denied.text + assert denied.json()['error']['code'] == 'permission_denied' + own = await http.get('/@receipt-bob/receipts', headers=bob_list) + assert own.status_code == 200 assert bob_post.request_id in [item['request_id'] for item in own.json()['items']] - assert not set(expected)&{item['request_id'] for item in own.json()['items']} + assert not set(expected) & {item['request_id'] for item in own.json()['items']} - for path in ('/@receipt-alice/receipts',f'/@receipt-alice/receipts/{target.request_id}'): - anonymous=await http.get(path) - assert anonymous.status_code==401 - assert anonymous.json()['error']['code']=='authentication_required' + for path in ('/@receipt-alice/receipts', f'/@receipt-alice/receipts/{target.request_id}'): + anonymous = await http.get(path) + assert anonymous.status_code == 401 + assert anonymous.json()['error']['code'] == 'authentication_required' assert 'items' not in anonymous.json() and 'receipt' not in anonymous.json() - unknown=await http.get('/@receipt-alice/receipts?after=x',headers=list_headers) - assert unknown.status_code==400 - assert unknown.json()['error']['code']=='unknown_query_parameter' - detail_query=await http.get(f'/@receipt-alice/receipts/{target.request_id}?limit=1', - headers=item_headers) - assert detail_query.json()['error']['code']=='unknown_query_parameter' - for bad in (0,201): - bad_headers=_headers(app,'communication.receipt_list',{'limit':bad},alice_key,alice) - rejected=await http.get(f'/@receipt-alice/receipts?limit={bad}',headers=bad_headers) - assert rejected.status_code==400,rejected.text - for path in ('/@receipt-alice/receipts',f'/@receipt-alice/receipts/{target.request_id}'): - written_path=await http.post(path,json={}) - assert written_path.status_code in {404,405} - - pages=[] - cursor=None + unknown = await http.get('/@receipt-alice/receipts?after=x', headers=list_headers) + assert unknown.status_code == 400 + assert unknown.json()['error']['code'] == 'unknown_query_parameter' + detail_query = await http.get( + f'/@receipt-alice/receipts/{target.request_id}?limit=1', headers=item_headers + ) + assert detail_query.json()['error']['code'] == 'unknown_query_parameter' + for bad in (0, 201): + bad_headers = _headers( + app, 'communication.receipt_list', {'limit': bad}, alice_key, alice + ) + rejected = await http.get(f'/@receipt-alice/receipts?limit={bad}', headers=bad_headers) + assert rejected.status_code == 400, rejected.text + for path in ('/@receipt-alice/receipts', f'/@receipt-alice/receipts/{target.request_id}'): + written_path = await http.post(path, json={}) + assert written_path.status_code in {404, 405} + + pages = [] + cursor = None while True: - args={'limit':1} if cursor is None else {'limit':1,'cursor':cursor} - page=await http.get('/@receipt-alice/receipts',params=args, - headers=_headers(app,'communication.receipt_list',args, - alice_key,alice)) - assert page.status_code==200,page.text - data=page.json() - assert len(data['items'])==1 + args = {'limit': 1} if cursor is None else {'limit': 1, 'cursor': cursor} + page = await http.get( + '/@receipt-alice/receipts', + params=args, + headers=_headers(app, 'communication.receipt_list', args, alice_key, alice), + ) + assert page.status_code == 200, page.text + data = page.json() + assert len(data['items']) == 1 pages.append(data['items'][0]['request_id']) - cursor=data.get('cursor') + cursor = data.get('cursor') if cursor is None: break assert data['next_requires_auth'] is True assert data['next'].startswith('/-/g/communication.receipt_list/j/') - stolen_args={'limit':1,'cursor':cursor} - stolen=await http.get('/@receipt-bob/receipts',params=stolen_args, - headers=_headers(app,'communication.receipt_list', - stolen_args,bob_key,bob)) - assert stolen.status_code==400 - assert stolen.json()['error']['code']=='cursor_query_mismatch' - assert pages==expected + stolen_args = {'limit': 1, 'cursor': cursor} + stolen = await http.get( + '/@receipt-bob/receipts', + params=stolen_args, + headers=_headers(app, 'communication.receipt_list', stolen_args, bob_key, bob), + ) + assert stolen.status_code == 400 + assert stolen.json()['error']['code'] == 'cursor_query_mismatch' + assert pages == expected async with app.metadata.transaction(write=False) as tx: - assert _state(tx)==before + assert _state(tx) == before @pytest.mark.asyncio async def test_receipt_projection_hides_resources_no_longer_readable(installed): - app,_=installed - alice_key,alice,_=await register(app,'receipt-share-alice') - bob_key,bob,_=await register(app,'receipt-share-bob') - note=await call(app,'identity.note_put',{'name':'receipt-note','body':'Private'}, - key=alice_key,subject=alice) - assert note.status=='ok',wire(note) - rid=note.resources[0].id - grant=await call(app,'sharing.grant',{'resource':rid,'grantee':bob, - 'expires_at':wire(NOW+timedelta(days=1))}, - key=alice_key,subject=alice) - assert grant.status=='ok',wire(grant) - sent=await call(app,'communication.send', - {'recipient':alice,'resource':wire(note.resources[0])}, - key=bob_key,subject=bob) - assert sent.status=='ok',wire(sent) + app, _ = installed + alice_key, alice, _ = await register(app, 'receipt-share-alice') + bob_key, bob, _ = await register(app, 'receipt-share-bob') + note = await call( + app, + 'identity.note_put', + {'name': 'receipt-note', 'body': 'Private'}, + key=alice_key, + subject=alice, + ) + assert note.status == 'ok', wire(note) + rid = note.resources[0].id + grant = await call( + app, + 'sharing.grant', + {'resource': rid, 'grantee': bob, 'expires_at': wire(NOW + timedelta(days=1))}, + key=alice_key, + subject=alice, + ) + assert grant.status == 'ok', wire(grant) + sent = await call( + app, + 'communication.send', + {'recipient': alice, 'resource': wire(note.resources[0])}, + key=bob_key, + subject=bob, + ) + assert sent.status == 'ok', wire(sent) assert rid in {ref.id for ref in sent.resources} - shown=await call(app,'communication.receipt_get',{'request_id':sent.request_id}, - key=bob_key,subject=bob) - assert shown.status=='ok',wire(shown) + shown = await call( + app, 'communication.receipt_get', {'request_id': sent.request_id}, key=bob_key, subject=bob + ) + assert shown.status == 'ok', wire(shown) assert rid in {ref['id'] for ref in shown.data['receipt']['resources']} - revoked=await call(app,'sharing.revoke',{'grant_id':grant.data['grant']['id']}, - key=alice_key,subject=alice) - assert revoked.status=='ok',wire(revoked) + revoked = await call( + app, 'sharing.revoke', {'grant_id': grant.data['grant']['id']}, key=alice_key, subject=alice + ) + assert revoked.status == 'ok', wire(revoked) async with app.metadata.transaction(write=False) as tx: - stored=tx.one('SELECT body FROM results WHERE subject=? AND request_id=?', - (bob,sent.request_id))[0] - hidden=await call(app,'communication.receipt_get',{'request_id':sent.request_id}, - key=bob_key,subject=bob) - assert hidden.status=='ok',wire(hidden) + stored = tx.one( + 'SELECT body FROM results WHERE subject=? AND request_id=?', (bob, sent.request_id) + )[0] + hidden = await call( + app, 'communication.receipt_get', {'request_id': sent.request_id}, key=bob_key, subject=bob + ) + assert hidden.status == 'ok', wire(hidden) assert rid not in {ref['id'] for ref in hidden.data['receipt']['resources']} assert hidden.data['receipt']['signed'] is True async with app.metadata.transaction(write=False) as tx: - assert tx.one('SELECT body FROM results WHERE subject=? AND request_id=?', - (bob,sent.request_id))[0]==stored + assert ( + tx.one( + 'SELECT body FROM results WHERE subject=? AND request_id=?', (bob, sent.request_id) + )[0] + == stored + ) diff --git a/tests/test_sync_checkpoint.py b/tests/test_sync_checkpoint.py index 06230af5..08ae4870 100644 --- a/tests/test_sync_checkpoint.py +++ b/tests/test_sync_checkpoint.py @@ -1,146 +1,240 @@ """Durable SyncCursor advancement requires an explicit signed ACK.""" + from datetime import timedelta import httpx import pytest +from test_service import NOW, call, register from msg.core.codec import b64, canonical, wire from msg.core.requests import request_for from msg.transports.http import create_app -from test_service import NOW, call, register -def sync_headers(app,key,subject,cursor): - packet=request_for('communication.sync',{'cursor':cursor},app.settings.service_url, - signer=key,subject=subject,expires_at=NOW+timedelta(seconds=60)) - return {'x-msg-request':b64(canonical(packet))} +def sync_headers(app, key, subject, cursor): + packet = request_for( + 'communication.sync', + {'cursor': cursor}, + app.settings.service_url, + signer=key, + subject=subject, + expires_at=NOW + timedelta(seconds=60), + ) + return {'x-msg-request': b64(canonical(packet))} -async def read_sync(http,app,key,subject,cursor): - return await http.get('/_r/s/'+cursor,headers=sync_headers(app,key,subject,cursor)) +async def read_sync(http, app, key, subject, cursor): + return await http.get('/_r/s/' + cursor, headers=sync_headers(app, key, subject, cursor)) @pytest.mark.asyncio async def test_checkpoint_expands_seen_beyond_64_and_requires_ack(installed): - app,_=installed - key,subject,_=await register(app,'checkpoint-many') - other_key,other_subject,_=await register(app,'checkpoint-other') + app, _ = installed + key, subject, _ = await register(app, 'checkpoint-many') + other_key, other_subject, _ = await register(app, 'checkpoint-other') for index in range(70): - result=await call(app,'content.post_create',{'parent':'/main','body':f'post {index}'}, - key=key,subject=subject) - assert result.status=='ok',wire(result) - async with httpx.AsyncClient(transport=httpx.ASGITransport(app=create_app(app)), - base_url='http://testserver') as http: - packet=request_for('communication.sync',{},app.settings.service_url, - signer=key,subject=subject,expires_at=NOW+timedelta(seconds=60)) - start=await http.get('/_r/s/start',headers={'x-msg-request':b64(canonical(packet))}) - assert start.status_code==200,start.text - old=start.json()['sync_cursor'] - opened=await call(app,'communication.sync_checkpoint_open',{'cursor':old}, - key=key,subject=subject,rid='checkpoint-open') - assert opened.status=='ok',wire(opened) - replay=await call(app,'communication.sync_checkpoint_open',{'cursor':old}, - key=key,subject=subject,rid='checkpoint-open') - assert replay.status=='ok' and replay.replayed - cursor=opened.data['sync_cursor'] - forbidden=await read_sync(http,app,other_key,other_subject,cursor) - assert forbidden.status_code==403 - assert forbidden.json()['error']['code']=='cursor_principal_mismatch' - page=await read_sync(http,app,key,subject,cursor) - again=await read_sync(http,app,key,subject,cursor) - assert page.status_code==again.status_code==200 - assert page.json()==again.json() + result = await call( + app, + 'content.post_create', + {'parent': '/main', 'body': f'post {index}'}, + key=key, + subject=subject, + ) + assert result.status == 'ok', wire(result) + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=create_app(app)), base_url='http://testserver' + ) as http: + packet = request_for( + 'communication.sync', + {}, + app.settings.service_url, + signer=key, + subject=subject, + expires_at=NOW + timedelta(seconds=60), + ) + start = await http.get('/_r/s/start', headers={'x-msg-request': b64(canonical(packet))}) + assert start.status_code == 200, start.text + old = start.json()['sync_cursor'] + opened = await call( + app, + 'communication.sync_checkpoint_open', + {'cursor': old}, + key=key, + subject=subject, + rid='checkpoint-open', + ) + assert opened.status == 'ok', wire(opened) + replay = await call( + app, + 'communication.sync_checkpoint_open', + {'cursor': old}, + key=key, + subject=subject, + rid='checkpoint-open', + ) + assert replay.status == 'ok' and replay.replayed + cursor = opened.data['sync_cursor'] + forbidden = await read_sync(http, app, other_key, other_subject, cursor) + assert forbidden.status_code == 403 + assert forbidden.json()['error']['code'] == 'cursor_principal_mismatch' + page = await read_sync(http, app, key, subject, cursor) + again = await read_sync(http, app, key, subject, cursor) + assert page.status_code == again.status_code == 200 + assert page.json() == again.json() assert page.json()['items'] async with app.metadata.transaction(write=False) as tx: - version=tx.one('SELECT version FROM sync_checkpoints WHERE subject=?',(subject,))[0] - assert version==0 - ack=page.json()['checkpoint_ack'] - committed=await call(app,'communication.sync_checkpoint_ack',{'ack':ack}, - key=key,subject=subject,rid='checkpoint-ack') - assert committed.status=='ok',wire(committed) - retry=await call(app,'communication.sync_checkpoint_ack',{'ack':ack}, - key=key,subject=subject,rid='checkpoint-ack') - assert retry.status=='ok' and retry.replayed - conflict=await call(app,'communication.sync_checkpoint_ack',{'ack':ack}, - key=key,subject=subject,rid='checkpoint-concurrent') - assert conflict.error.code=='checkpoint_conflict' - stale_get=await read_sync(http,app,key,subject,cursor) - assert stale_get.status_code==400 - assert stale_get.json()['error']['code']=='checkpoint_conflict' - second=await read_sync(http,app,key,subject,committed.data['sync_cursor']) - assert second.status_code==200,second.text - next_ack=await call(app,'communication.sync_checkpoint_ack', - {'ack':second.json()['checkpoint_ack']},key=key,subject=subject) - assert next_ack.status=='ok',wire(next_ack) + version = tx.one('SELECT version FROM sync_checkpoints WHERE subject=?', (subject,))[0] + assert version == 0 + ack = page.json()['checkpoint_ack'] + committed = await call( + app, + 'communication.sync_checkpoint_ack', + {'ack': ack}, + key=key, + subject=subject, + rid='checkpoint-ack', + ) + assert committed.status == 'ok', wire(committed) + retry = await call( + app, + 'communication.sync_checkpoint_ack', + {'ack': ack}, + key=key, + subject=subject, + rid='checkpoint-ack', + ) + assert retry.status == 'ok' and retry.replayed + conflict = await call( + app, + 'communication.sync_checkpoint_ack', + {'ack': ack}, + key=key, + subject=subject, + rid='checkpoint-concurrent', + ) + assert conflict.error.code == 'checkpoint_conflict' + stale_get = await read_sync(http, app, key, subject, cursor) + assert stale_get.status_code == 400 + assert stale_get.json()['error']['code'] == 'checkpoint_conflict' + second = await read_sync(http, app, key, subject, committed.data['sync_cursor']) + assert second.status_code == 200, second.text + next_ack = await call( + app, + 'communication.sync_checkpoint_ack', + {'ack': second.json()['checkpoint_ack']}, + key=key, + subject=subject, + ) + assert next_ack.status == 'ok', wire(next_ack) async with app.metadata.transaction(write=False) as tx: - body=tx.one('SELECT body FROM sync_checkpoints WHERE subject=?',(subject,))[0] + body = tx.one('SELECT body FROM sync_checkpoints WHERE subject=?', (subject,))[0] from msg.core.codec import loads - assert len(loads(body)['seen'])>=70 + + assert len(loads(body)['seen']) >= 70 @pytest.mark.asyncio async def test_checkpoint_permission_gain_and_retention_fail_closed(installed): - app,_=installed - key,subject,_=await register(app,'checkpoint-stale') - created=await call(app,'content.post_create',{'parent':'/main','body':'sample'}, - key=key,subject=subject) - async with httpx.AsyncClient(transport=httpx.ASGITransport(app=create_app(app)), - base_url='http://testserver') as http: - packet=request_for('communication.sync',{},app.settings.service_url, - signer=key,subject=subject,expires_at=NOW+timedelta(seconds=60)) - start=await http.get('/_r/s/start',headers={'x-msg-request':b64(canonical(packet))}) - old=start.json()['sync_cursor'] - opened=await call(app,'communication.sync_checkpoint_open',{'cursor':old}, - key=key,subject=subject) - cursor=opened.data['sync_cursor'] + app, _ = installed + key, subject, _ = await register(app, 'checkpoint-stale') + created = await call( + app, 'content.post_create', {'parent': '/main', 'body': 'sample'}, key=key, subject=subject + ) + assert created.status == 'ok' + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=create_app(app)), base_url='http://testserver' + ) as http: + packet = request_for( + 'communication.sync', + {}, + app.settings.service_url, + signer=key, + subject=subject, + expires_at=NOW + timedelta(seconds=60), + ) + start = await http.get('/_r/s/start', headers={'x-msg-request': b64(canonical(packet))}) + old = start.json()['sync_cursor'] + opened = await call( + app, 'communication.sync_checkpoint_open', {'cursor': old}, key=key, subject=subject + ) + cursor = opened.data['sync_cursor'] async with app.metadata.transaction(write=True) as tx: - tx.set_setting('authorization_epoch',tx.setting('authorization_epoch',0)+1) - stale=await read_sync(http,app,key,subject,cursor) - assert stale.status_code==400 - assert stale.json()['error']['code']=='resync_required' + tx.set_setting('authorization_epoch', tx.setting('authorization_epoch', 0) + 1) + stale = await read_sync(http, app, key, subject, cursor) + assert stale.status_code == 400 + assert stale.json()['error']['code'] == 'resync_required' async with app.metadata.transaction(write=True) as tx: - tx.set_setting('authorization_epoch',tx.setting('authorization_epoch',0)-1) - tx.set_setting('sync_floor',10**9) - expired=await read_sync(http,app,key,subject,cursor) - assert expired.status_code==400 - assert expired.json()['error']['code']=='resync_required' + tx.set_setting('authorization_epoch', tx.setting('authorization_epoch', 0) - 1) + tx.set_setting('sync_floor', 10**9) + expired = await read_sync(http, app, key, subject, cursor) + assert expired.status_code == 400 + assert expired.json()['error']['code'] == 'resync_required' @pytest.mark.asyncio async def test_checkpoint_drains_known_revocations_one_ack_at_a_time(installed): - app,_=installed - key,subject,_=await register(app,'checkpoint-revokes') - created=[] + app, _ = installed + key, subject, _ = await register(app, 'checkpoint-revokes') + created = [] for index in range(3): - post=await call(app,'content.post_create',{'parent':'/main','body':f'known {index}'}, - key=key,subject=subject) + post = await call( + app, + 'content.post_create', + {'parent': '/main', 'body': f'known {index}'}, + key=key, + subject=subject, + ) created.append(post) - packet=request_for('communication.sync',{},app.settings.service_url, - signer=key,subject=subject,expires_at=NOW+timedelta(seconds=60)) - start=await app.executor.execute(packet) - opened=await call(app,'communication.sync_checkpoint_open', - {'cursor':start.data['sync_cursor']},key=key,subject=subject) - cursor=opened.data['sync_cursor'] + packet = request_for( + 'communication.sync', + {}, + app.settings.service_url, + signer=key, + subject=subject, + expires_at=NOW + timedelta(seconds=60), + ) + start = await app.executor.execute(packet) + opened = await call( + app, + 'communication.sync_checkpoint_open', + {'cursor': start.data['sync_cursor']}, + key=key, + subject=subject, + ) + cursor = opened.data['sync_cursor'] for post in created: - rid=post.resources[0].id - locked=await call(app,'content.chmod',{'id':rid,'mode':'0000'},key=key, - subject=subject,expected=((rid,post.data['generation']),)) - assert locked.status=='ok',wire(locked) - seen=[] + rid = post.resources[0].id + locked = await call( + app, + 'content.chmod', + {'id': rid, 'mode': '0000'}, + key=key, + subject=subject, + expected=((rid, post.data['generation']),), + ) + assert locked.status == 'ok', wire(locked) + seen = [] for _ in range(3): - page=await call(app,'communication.sync',{'cursor':cursor,'limit':1}, - key=key,subject=subject) - assert page.status=='ok',wire(page) + page = await call( + app, 'communication.sync', {'cursor': cursor, 'limit': 1}, key=key, subject=subject + ) + assert page.status == 'ok', wire(page) assert page.data['resync_required'] is True - assert len(page.data['items'])==1 - item=page.data['items'][0] - assert set(item)=={'kind','ref'} and item['kind']=='revoked' + assert len(page.data['items']) == 1 + item = page.data['items'][0] + assert set(item) == {'kind', 'ref'} and item['kind'] == 'revoked' seen.append(item['ref']['id']) - ack=await call(app,'communication.sync_checkpoint_ack', - {'ack':page.data['checkpoint_ack']},key=key,subject=subject) - assert ack.status=='ok',wire(ack) - cursor=ack.data['sync_cursor'] - assert set(seen)=={post.resources[0].id for post in created} - final=await call(app,'communication.sync',{'cursor':cursor,'limit':1}, - key=key,subject=subject) - assert final.error.code=='resync_required' + ack = await call( + app, + 'communication.sync_checkpoint_ack', + {'ack': page.data['checkpoint_ack']}, + key=key, + subject=subject, + ) + assert ack.status == 'ok', wire(ack) + cursor = ack.data['sync_cursor'] + assert set(seen) == {post.resources[0].id for post in created} + final = await call( + app, 'communication.sync', {'cursor': cursor, 'limit': 1}, key=key, subject=subject + ) + assert final.error.code == 'resync_required' diff --git a/tests/test_sync_cursor.py b/tests/test_sync_cursor.py index 29653646..9c7cfe4a 100644 --- a/tests/test_sync_cursor.py +++ b/tests/test_sync_cursor.py @@ -1,111 +1,151 @@ """Independent SyncCursor never doubles as a page cursor or read receipt.""" + from datetime import timedelta import httpx import pytest +from test_service import NOW, call, register from msg.core.codec import b64, canonical, wire from msg.core.errors import Failure from msg.core.requests import request_for from msg.plugins.communication import seal_sync_seen from msg.transports.http import create_app -from test_service import NOW, call, register -def signed_path(app,key,subject,path,args): - packet=request_for('communication.sync',args,app.settings.service_url, - signer=key,subject=subject, - expires_at=NOW+timedelta(seconds=60)) - return path+'/p/'+b64(canonical(packet)) +def signed_path(app, key, subject, path, args): + packet = request_for( + 'communication.sync', + args, + app.settings.service_url, + signer=key, + subject=subject, + expires_at=NOW + timedelta(seconds=60), + ) + return path + '/p/' + b64(canonical(packet)) -def signed_header(app,key,subject,args): - packet=request_for('communication.sync',args,app.settings.service_url, - signer=key,subject=subject, - expires_at=NOW+timedelta(seconds=60)) - return {'x-msg-request':b64(canonical(packet))} +def signed_header(app, key, subject, args): + packet = request_for( + 'communication.sync', + args, + app.settings.service_url, + signer=key, + subject=subject, + expires_at=NOW + timedelta(seconds=60), + ) + return {'x-msg-request': b64(canonical(packet))} async def state(app): async with app.metadata.transaction(write=False) as tx: - return (tx.one('SELECT COUNT(*) FROM events')[0], - tx.one('SELECT COUNT(*) FROM jobs')[0], - tx.one('SELECT COUNT(*) FROM reactions')[0], - tx.one('SELECT COUNT(*) FROM messages')[0], - tx.one('SELECT COUNT(*) FROM credentials')[0], - tx.one('SELECT COUNT(*) FROM revisions')[0], - tx.one('SELECT SUM(generation) FROM resources')[0], - tx.one('SELECT COUNT(*) FROM transfers')[0]) + return ( + tx.one('SELECT COUNT(*) FROM events')[0], + tx.one('SELECT COUNT(*) FROM jobs')[0], + tx.one('SELECT COUNT(*) FROM reactions')[0], + tx.one('SELECT COUNT(*) FROM messages')[0], + tx.one('SELECT COUNT(*) FROM credentials')[0], + tx.one('SELECT COUNT(*) FROM revisions')[0], + tx.one('SELECT SUM(generation) FROM resources')[0], + tx.one('SELECT COUNT(*) FROM transfers')[0], + ) @pytest.mark.asyncio async def test_sync_cursor_tracks_visible_create_modify_archive_and_known_revocation(installed): app, _ = installed - key,user,_=await register(app,'sync-owner') - first=await call(app,'content.post_create',{'parent':'/main','body':'first'}, - key=key,subject=user) - rid=first.resources[0].id - async with httpx.AsyncClient(transport=httpx.ASGITransport(app=create_app(app)), - base_url='http://testserver') as http: - opened=await http.get(signed_path(app,key,user,'/_read/s/start',{})) - assert opened.status_code==200,opened.text - opened_short=await http.get(signed_path(app,key,user,'/_r/s/start',{})) - assert opened_short.status_code==200 - assert opened.content==opened_short.content - assert opened.headers['etag']==opened_short.headers['etag'] + key, user, _ = await register(app, 'sync-owner') + first = await call( + app, 'content.post_create', {'parent': '/main', 'body': 'first'}, key=key, subject=user + ) + rid = first.resources[0].id + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=create_app(app)), base_url='http://testserver' + ) as http: + opened = await http.get(signed_path(app, key, user, '/_read/s/start', {})) + assert opened.status_code == 200, opened.text + opened_short = await http.get(signed_path(app, key, user, '/_r/s/start', {})) + assert opened_short.status_code == 200 + assert opened.content == opened_short.content + assert opened.headers['etag'] == opened_short.headers['etag'] assert opened.json()['next'].startswith('/_r/s/') - assert rid in {item['ref']['id'] for item in opened.json()['items'] - if item.get('ref')} - cursor=opened.json()['sync_cursor'] + assert rid in {item['ref']['id'] for item in opened.json()['items'] if item.get('ref')} + cursor = opened.json()['sync_cursor'] assert rid not in str(app.cursors.inspect(cursor)) - before=await state(app) - long=await http.get(signed_path(app,key,user,'/_read/s/'+cursor,{'cursor':cursor})) - short=await http.get(signed_path(app,key,user,'/_r/s/'+cursor,{'cursor':cursor})) - assert long.status_code==short.status_code==200 - assert long.content==short.content - assert long.headers['etag']==short.headers['etag'] - assert long.json()['sync_cursor']==short.json()['sync_cursor'] - long_cursor=app.cursors.inspect(long.json()['sync_cursor']) - short_cursor=app.cursors.inspect(short.json()['sync_cursor']) - assert long_cursor['position']['seq']==short_cursor['position']['seq'] - assert long_cursor['query']==short_cursor['query'] - assert long_cursor['position']['seen_ciphertext']==short_cursor['position']['seen_ciphertext'] - assert await state(app)==before - second=await call(app,'content.post_create',{'parent':'/main','body':'second'}, - key=key,subject=user) - new=await http.get(signed_path(app,key,user,opened.json()['next'],{'cursor':cursor})) - assert new.status_code==200 - assert any(item['kind']=='created' and item['ref']['id']==second.resources[0].id - for item in new.json()['items']) - cursor=new.json()['sync_cursor'] - edited=await call(app,'content.post_edit', - {'id':rid,'expected_revision':first.resources[0].revision,'body':'updated'}, - key=key,subject=user,expected=((rid,first.data['generation']),)) - changed=await http.get(signed_path(app,key,user,'/_r/s/'+cursor,{'cursor':cursor})) - assert any(item['kind']=='modified' and item['ref']['id']==rid - for item in changed.json()['items']) - cursor=changed.json()['sync_cursor'] - archived=await call(app,'content.archive',{'id':rid},key=key,subject=user, - expected=((rid,edited.data['generation']),)) - assert archived.status=='ok' - archive=await http.get(signed_path(app,key,user,'/_r/s/'+cursor,{'cursor':cursor})) - assert archive.status_code==400 - assert archive.json()['error']['code']=='resync_required' - restarted=await http.get(signed_path(app,key,user,'/_r/s/start',{})) - assert restarted.status_code==200 - assert any(item['kind']=='archived' and item['ref']['id']==rid - for item in restarted.json()['items']) - cursor=restarted.json()['sync_cursor'] + before = await state(app) + long = await http.get(signed_path(app, key, user, '/_read/s/' + cursor, {'cursor': cursor})) + short = await http.get(signed_path(app, key, user, '/_r/s/' + cursor, {'cursor': cursor})) + assert long.status_code == short.status_code == 200 + assert long.content == short.content + assert long.headers['etag'] == short.headers['etag'] + assert long.json()['sync_cursor'] == short.json()['sync_cursor'] + long_cursor = app.cursors.inspect(long.json()['sync_cursor']) + short_cursor = app.cursors.inspect(short.json()['sync_cursor']) + assert long_cursor['position']['seq'] == short_cursor['position']['seq'] + assert long_cursor['query'] == short_cursor['query'] + assert ( + long_cursor['position']['seen_ciphertext'] + == short_cursor['position']['seen_ciphertext'] + ) + assert await state(app) == before + second = await call( + app, 'content.post_create', {'parent': '/main', 'body': 'second'}, key=key, subject=user + ) + new = await http.get(signed_path(app, key, user, opened.json()['next'], {'cursor': cursor})) + assert new.status_code == 200 + assert any( + item['kind'] == 'created' and item['ref']['id'] == second.resources[0].id + for item in new.json()['items'] + ) + cursor = new.json()['sync_cursor'] + edited = await call( + app, + 'content.post_edit', + {'id': rid, 'expected_revision': first.resources[0].revision, 'body': 'updated'}, + key=key, + subject=user, + expected=((rid, first.data['generation']),), + ) + changed = await http.get(signed_path(app, key, user, '/_r/s/' + cursor, {'cursor': cursor})) + assert any( + item['kind'] == 'modified' and item['ref']['id'] == rid + for item in changed.json()['items'] + ) + cursor = changed.json()['sync_cursor'] + archived = await call( + app, + 'content.archive', + {'id': rid}, + key=key, + subject=user, + expected=((rid, edited.data['generation']),), + ) + assert archived.status == 'ok' + archive = await http.get(signed_path(app, key, user, '/_r/s/' + cursor, {'cursor': cursor})) + assert archive.status_code == 400 + assert archive.json()['error']['code'] == 'resync_required' + restarted = await http.get(signed_path(app, key, user, '/_r/s/start', {})) + assert restarted.status_code == 200 + assert any( + item['kind'] == 'archived' and item['ref']['id'] == rid + for item in restarted.json()['items'] + ) + cursor = restarted.json()['sync_cursor'] async with app.metadata.transaction(write=False) as tx: - generation=(await tx.resource(second.resources[0].id)).generation - locked=await call(app,'content.chmod', - {'id':second.resources[0].id,'mode':'0000'},key=key,subject=user, - expected=((second.resources[0].id,generation),)) - assert locked.status=='ok' - revoked=await http.get(signed_path(app,key,user,'/_r/s/'+cursor,{'cursor':cursor})) - assert revoked.status_code==200 - lost=[item for item in revoked.json()['items'] if item['kind']=='revoked'] - assert lost==[{'kind':'revoked','ref':{'id':second.resources[0].id}}] + generation = (await tx.resource(second.resources[0].id)).generation + locked = await call( + app, + 'content.chmod', + {'id': second.resources[0].id, 'mode': '0000'}, + key=key, + subject=user, + expected=((second.resources[0].id, generation),), + ) + assert locked.status == 'ok' + revoked = await http.get(signed_path(app, key, user, '/_r/s/' + cursor, {'cursor': cursor})) + assert revoked.status_code == 200 + lost = [item for item in revoked.json()['items'] if item['kind'] == 'revoked'] + assert lost == [{'kind': 'revoked', 'ref': {'id': second.resources[0].id}}] assert revoked.json()['resync_required'] is True assert 'sync_cursor' not in revoked.json() and 'next' not in revoked.json() assert 'second' not in revoked.text @@ -114,94 +154,135 @@ async def test_sync_cursor_tracks_visible_create_modify_archive_and_known_revoca @pytest.mark.asyncio async def test_sync_cursor_rejects_other_kinds_expiry_and_expired_window(installed): app, _ = installed - key,user,_=await register(app,'sync-window') - other_key,other,_=await register(app,'sync-other') - async with httpx.AsyncClient(transport=httpx.ASGITransport(app=create_app(app)), - base_url='http://testserver') as http: - opened=await http.get(signed_path(app,key,user,'/_r/s/start',{})) - assert opened.status_code==200 - token=opened.json()['sync_cursor'] - alien=await http.get(signed_path(app,other_key,other,'/_r/s/'+token,{'cursor':token})) - assert alien.status_code==403 - fake_page=app.cursors.encode('page','x','y') - kind=await http.get(signed_path(app,key,user,'/_r/s/'+fake_page,{'cursor':fake_page})) - assert kind.status_code==400 - assert kind.json()['error']['code']=='cursor_kind_mismatch' - decoded=app.cursors.inspect(token) - expired=app.cursors.encode('sync-v2',decoded['query'], - {**decoded['position'],'expires_at':wire(NOW-timedelta(seconds=1))}) - stale=await http.get(signed_path(app,key,user,'/_r/s/'+expired,{'cursor':expired})) - assert stale.status_code==400 - assert stale.json()['error']['code']=='resync_required' + key, user, _ = await register(app, 'sync-window') + other_key, other, _ = await register(app, 'sync-other') + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=create_app(app)), base_url='http://testserver' + ) as http: + opened = await http.get(signed_path(app, key, user, '/_r/s/start', {})) + assert opened.status_code == 200 + token = opened.json()['sync_cursor'] + alien = await http.get( + signed_path(app, other_key, other, '/_r/s/' + token, {'cursor': token}) + ) + assert alien.status_code == 403 + fake_page = app.cursors.encode('page', 'x', 'y') + kind = await http.get( + signed_path(app, key, user, '/_r/s/' + fake_page, {'cursor': fake_page}) + ) + assert kind.status_code == 400 + assert kind.json()['error']['code'] == 'cursor_kind_mismatch' + decoded = app.cursors.inspect(token) + expired = app.cursors.encode( + 'sync-v2', + decoded['query'], + {**decoded['position'], 'expires_at': wire(NOW - timedelta(seconds=1))}, + ) + stale = await http.get(signed_path(app, key, user, '/_r/s/' + expired, {'cursor': expired})) + assert stale.status_code == 400 + assert stale.json()['error']['code'] == 'resync_required' async with app.metadata.transaction(write=True) as tx: - tx.set_setting('sync_floor',decoded['position']['seq']+1) - window=await http.get(signed_path(app,key,user,'/_r/s/'+token,{'cursor':token})) - assert window.status_code==400 - assert window.json()['error']['code']=='resync_required' + tx.set_setting('sync_floor', decoded['position']['seq'] + 1) + window = await http.get(signed_path(app, key, user, '/_r/s/' + token, {'cursor': token})) + assert window.status_code == 400 + assert window.json()['error']['code'] == 'resync_required' @pytest.mark.asyncio async def test_sync_never_emits_unseen_private_reference(installed): app, _ = installed - observer,observer_id,_=await register(app,'sync-observer') - writer,writer_id,_=await register(app,'sync-writer') - async with httpx.AsyncClient(transport=httpx.ASGITransport(app=create_app(app)), - base_url='http://testserver') as http: - opened=await http.get(signed_path(app,observer,observer_id,'/_r/s/start',{})) - token=opened.json()['sync_cursor'] - post=await call(app,'content.post_create',{'parent':'/main','body':'unseen secret'}, - key=writer,subject=writer_id) - rid=post.resources[0].id - await call(app,'content.chmod',{'id':rid,'mode':'0600'},key=writer, - subject=writer_id,expected=((rid,post.data['generation']),)) - before=await state(app) - result=await http.get(signed_path(app,observer,observer_id,'/_r/s/'+token, - {'cursor':token})) - assert result.status_code==400 - assert result.json()['error']['code']=='resync_required' + observer, observer_id, _ = await register(app, 'sync-observer') + writer, writer_id, _ = await register(app, 'sync-writer') + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=create_app(app)), base_url='http://testserver' + ) as http: + opened = await http.get(signed_path(app, observer, observer_id, '/_r/s/start', {})) + token = opened.json()['sync_cursor'] + post = await call( + app, + 'content.post_create', + {'parent': '/main', 'body': 'unseen secret'}, + key=writer, + subject=writer_id, + ) + rid = post.resources[0].id + await call( + app, + 'content.chmod', + {'id': rid, 'mode': '0600'}, + key=writer, + subject=writer_id, + expected=((rid, post.data['generation']),), + ) + before = await state(app) + result = await http.get( + signed_path(app, observer, observer_id, '/_r/s/' + token, {'cursor': token}) + ) + assert result.status_code == 400 + assert result.json()['error']['code'] == 'resync_required' assert rid not in result.text and 'unseen secret' not in result.text - assert await state(app)==before + assert await state(app) == before @pytest.mark.asyncio async def test_sync_seen_window_has_a_hard_bounded_size(installed): app, _ = installed - with pytest.raises(Failure,match='resync_required'): - seal_sync_seen(app,[f'r_{index}' for index in range(65)]) + with pytest.raises(Failure, match='resync_required'): + seal_sync_seen(app, [f'r_{index}' for index in range(65)]) @pytest.mark.asyncio async def test_sync_permission_gain_requires_replay_of_old_events(installed): app, _ = installed - observer,observer_id,_=await register(app,'sync-gain-observer') - writer,writer_id,_=await register(app,'sync-gain-writer') - watched=await call(app,'communication.watch',{'id':'/main'}, - key=observer,subject=observer_id) - assert watched.status=='ok' - post=await call(app,'content.post_create',{'parent':'/main','body':'old private body'}, - key=writer,subject=writer_id) - rid=post.resources[0].id - private=await call(app,'content.chmod',{'id':rid,'mode':'0600'},key=writer, - subject=writer_id,expected=((rid,post.data['generation']),)) - assert private.status=='ok' - async with httpx.AsyncClient(transport=httpx.ASGITransport(app=create_app(app)), - base_url='http://testserver') as http: - opened=await http.get(signed_path(app,observer,observer_id,'/_r/s/start',{})) - assert opened.status_code==200 + observer, observer_id, _ = await register(app, 'sync-gain-observer') + writer, writer_id, _ = await register(app, 'sync-gain-writer') + watched = await call( + app, 'communication.watch', {'id': '/main'}, key=observer, subject=observer_id + ) + assert watched.status == 'ok' + post = await call( + app, + 'content.post_create', + {'parent': '/main', 'body': 'old private body'}, + key=writer, + subject=writer_id, + ) + rid = post.resources[0].id + private = await call( + app, + 'content.chmod', + {'id': rid, 'mode': '0600'}, + key=writer, + subject=writer_id, + expected=((rid, post.data['generation']),), + ) + assert private.status == 'ok' + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=create_app(app)), base_url='http://testserver' + ) as http: + opened = await http.get(signed_path(app, observer, observer_id, '/_r/s/start', {})) + assert opened.status_code == 200 assert rid not in opened.text - cursor=opened.json()['sync_cursor'] - public=await call(app,'content.chmod',{'id':rid,'mode':'0644'},key=writer, - subject=writer_id,expected=((rid,private.data['generation']),)) - assert public.status=='ok' - before=await state(app) - stale=await http.get(signed_path(app,observer,observer_id,'/_r/s/'+cursor, - {'cursor':cursor})) - assert stale.status_code==400 - assert stale.json()['error']['code']=='resync_required' + cursor = opened.json()['sync_cursor'] + public = await call( + app, + 'content.chmod', + {'id': rid, 'mode': '0644'}, + key=writer, + subject=writer_id, + expected=((rid, private.data['generation']),), + ) + assert public.status == 'ok' + before = await state(app) + stale = await http.get( + signed_path(app, observer, observer_id, '/_r/s/' + cursor, {'cursor': cursor}) + ) + assert stale.status_code == 400 + assert stale.json()['error']['code'] == 'resync_required' assert rid not in stale.text and 'old private body' not in stale.text - assert await state(app)==before - restarted=await http.get(signed_path(app,observer,observer_id,'/_r/s/start',{})) - assert restarted.status_code==200 + assert await state(app) == before + restarted = await http.get(signed_path(app, observer, observer_id, '/_r/s/start', {})) + assert restarted.status_code == 200 # A fresh bounded replay can discover the now-visible old event. assert rid in restarted.text @@ -209,56 +290,71 @@ async def test_sync_permission_gain_requires_replay_of_old_events(installed): @pytest.mark.asyncio async def test_sync_over_64_refs_fails_without_advancing_cursor(installed): app, _ = installed - key,user,_=await register(app,'sync-over-64') + key, user, _ = await register(app, 'sync-over-64') for index in range(65): - post=await call(app,'content.post_create',{'parent':'/main','body':f'item {index}'}, - key=key,subject=user) - assert post.status=='ok' - before=await state(app) - async with httpx.AsyncClient(transport=httpx.ASGITransport(app=create_app(app)), - base_url='http://testserver') as http: - first=await http.get(signed_path(app,key,user,'/_r/s/start',{})) - assert first.status_code==200 - cursor=first.json()['sync_cursor'] - over=await http.get('/_r/s/'+cursor, - headers=signed_header(app,key,user,{'cursor':cursor})) - assert over.status_code==400,over.text - assert over.json()['error']['code']=='resync_required' + post = await call( + app, + 'content.post_create', + {'parent': '/main', 'body': f'item {index}'}, + key=key, + subject=user, + ) + assert post.status == 'ok' + before = await state(app) + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=create_app(app)), base_url='http://testserver' + ) as http: + first = await http.get(signed_path(app, key, user, '/_r/s/start', {})) + assert first.status_code == 200 + cursor = first.json()['sync_cursor'] + over = await http.get( + '/_r/s/' + cursor, headers=signed_header(app, key, user, {'cursor': cursor}) + ) + assert over.status_code == 400, over.text + assert over.json()['error']['code'] == 'resync_required' assert 'sync_cursor' not in over.json() and 'next' not in over.json() # Replaying from the beginning is not a long-term recovery strategy: # the same exact seen-set boundary is reached again. Keep this as a # regression until a /-/ checkpoint operation can persist exact state. - restarted=await http.get(signed_path(app,key,user,'/_r/s/start',{})) - assert restarted.status_code==200 - replay_cursor=restarted.json()['sync_cursor'] - replay=await http.get('/_r/s/'+replay_cursor, - headers=signed_header(app,key,user,{'cursor':replay_cursor})) - assert replay.status_code==400 - assert replay.json()['error']['code']=='resync_required' + restarted = await http.get(signed_path(app, key, user, '/_r/s/start', {})) + assert restarted.status_code == 200 + replay_cursor = restarted.json()['sync_cursor'] + replay = await http.get( + '/_r/s/' + replay_cursor, + headers=signed_header(app, key, user, {'cursor': replay_cursor}), + ) + assert replay.status_code == 400 + assert replay.json()['error']['code'] == 'resync_required' assert 'sync_cursor' not in replay.json() and 'next' not in replay.json() - assert await state(app)==before + assert await state(app) == before @pytest.mark.asyncio async def test_sync_topic_membership_change_invalidates_old_cursor(installed): app, _ = installed - key,user,_=await register(app,'sync-topic-member') - creator,creator_id,_=await register(app,'sync-topic-creator') - topic=await call(app,'content.topic_create',{'parent':'/main','name':'sync-topic'}, - key=creator,subject=creator_id) - assert topic.status=='ok' - rid=topic.resources[0].id - async with httpx.AsyncClient(transport=httpx.ASGITransport(app=create_app(app)), - base_url='http://testserver') as http: - opened=await http.get(signed_path(app,key,user,'/_r/s/start',{})) - assert opened.status_code==200 - cursor=opened.json()['sync_cursor'] - joined=await call(app,'content.topic_join',{'id':rid},key=key,subject=user) - assert joined.status=='ok' - before=await state(app) - stale=await http.get(signed_path(app,key,user,'/_r/s/'+cursor,{'cursor':cursor})) - assert stale.status_code==400 - assert stale.json()['error']['code']=='resync_required' - assert await state(app)==before - restarted=await http.get(signed_path(app,key,user,'/_r/s/start',{})) - assert restarted.status_code==200 + key, user, _ = await register(app, 'sync-topic-member') + creator, creator_id, _ = await register(app, 'sync-topic-creator') + topic = await call( + app, + 'content.topic_create', + {'parent': '/main', 'name': 'sync-topic'}, + key=creator, + subject=creator_id, + ) + assert topic.status == 'ok' + rid = topic.resources[0].id + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=create_app(app)), base_url='http://testserver' + ) as http: + opened = await http.get(signed_path(app, key, user, '/_r/s/start', {})) + assert opened.status_code == 200 + cursor = opened.json()['sync_cursor'] + joined = await call(app, 'content.topic_join', {'id': rid}, key=key, subject=user) + assert joined.status == 'ok' + before = await state(app) + stale = await http.get(signed_path(app, key, user, '/_r/s/' + cursor, {'cursor': cursor})) + assert stale.status_code == 400 + assert stale.json()['error']['code'] == 'resync_required' + assert await state(app) == before + restarted = await http.get(signed_path(app, key, user, '/_r/s/start', {})) + assert restarted.status_code == 200 diff --git a/tests/test_system_rule_migration.py b/tests/test_system_rule_migration.py index 0923f8c3..c4124fa7 100644 --- a/tests/test_system_rule_migration.py +++ b/tests/test_system_rule_migration.py @@ -1,4 +1,5 @@ """Release rule identity survives an explicitly declared source relocation.""" + import shutil import pytest @@ -9,57 +10,64 @@ @pytest.mark.asyncio -@pytest.mark.parametrize("container", [tuple, list]) -async def test_source_move_preserves_public_resource_and_revision(installed,tmp_path,container): - app,_=installed - source=tmp_path/'system' - shutil.copytree(system_source_root(),source) - original=source/'rules'/'identity.md' - moved=source/'rules'/'moved'/'identity.md' +@pytest.mark.parametrize('container', [tuple, list]) +async def test_source_move_preserves_public_resource_and_revision(installed, tmp_path, container): + app, _ = installed + source = tmp_path / 'system' + shutil.copytree(system_source_root(), source) + original = source / 'rules' / 'identity.md' + moved = source / 'rules' / 'moved' / 'identity.md' moved.parent.mkdir() original.rename(moved) - paths={'msg.identity':'rules/moved/identity.md'} - declaration={'msg.identity':container(('rules/identity.md','rules/moved/identity.md'))} + paths = {'msg.identity': 'rules/moved/identity.md'} + declaration = {'msg.identity': container(('rules/identity.md', 'rules/moved/identity.md'))} async with app.metadata.transaction(write=False) as tx: - before=await tx.resource('r_rule_identity') - with pytest.raises(Failure,match='system_source_migration_required'): + before = await tx.resource('r_rule_identity') + with pytest.raises(Failure, match='system_source_migration_required'): async with app.metadata.transaction(write=True) as tx: - await sync_system_sources(tx,app.contents,NOW,source_root=source,source_paths=paths) + await sync_system_sources(tx, app.contents, NOW, source_root=source, source_paths=paths) async with app.metadata.transaction(write=True) as tx: - await sync_system_sources(tx,app.contents,NOW,source_root=source, - source_paths=paths,migrations=declaration) + await sync_system_sources( + tx, app.contents, NOW, source_root=source, source_paths=paths, migrations=declaration + ) async with app.metadata.transaction(write=False) as tx: - after=await tx.resource('r_rule_identity') - row=tx.one('SELECT source_path,rule_id,revision_id FROM system_sources WHERE resource_id=?', - ('r_rule_identity',)) - assert (after.id,after.revision,after.generation)==( - before.id,before.revision,before.generation) - assert row==('docs/system/rules/moved/identity.md','msg.identity',before.revision) + after = await tx.resource('r_rule_identity') + row = tx.one( + 'SELECT source_path,rule_id,revision_id FROM system_sources WHERE resource_id=?', + ('r_rule_identity',), + ) + assert (after.id, after.revision, after.generation) == ( + before.id, + before.revision, + before.generation, + ) + assert row == ('docs/system/rules/moved/identity.md', 'msg.identity', before.revision) # The same release declaration remains valid on every subsequent load. async with app.metadata.transaction(write=True) as tx: - await sync_system_sources(tx,app.contents,NOW,source_root=source, - source_paths=paths,migrations=declaration) + await sync_system_sources( + tx, app.contents, NOW, source_root=source, source_paths=paths, migrations=declaration + ) async with app.metadata.transaction(write=False) as tx: - assert (await tx.resource('r_rule_identity')).revision==before.revision + assert (await tx.resource('r_rule_identity')).revision == before.revision @pytest.mark.asyncio -async def test_unknown_duplicate_and_dangling_rule_sources_are_rejected(installed,tmp_path): - app,_=installed - source=tmp_path/'system' - shutil.copytree(system_source_root(),source) - extra=source/'rules'/'extra.md' +async def test_unknown_duplicate_and_dangling_rule_sources_are_rejected(installed, tmp_path): + app, _ = installed + source = tmp_path / 'system' + shutil.copytree(system_source_root(), source) + extra = source / 'rules' / 'extra.md' extra.write_text('\n') - with pytest.raises(Failure,match='unknown_rule_id'): + with pytest.raises(Failure, match='unknown_rule_id'): async with app.metadata.transaction(write=True) as tx: - await sync_system_sources(tx,app.contents,NOW,source_root=source) - extra.write_text((source/'rules'/'identity.md').read_text()) - with pytest.raises(Failure,match='duplicate_rule_id'): + await sync_system_sources(tx, app.contents, NOW, source_root=source) + extra.write_text((source / 'rules' / 'identity.md').read_text()) + with pytest.raises(Failure, match='duplicate_rule_id'): async with app.metadata.transaction(write=True) as tx: - await sync_system_sources(tx,app.contents,NOW,source_root=source) + await sync_system_sources(tx, app.contents, NOW, source_root=source) extra.unlink() - identity=source/'rules'/'identity.md' - identity.write_text(identity.read_text()+'\n\n') - with pytest.raises(Failure,match='dangling_requires_rules'): + identity = source / 'rules' / 'identity.md' + identity.write_text(identity.read_text() + '\n\n') + with pytest.raises(Failure, match='dangling_requires_rules'): async with app.metadata.transaction(write=True) as tx: - await sync_system_sources(tx,app.contents,NOW,source_root=source) + await sync_system_sources(tx, app.contents, NOW, source_root=source) diff --git a/tests/test_system_rule_retirement.py b/tests/test_system_rule_retirement.py index b067fb44..937452d0 100644 --- a/tests/test_system_rule_retirement.py +++ b/tests/test_system_rule_retirement.py @@ -1,15 +1,16 @@ """Release-only retirement preserves history and refuses dangling dependencies.""" -import shutil + import re +import shutil from dataclasses import replace import pytest +from test_service import NOW, call, register from msg.bootstrap import sync_system_sources, system_source_root from msg.core.codec import wire from msg.core.errors import Failure from msg.core.models import ResourceRef -from test_service import NOW, call, register async def retirement(app, tmp_path): @@ -19,19 +20,31 @@ async def retirement(app, tmp_path): for path in source.rglob('*.md'): text = path.read_text() if '/_rules/recovery' in text: - text = re.sub(r'version: (\d+)', lambda match: 'version: ' + str(int(match[1])+1), - text, count=1) - path.write_text('\n'.join(line for line in text.splitlines() - if '/_rules/recovery' not in line) + '\n') + text = re.sub( + r'version: (\d+)', lambda match: 'version: ' + str(int(match[1]) + 1), text, count=1 + ) + path.write_text( + '\n'.join(line for line in text.splitlines() if '/_rules/recovery' not in line) + + '\n' + ) async with app.metadata.transaction(write=False) as tx: - row = tx.one('SELECT source_path,source_version,source_digest FROM system_sources ' - 'WHERE rule_id=?', ('msg.recovery',)) - return source, {'msg.recovery': {'source_path': row[0].removeprefix('docs/system/'), - 'version': row[1], 'digest': row[2]}} + row = tx.one( + 'SELECT source_path,source_version,source_digest FROM system_sources WHERE rule_id=?', + ('msg.recovery',), + ) + return source, { + 'msg.recovery': { + 'source_path': row[0].removeprefix('docs/system/'), + 'version': row[1], + 'digest': row[2], + } + } @pytest.mark.asyncio -async def test_declared_retirement_preserves_history_is_idempotent_and_read_only(installed, tmp_path, monkeypatch): +async def test_declared_retirement_preserves_history_is_idempotent_and_read_only( + installed, tmp_path, monkeypatch +): app, _ = installed source, declaration = await retirement(app, tmp_path) async with app.metadata.transaction(write=False) as tx: @@ -40,16 +53,25 @@ async def test_declared_retirement_preserves_history_is_idempotent_and_read_only original = await app.contents.read_bytes(revision.content) for _ in range(2): async with app.metadata.transaction(write=True) as tx: - await sync_system_sources(tx, app.contents, NOW, source_root=source, - retirements=declaration, registry=app.registry) + await sync_system_sources( + tx, + app.contents, + NOW, + source_root=source, + retirements=declaration, + registry=app.registry, + ) async with app.metadata.transaction(write=False) as tx: after = await tx.resource(before.id) assert after.state == 'archived' assert (after.revision, after.generation) == (before.revision, before.generation + 1) - assert wire(await tx.revision(ResourceRef(id=before.id, revision=before.revision))) == wire(revision) + assert wire( + await tx.revision(ResourceRef(id=before.id, revision=before.revision)) + ) == wire(revision) assert await app.contents.read_bytes(revision.content) == original import msg.bootstrap as bootstrap from msg.application import Application + monkeypatch.setattr(bootstrap, 'SOURCE_RETIREMENTS', declaration) monkeypatch.setattr(bootstrap, 'system_source_root', lambda: source) reloaded = Application(app.settings, clock=lambda: NOW) @@ -65,7 +87,10 @@ async def test_declared_retirement_preserves_history_is_idempotent_and_read_only key, subject, _ = await register(app, 'retirement-writer') for operation, args in ( ('content.chmod', {'id': before.id, 'mode': '0777'}), - ('content.post_edit', {'id': before.id, 'expected_revision': before.revision, 'body': 'replace'}), + ( + 'content.post_edit', + {'id': before.id, 'expected_revision': before.revision, 'body': 'replace'}, + ), ): result = await call(app, operation, args, key=key, subject=subject) assert result.error.code == 'system_managed_resource', wire(result) @@ -76,12 +101,21 @@ async def test_declared_retirement_preserves_history_is_idempotent_and_read_only async def test_retirement_must_pin_current_release_metadata(installed, tmp_path, mismatch): app, _ = installed source, declaration = await retirement(app, tmp_path) - declaration['msg.recovery'][mismatch] = {'version': 999, 'digest': 'sha256:' + '0' * 64, - 'source_path': 'rules/wrong.md'}[mismatch] + declaration['msg.recovery'][mismatch] = { + 'version': 999, + 'digest': 'sha256:' + '0' * 64, + 'source_path': 'rules/wrong.md', + }[mismatch] with pytest.raises(Failure, match='^system_source_retirement_mismatch$'): async with app.metadata.transaction(write=True) as tx: - await sync_system_sources(tx, app.contents, NOW, source_root=source, - retirements=declaration, registry=app.registry) + await sync_system_sources( + tx, + app.contents, + NOW, + source_root=source, + retirements=declaration, + registry=app.registry, + ) async with app.metadata.transaction(write=False) as tx: assert (await tx.resource('r_rule_recovery')).state == 'active' @@ -94,7 +128,9 @@ async def test_retirement_rejects_active_dependencies(installed, tmp_path, depen registry = app.registry if dependency == 'operation': key = ('discovery.get', 1) - registry._operations[key] = replace(registry.operation(*key), requires_rules=('msg.recovery',)) + registry._operations[key] = replace( + registry.operation(*key), requires_rules=('msg.recovery',) + ) elif dependency == 'index': with (source / 'rules/_index.md').open('a') as handle: handle.write('\n[Recovery](/_rules/recovery)\n') @@ -105,14 +141,22 @@ async def test_retirement_rejects_active_dependencies(installed, tmp_path, depen registry = None with pytest.raises(Failure, match='^system_source_retirement_referenced$'): async with app.metadata.transaction(write=True) as tx: - await sync_system_sources(tx, app.contents, NOW, source_root=source, - retirements=declaration, registry=registry) + await sync_system_sources( + tx, + app.contents, + NOW, + source_root=source, + retirements=declaration, + registry=registry, + ) async with app.metadata.transaction(write=False) as tx: assert (await tx.resource('r_rule_recovery')).state == 'active' @pytest.mark.asyncio -async def test_fresh_install_does_not_invent_retired_history(installed, tmp_path, pg_dsn, monkeypatch): +async def test_fresh_install_does_not_invent_retired_history( + installed, tmp_path, pg_dsn, monkeypatch +): import msg.bootstrap as bootstrap from msg.admin.root import _provision from msg.application import Application @@ -121,22 +165,28 @@ async def test_fresh_install_does_not_invent_retired_history(installed, tmp_path source, declaration = await retirement(installed[0], tmp_path) monkeypatch.setattr(bootstrap, 'SOURCE_RETIREMENTS', declaration) monkeypatch.setattr(bootstrap, 'system_source_root', lambda: source) - app = Application(write_example(tmp_path / 'fresh-config', tmp_path / 'fresh-data', - postgres_dsn=pg_dsn), clock=lambda: NOW) + app = Application( + write_example(tmp_path / 'fresh-config', tmp_path / 'fresh-data', postgres_dsn=pg_dsn), + clock=lambda: NOW, + ) try: await _provision(app, 'test-retirement-passphrase') async with app.metadata.transaction(write=False) as tx: assert tx.one('SELECT id FROM resources WHERE id=?', ('r_rule_recovery',)) is None - assert tx.one('SELECT resource_id FROM system_sources WHERE rule_id=?', ('msg.recovery',)) is None + assert ( + tx.one('SELECT resource_id FROM system_sources WHERE rule_id=?', ('msg.recovery',)) + is None + ) assert tx.one('SELECT id FROM resources WHERE id=?', ('r_rule_security',)) is not None finally: await app.close() def test_registry_freeze_rejects_dependency_on_release_retired_rule(monkeypatch): - import msg.bootstrap as bootstrap from test_registry_operation_rules import assemble + import msg.bootstrap as bootstrap + monkeypatch.setattr(bootstrap, 'SOURCE_RETIREMENTS', {'msg.recovery': {}}) registry = assemble('recovery.custom', requires_rules=('msg.recovery',)) with pytest.raises(Failure, match='^dangling_requires_rules$'): @@ -149,26 +199,38 @@ async def test_retirement_cannot_implicitly_reactivate_archived_source(installed app, _ = installed source, declaration = await retirement(app, tmp_path) async with app.metadata.transaction(write=True) as tx: - await sync_system_sources(tx, app.contents, NOW, source_root=source, - retirements=declaration, registry=app.registry) + await sync_system_sources( + tx, + app.contents, + NOW, + source_root=source, + retirements=declaration, + registry=app.registry, + ) shutil.copyfile(system_source_root() / 'rules/recovery.md', source / 'rules/recovery.md') with pytest.raises(Failure, match='^system_source_retirement_required$'): async with app.metadata.transaction(write=True) as tx: - await sync_system_sources(tx, app.contents, NOW, source_root=source, - registry=app.registry) + await sync_system_sources( + tx, app.contents, NOW, source_root=source, registry=app.registry + ) async with app.metadata.transaction(write=False) as tx: assert (await tx.resource('r_rule_recovery')).state == 'archived' @pytest.mark.asyncio -@pytest.mark.parametrize(('change', 'code'), [ - ('unknown', 'system_source_invalid_retirement'), - ('mandatory', 'system_source_invalid_retirement'), - ('malformed', 'system_source_invalid_retirement'), - ('source_present', 'system_source_retirement_source_present'), - ('pointer_drift', 'system_source_pointer_drift'), -]) -async def test_retirement_refuses_invalid_declarations_or_source_drift(installed, tmp_path, change, code): +@pytest.mark.parametrize( + ('change', 'code'), + [ + ('unknown', 'system_source_invalid_retirement'), + ('mandatory', 'system_source_invalid_retirement'), + ('malformed', 'system_source_invalid_retirement'), + ('source_present', 'system_source_retirement_source_present'), + ('pointer_drift', 'system_source_pointer_drift'), + ], +) +async def test_retirement_refuses_invalid_declarations_or_source_drift( + installed, tmp_path, change, code +): app, _ = installed source, declaration = await retirement(app, tmp_path) if change == 'unknown': @@ -182,9 +244,17 @@ async def test_retirement_refuses_invalid_declarations_or_source_drift(installed elif change == 'pointer_drift': async with app.metadata.transaction(write=True) as tx: resource = await tx.resource('r_rule_recovery') - await tx.replace(replace(resource, revision=None, generation=resource.generation+1), - resource.generation) + await tx.replace( + replace(resource, revision=None, generation=resource.generation + 1), + resource.generation, + ) with pytest.raises(Failure, match='^' + code + '$'): async with app.metadata.transaction(write=True) as tx: - await sync_system_sources(tx, app.contents, NOW, source_root=source, - retirements=declaration, registry=app.registry) + await sync_system_sources( + tx, + app.contents, + NOW, + source_root=source, + retirements=declaration, + registry=app.registry, + ) diff --git a/tests/test_system_rules.py b/tests/test_system_rules.py index 94054ffd..06e35c89 100644 --- a/tests/test_system_rules.py +++ b/tests/test_system_rules.py @@ -1,27 +1,29 @@ """Release-owned rules and editable wiki have separate authority boundaries.""" -import shutil + import re +import shutil from dataclasses import replace +import httpx import pytest import tiktoken -import httpx +from test_service import NOW, call, register from msg.core.codec import digest, wire from msg.core.errors import Failure from msg.transports.http import create_app -from test_service import NOW, call, register def test_source_rule_token_budgets_and_index_links(): from msg.bootstrap import RULE_NAMES, system_source_root - root=system_source_root() - encode=tiktoken.get_encoding('cl100k_base').encode - assert len(encode((root/'AGENTS.md').read_text())) <= 200 - index=(root/'rules'/'_index.md').read_text() + + root = system_source_root() + encode = tiktoken.get_encoding('cl100k_base').encode + assert len(encode((root / 'AGENTS.md').read_text())) <= 200 + index = (root / 'rules' / '_index.md').read_text() assert len(encode(index)) <= 400 for name in RULE_NAMES: - source=(root/'rules'/(name+'.md')).read_text() + source = (root / 'rules' / (name + '.md')).read_text() assert len(encode(source)) <= 200 assert f'](/_rules/{name})' in index @@ -34,48 +36,86 @@ async def test_short_bootstrap_rule_index_shards_and_editable_wiki(installed): assert intro.status == index.status == 'ok', (wire(intro), wire(index)) assert '/_rules' in intro.data['content'] and '/wiki' in intro.data['content'] assert len(intro.data['content'].split()) < 150 - async with httpx.AsyncClient(transport=httpx.ASGITransport(app=create_app(app)), - base_url='http://testserver') as http: - index_path=await http.get('/_rules/_index.md') - index_default=await http.get('/_rules') - index_slash=await http.get('/_rules/') - assert index_path.status_code==index_default.status_code==index_slash.status_code==200 - assert index_path.content==index_default.content==index_slash.content - assert index_path.headers['etag']==index_default.headers['etag']==index_slash.headers['etag'] + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=create_app(app)), base_url='http://testserver' + ) as http: + index_path = await http.get('/_rules/_index.md') + index_default = await http.get('/_rules') + index_slash = await http.get('/_rules/') + assert index_path.status_code == index_default.status_code == index_slash.status_code == 200 + assert index_path.content == index_default.content == index_slash.content + assert ( + index_path.headers['etag'] + == index_default.headers['etag'] + == index_slash.headers['etag'] + ) assert 'location' not in index_default.headers - assert (await http.post('/_rules')).status_code==405 - seen=set() - for name in ('identity','read-write','auth','topics','files','recovery','security','protocol'): + assert (await http.post('/_rules')).status_code == 405 + seen = set() + for name in ( + 'identity', + 'read-write', + 'auth', + 'topics', + 'files', + 'recovery', + 'security', + 'protocol', + ): shard = await call(app, 'discovery.get', {'id': '/_rules/' + name}) assert shard.status == 'ok' and 'rule_id:' in shard.data['content'], name assert '/_rules/' + name in index.data['content'] - rule_id=re.search(r'rule_id: ([a-z][a-z0-9.\-]*)',shard.data['content']).group(1) + rule_id = re.search(r'rule_id: ([a-z][a-z0-9.\-]*)', shard.data['content']).group(1) assert rule_id not in seen seen.add(rule_id) key, subject, _ = await register(app, 'wiki-editor') before = digest(index.data['content']) - article = await call(app, 'content.post_create', {'parent': '/wiki', - 'body': 'Community explanation, not platform authority.'}, key=key, subject=subject) + article = await call( + app, + 'content.post_create', + {'parent': '/wiki', 'body': 'Community explanation, not platform authority.'}, + key=key, + subject=subject, + ) assert article.status == 'ok', wire(article) - edited = await call(app, 'content.post_edit', {'id': article.resources[0].id, - 'expected_revision': article.resources[0].revision, - 'body': 'A corrected community explanation.'}, key=key, subject=subject, - expected=((article.resources[0].id,article.data['generation']),)) + edited = await call( + app, + 'content.post_edit', + { + 'id': article.resources[0].id, + 'expected_revision': article.resources[0].revision, + 'body': 'A corrected community explanation.', + }, + key=key, + subject=subject, + expected=((article.resources[0].id, article.data['generation']),), + ) assert edited.status == 'ok', wire(edited) unchanged = await call(app, 'discovery.get', {'id': '/_rules/_index.md'}) assert digest(unchanged.data['content']) == before - for op, args in (('content.chmod', {'id': '/AGENTS.md', 'mode': '0777'}), - ('content.move', {'id': '/_rules/_index.md', 'parent': '/wiki'}), - ('content.post_edit', {'id': '/AGENTS.md', 'expected_revision': 'unused', 'body': 'replace'}), - ('content.post_create', {'parent': '/_rules', 'body': 'fake rule'})): - expected=(('r_rule_index',index.data['generation']),) if op=='content.move' else () + for op, args in ( + ('content.chmod', {'id': '/AGENTS.md', 'mode': '0777'}), + ('content.move', {'id': '/_rules/_index.md', 'parent': '/wiki'}), + ( + 'content.post_edit', + {'id': '/AGENTS.md', 'expected_revision': 'unused', 'body': 'replace'}, + ), + ('content.post_create', {'parent': '/_rules', 'body': 'fake rule'}), + ): + expected = (('r_rule_index', index.data['generation']),) if op == 'content.move' else () result = await call(app, op, args, key=key, subject=subject, expected=expected) - assert result.status == 'error' and result.error.code == 'system_managed_resource', (op, wire(result)) + assert result.status == 'error' and result.error.code == 'system_managed_resource', ( + op, + wire(result), + ) @pytest.mark.asyncio -async def test_per_file_upgrade_changes_only_edited_revision_and_rejects_deleted_source(installed,tmp_path): - from msg.bootstrap import system_source_root, sync_system_sources +async def test_per_file_upgrade_changes_only_edited_revision_and_rejects_deleted_source( + installed, tmp_path +): + from msg.bootstrap import sync_system_sources, system_source_root + app, _ = installed source = tmp_path / 'system' shutil.copytree(system_source_root(), source) @@ -83,62 +123,74 @@ async def test_per_file_upgrade_changes_only_edited_revision_and_rejects_deleted agents_before = (await tx.resource('r_agents')).revision identity_before = (await tx.resource('r_rule_identity')).revision identity = source / 'rules' / 'identity.md' - identity.write_text(identity.read_text().replace('version: 1', 'version: 2') + '\nNew release guidance.\n') + identity.write_text( + identity.read_text().replace('version: 1', 'version: 2') + '\nNew release guidance.\n' + ) async with app.metadata.transaction(write=True) as tx: - await sync_system_sources(tx,app.contents,NOW,source_root=source) + await sync_system_sources(tx, app.contents, NOW, source_root=source) async with app.metadata.transaction(write=False) as tx: assert (await tx.resource('r_agents')).revision == agents_before current = await tx.resource('r_rule_identity') assert current.revision != identity_before - assert (await tx.revision(__import__('msg.core.models',fromlist=['ResourceRef']).ResourceRef( - id=current.id,revision=current.revision))).parents == (identity_before,) - row=tx.one('SELECT source_kind,source_version,source_digest FROM system_sources WHERE resource_id=?', - ('r_rule_identity',)) + assert ( + await tx.revision( + __import__('msg.core.models', fromlist=['ResourceRef']).ResourceRef( + id=current.id, revision=current.revision + ) + ) + ).parents == (identity_before,) + row = tx.one( + 'SELECT source_kind,source_version,source_digest FROM system_sources WHERE resource_id=?', + ('r_rule_identity',), + ) assert row[0] == 'release' and row[1] == 2 and row[2].startswith('sha256:') identity.write_text(identity.read_text() + '\nChanged without a version bump.\n') with pytest.raises(Failure, match='system_source_version_required'): async with app.metadata.transaction(write=True) as tx: - await sync_system_sources(tx,app.contents,NOW,source_root=source) - identity.write_text(identity.read_text().replace('\nChanged without a version bump.\n','')) + await sync_system_sources(tx, app.contents, NOW, source_root=source) + identity.write_text(identity.read_text().replace('\nChanged without a version bump.\n', '')) (source / 'rules' / 'files.md').unlink() with pytest.raises(Failure, match='system_source_deleted_requires_migration'): async with app.metadata.transaction(write=True) as tx: - await sync_system_sources(tx,app.contents,NOW,source_root=source) + await sync_system_sources(tx, app.contents, NOW, source_root=source) @pytest.mark.asyncio -async def test_new_application_load_syncs_changed_packaged_release(installed,tmp_path,monkeypatch): +async def test_new_application_load_syncs_changed_packaged_release( + installed, tmp_path, monkeypatch +): import msg.bootstrap as bootstrap from msg.application import Application + app, _ = installed - source=tmp_path/'system' - shutil.copytree(bootstrap.system_source_root(),source) - rules=source/'rules'/'topics.md' - rules.write_text(rules.read_text().replace('version: 1','version: 2')+'\nRelease update.\n') - monkeypatch.setattr(bootstrap,'system_source_root',lambda:source) + source = tmp_path / 'system' + shutil.copytree(bootstrap.system_source_root(), source) + rules = source / 'rules' / 'topics.md' + rules.write_text(rules.read_text().replace('version: 1', 'version: 2') + '\nRelease update.\n') + monkeypatch.setattr(bootstrap, 'system_source_root', lambda: source) async with app.metadata.transaction(write=False) as tx: - before=(await tx.resource('r_rule_topics')).revision - next_app=Application(app.settings,clock=lambda:NOW) + before = (await tx.resource('r_rule_topics')).revision + next_app = Application(app.settings, clock=lambda: NOW) try: await next_app.load() async with next_app.metadata.transaction(write=False) as tx: - changed=(await tx.resource('r_rule_topics')).revision - assert changed!=before - count=tx.one('SELECT COUNT(*) FROM revisions')[0] + changed = (await tx.resource('r_rule_topics')).revision + assert changed != before + count = tx.one('SELECT COUNT(*) FROM revisions')[0] finally: await next_app.close() - again=Application(app.settings,clock=lambda:NOW) + again = Application(app.settings, clock=lambda: NOW) try: await again.load() async with again.metadata.transaction(write=False) as tx: - assert (await tx.resource('r_rule_topics')).revision==changed - assert tx.one('SELECT COUNT(*) FROM revisions')[0]==count + assert (await tx.resource('r_rule_topics')).revision == changed + assert tx.one('SELECT COUNT(*) FROM revisions')[0] == count finally: await again.close() - (source/'rules'/'files.md').unlink() - missing=Application(app.settings,clock=lambda:NOW) + (source / 'rules' / 'files.md').unlink() + missing = Application(app.settings, clock=lambda: NOW) try: - with pytest.raises(Failure,match='system_source_deleted_requires_migration'): + with pytest.raises(Failure, match='system_source_deleted_requires_migration'): await missing.load() finally: await missing.close() @@ -147,11 +199,13 @@ async def test_new_application_load_syncs_changed_packaged_release(installed,tmp @pytest.mark.asyncio async def test_rule_sync_rejects_current_pointer_drift(installed): from msg.bootstrap import sync_system_sources + app, _ = installed async with app.metadata.transaction(write=True) as tx: - rule=await tx.resource('r_rule_identity') - await tx.replace(replace(rule,revision=None,generation=rule.generation+1), - rule.generation) - with pytest.raises(Failure,match='system_source_pointer_drift'): + rule = await tx.resource('r_rule_identity') + await tx.replace( + replace(rule, revision=None, generation=rule.generation + 1), rule.generation + ) + with pytest.raises(Failure, match='system_source_pointer_drift'): async with app.metadata.transaction(write=True) as tx: - await sync_system_sources(tx,app.contents,NOW) + await sync_system_sources(tx, app.contents, NOW) diff --git a/tests/test_tags_index.py b/tests/test_tags_index.py index bec529ba..b6e3e9dd 100644 --- a/tests/test_tags_index.py +++ b/tests/test_tags_index.py @@ -1,4 +1,5 @@ """Tags are normalized resource metadata; index/search never bypass ACL.""" + from dataclasses import replace from datetime import timedelta @@ -17,32 +18,55 @@ @pytest.mark.asyncio async def test_taggable_types_normalize_tags_and_preserve_revision(installed): app, _ = installed - assert all(app.registry.resource_type(kind).taggable for kind in ('post','topic','repo')) + assert all(app.registry.resource_type(kind).taggable for kind in ('post', 'topic', 'repo')) key, uid, cert = await register(app, 'tag-normalization') - created = await call(app, 'content.post_create', {'parent': '/main', 'body': 'tagged'}, - key=key, subject=uid, certs=(cert,)) + created = await call( + app, + 'content.post_create', + {'parent': '/main', 'body': 'tagged'}, + key=key, + subject=uid, + certs=(cert,), + ) rid, revision = created.resources[0].id, created.resources[0].revision - tagged = await call(app, 'content.tags_set', - {'id': rid, 'tags': [' Café ', 'café', 'ＡＩ', 'ai']}, - key=key, subject=uid, certs=(cert,), - expected=((rid, created.data['generation']),)) + tagged = await call( + app, + 'content.tags_set', + {'id': rid, 'tags': [' Café ', 'café', 'ＡＩ', 'ai']}, + key=key, + subject=uid, + certs=(cert,), + expected=((rid, created.data['generation']),), + ) assert tagged.status == 'ok', tagged.error - projection = await call(app, 'discovery.get', {'id': rid}, - key=key, subject=uid, certs=(cert,)) + projection = await call(app, 'discovery.get', {'id': rid}, key=key, subject=uid, certs=(cert,)) assert projection.data['tags'] == ('ai', 'café') async with app.metadata.transaction(write=False) as tx: resource = await tx.resource(rid) assert resource.tags == ('ai', 'café') assert resource.revision == revision - assert tx.rows('SELECT tag FROM resource_tags WHERE resource_id=? ORDER BY tag', - (rid,)) == [('ai',), ('café',)] - duplicate = await call(app, 'content.tags_set', {'id': rid, 'tags': ['new']}, - key=key, subject=uid, certs=(cert,), - expected=((rid, created.data['generation']),)) + assert tx.rows( + 'SELECT tag FROM resource_tags WHERE resource_id=? ORDER BY tag', (rid,) + ) == [('ai',), ('café',)] + duplicate = await call( + app, + 'content.tags_set', + {'id': rid, 'tags': ['new']}, + key=key, + subject=uid, + certs=(cert,), + expected=((rid, created.data['generation']),), + ) assert duplicate.error.code == 'generation_conflict' - invalid = await call(app, 'content.tags_set', {'id': rid, 'tags': ['bad/tag']}, - key=key, subject=uid, certs=(cert,), - expected=((rid, tagged.data['generation']),)) + invalid = await call( + app, + 'content.tags_set', + {'id': rid, 'tags': ['bad/tag']}, + key=key, + subject=uid, + certs=(cert,), + expected=((rid, tagged.data['generation']),), + ) assert invalid.error.code == 'invalid_tag' @@ -50,21 +74,40 @@ async def test_taggable_types_normalize_tags_and_preserve_revision(installed): async def test_topic_and_repo_use_the_same_tag_index(installed): app, _ = installed key, uid, cert = await register(app, 'tag-topic-repo') - topic = await call(app, 'content.topic_create', {'parent': '/main', 'name': 'tag-topic'}, - key=key, subject=uid, certs=(cert,)) - repo = await call(app, 'git.create', {'parent': uid, 'name': 'tag-repo.git'}, - key=key, subject=uid, certs=(cert,)) + topic = await call( + app, + 'content.topic_create', + {'parent': '/main', 'name': 'tag-topic'}, + key=key, + subject=uid, + certs=(cert,), + ) + repo = await call( + app, + 'git.create', + {'parent': uid, 'name': 'tag-repo.git'}, + key=key, + subject=uid, + certs=(cert,), + ) assert topic.status == repo.status == 'ok' ids = [] for result in (topic, repo): rid = result.resources[0].id - tagged = await call(app, 'content.tags_set', {'id': rid, 'tags': ['Shared']}, - key=key, subject=uid, certs=(cert,), - expected=((rid, result.data['generation']),)) + tagged = await call( + app, + 'content.tags_set', + {'id': rid, 'tags': ['Shared']}, + key=key, + subject=uid, + certs=(cert,), + expected=((rid, result.data['generation']),), + ) assert tagged.status == 'ok', tagged.error ids.append(rid) - async with httpx.AsyncClient(transport=httpx.ASGITransport(app=create_app(app)), - base_url='http://testserver') as http: + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=create_app(app)), base_url='http://testserver' + ) as http: response = await http.get('/_i/by-tag/shared') assert response.status_code == 200, response.text assert {row['id'] for row in response.json()['items']} == set(ids) @@ -76,19 +119,31 @@ async def test_by_tag_index_aliases_paginate_and_hide_revoked_resources(installe key, uid, cert = await register(app, 'tag-index') posts = [] for number in range(3): - created = await call(app, 'content.post_create', - {'parent': '/main', 'body': f'tag post {number}'}, - key=key, subject=uid, certs=(cert,)) + created = await call( + app, + 'content.post_create', + {'parent': '/main', 'body': f'tag post {number}'}, + key=key, + subject=uid, + certs=(cert,), + ) rid = created.resources[0].id - tagged = await call(app, 'content.tags_set', {'id': rid, 'tags': ['Café']}, - key=key, subject=uid, certs=(cert,), - expected=((rid, created.data['generation']),)) + tagged = await call( + app, + 'content.tags_set', + {'id': rid, 'tags': ['Café']}, + key=key, + subject=uid, + certs=(cert,), + expected=((rid, created.data['generation']),), + ) assert tagged.status == 'ok' posts.append((rid, tagged.data['generation'])) async with app.metadata.transaction(write=False) as tx: events_before = tx.one('SELECT COUNT(*) FROM events')[0] - async with httpx.AsyncClient(transport=httpx.ASGITransport(app=create_app(app)), - base_url='http://testserver') as http: + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=create_app(app)), base_url='http://testserver' + ) as http: long = await http.get('/_index/by-tag/caf%C3%A9?limit=1') short = await http.get('/_i/by-tag/caf%C3%A9?limit=1') assert long.status_code == short.status_code == 200, (long.text, short.text) @@ -121,40 +176,86 @@ async def test_by_tag_index_aliases_paginate_and_hide_revoked_resources(installe async def test_by_tag_visibility_changes_immediately_and_signed_search_can_read_private(installed): app, _ = installed key, uid, cert = await register(app, 'tag-private') - created = await call(app, 'content.post_create', {'parent': '/main', 'body': 'secret tag'}, - key=key, subject=uid, certs=(cert,)) + created = await call( + app, + 'content.post_create', + {'parent': '/main', 'body': 'secret tag'}, + key=key, + subject=uid, + certs=(cert,), + ) rid = created.resources[0].id - tagged = await call(app, 'content.tags_set', {'id': rid, 'tags': ['Private']}, - key=key, subject=uid, certs=(cert,), - expected=((rid, created.data['generation']),)) - async with httpx.AsyncClient(transport=httpx.ASGITransport(app=create_app(app)), - base_url='http://testserver') as http: + tagged = await call( + app, + 'content.tags_set', + {'id': rid, 'tags': ['Private']}, + key=key, + subject=uid, + certs=(cert,), + expected=((rid, created.data['generation']),), + ) + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=create_app(app)), base_url='http://testserver' + ) as http: before = await http.get('/_index/by-tag/private') assert [item['id'] for item in before.json()['items']] == [rid] - await call(app, 'content.chmod', {'id': rid, 'mode': '0600'}, key=key, subject=uid, - certs=(cert,), expected=((rid, tagged.data['generation']),)) - async with httpx.AsyncClient(transport=httpx.ASGITransport(app=create_app(app)), - base_url='http://testserver') as http: + await call( + app, + 'content.chmod', + {'id': rid, 'mode': '0600'}, + key=key, + subject=uid, + certs=(cert,), + expected=((rid, tagged.data['generation']),), + ) + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=create_app(app)), base_url='http://testserver' + ) as http: path = '/_index/by-tag/private' anonymous = await http.get(path) assert anonymous.status_code == 200 assert anonymous.json()['items'] == [] assert 'etag' not in anonymous.headers - packet = request_for('discovery.list', {'tag': 'private', 'limit': 50}, - app.settings.service_url, signer=key, subject=uid, - expires_at=NOW + timedelta(seconds=120)) + packet = request_for( + 'discovery.list', + {'tag': 'private', 'limit': 50}, + app.settings.service_url, + signer=key, + subject=uid, + expires_at=NOW + timedelta(seconds=120), + ) permitted = await http.get(path, headers={'X-Msg-Request': b64(canonical(packet))}) assert permitted.status_code == 200, permitted.text assert [item['id'] for item in permitted.json()['items']] == [rid] - wrong = await http.get(path, headers={'X-Msg-Request': b64(canonical(request_for( - 'discovery.list', {'tag': 'other', 'limit': 50}, app.settings.service_url, - signer=key, subject=uid, expires_at=NOW + timedelta(seconds=120))))}) + wrong = await http.get( + path, + headers={ + 'X-Msg-Request': b64( + canonical( + request_for( + 'discovery.list', + {'tag': 'other', 'limit': 50}, + app.settings.service_url, + signer=key, + subject=uid, + expires_at=NOW + timedelta(seconds=120), + ) + ) + ) + }, + ) assert wrong.status_code == 400 assert wrong.json()['error']['code'] == 'representation_mismatch' - changed = await call(app, 'content.tags_set', {'id': rid, 'tags': []}, - key=key, subject=uid, certs=(cert,), - expected=((rid, tagged.data['generation'] + 1),)) + changed = await call( + app, + 'content.tags_set', + {'id': rid, 'tags': []}, + key=key, + subject=uid, + certs=(cert,), + expected=((rid, tagged.data['generation'] + 1),), + ) assert changed.status == 'ok', changed.error removed = await http.get(path, headers={'X-Msg-Request': b64(canonical(packet))}) assert removed.status_code == 200 and removed.json()['items'] == [] @@ -171,10 +272,14 @@ async def test_tags_index_has_same_transactional_contract_on_both_stores(backend async with store.transaction(write=True) as tx: old = await tx.resource('post') assert old.tags == ('ai',) - assert tx.rows('SELECT resource_id FROM resource_tags WHERE tag=?', ('ai',)) == [('post',)] + assert tx.rows('SELECT resource_id FROM resource_tags WHERE tag=?', ('ai',)) == [ + ('post',) + ] await tx.replace(replace(old, tags=('café',), generation=1), 0) async with store.transaction(write=False) as tx: assert (await tx.resource('post')).tags == ('café',) - assert tx.rows('SELECT tag FROM resource_tags WHERE resource_id=?', ('post',)) == [('café',)] + assert tx.rows('SELECT tag FROM resource_tags WHERE resource_id=?', ('post',)) == [ + ('café',) + ] finally: await store.close() diff --git a/tests/test_task_a_custodial_binding.py b/tests/test_task_a_custodial_binding.py index 264f2523..b7343f3d 100644 --- a/tests/test_task_a_custodial_binding.py +++ b/tests/test_task_a_custodial_binding.py @@ -3,6 +3,7 @@ These cases use real PostgreSQL and real Ed25519/age; a corrupted row must not become trusted merely because there is no ciphertext ACK to verify. """ + from copy import deepcopy import pytest @@ -13,7 +14,7 @@ from msg.core.models import Credential from msg.security.age_keys import encryption_key_id, generate_age_key, public_from_recipient from msg.security.crypto import Ed25519Signer -from msg.security.custodial_migration import snapshot +from msg.security.custodial_migration import snapshot as snapshot from msg.storage.postgres import PostgresMetadataStore @@ -22,25 +23,55 @@ async def bound_history(pg_dsn): store = PostgresMetadataStore(pg_dsn) old, new = Ed25519Signer.generate(), Ed25519Signer.generate() old_recipient, recipient = generate_age_key()[1], generate_age_key()[1] - challenge = {'subject_id': 'u_owner', 'challenge_id': 'cupg_bound', - 'public_key': b64(new.public_key), 'encryption_recipient': recipient} - details = {'age_inventory': [], 'new_identity_key_id': new.key_id, - 'old_identity_key_id': old.key_id, - 'new_encryption_key_id': encryption_key_id(public_from_recipient(recipient)), - 'old_encryption_key_id': encryption_key_id(public_from_recipient(old_recipient)), - 'server_signature': wire(old.sign(canonical(challenge), - purpose='custodial-upgrade-challenge'))} - credential = Credential(id=old.key_id, subject_id='u_owner', kind='signing_key', - verifier=old.public_key, ceiling=(), not_before=NOW, - expires_at=None, revoked_at=None) + challenge = { + 'subject_id': 'u_owner', + 'challenge_id': 'cupg_bound', + 'public_key': b64(new.public_key), + 'encryption_recipient': recipient, + } + details = { + 'age_inventory': [], + 'new_identity_key_id': new.key_id, + 'old_identity_key_id': old.key_id, + 'new_encryption_key_id': encryption_key_id(public_from_recipient(recipient)), + 'old_encryption_key_id': encryption_key_id(public_from_recipient(old_recipient)), + 'server_signature': wire( + old.sign(canonical(challenge), purpose='custodial-upgrade-challenge') + ), + } + credential = Credential( + id=old.key_id, + subject_id='u_owner', + kind='signing_key', + verifier=old.public_key, + ceiling=(), + not_before=NOW, + expires_at=None, + revoked_at=None, + ) async with store.transaction(write=True) as tx: - tx.execute('INSERT INTO credentials VALUES (?,?,?)', - (old.key_id, 'u_owner', canonical(credential).decode()), write=True) - for key, address, primary in ((details['old_encryption_key_id'], old_recipient, 0), - (details['new_encryption_key_id'], recipient, 1)): - tx.execute('INSERT INTO encryption_subkeys VALUES (?,?,?,?,?,?,?)', - (key, 'u_owner', address, b64(public_from_recipient(address)), - wire(NOW), wire(NOW) if not primary else None, primary), write=True) + tx.execute( + 'INSERT INTO credentials VALUES (?,?,?)', + (old.key_id, 'u_owner', canonical(credential).decode()), + write=True, + ) + for key, address, primary in ( + (details['old_encryption_key_id'], old_recipient, 0), + (details['new_encryption_key_id'], recipient, 1), + ): + tx.execute( + 'INSERT INTO encryption_subkeys VALUES (?,?,?,?,?,?,?)', + ( + key, + 'u_owner', + address, + b64(public_from_recipient(address)), + wire(NOW), + wire(NOW) if not primary else None, + primary, + ), + write=True, + ) yield store, challenge, details await store.close() @@ -57,21 +88,33 @@ async def test_valid_empty_inventory_is_read_only_and_not_backup_retirement(boun @pytest.mark.asyncio -@pytest.mark.parametrize('field', ['subject_id', 'challenge_id', 'public_key', 'encryption_recipient']) +@pytest.mark.parametrize( + 'field', ['subject_id', 'challenge_id', 'public_key', 'encryption_recipient'] +) async def test_persisted_challenge_tampering_is_rejected_without_any_acks(bound_history, field): store, challenge, details = bound_history changed = dict(challenge) - changed[field] = {'subject_id': 'u_other', 'challenge_id': 'cupg_other', - 'public_key': b64(Ed25519Signer.generate().public_key), - 'encryption_recipient': generate_age_key()[1]}[field] + changed[field] = { + 'subject_id': 'u_other', + 'challenge_id': 'cupg_other', + 'public_key': b64(Ed25519Signer.generate().public_key), + 'encryption_recipient': generate_age_key()[1], + }[field] async with store.transaction(write=False) as tx: with pytest.raises(Failure, match='^custodial_challenge_invalid$'): await snapshot(tx, 'u_owner', 'pending_rewrap', changed, details) @pytest.mark.asyncio -@pytest.mark.parametrize('field', ['new_identity_key_id', 'new_encryption_key_id', - 'old_identity_key_id', 'old_encryption_key_id']) +@pytest.mark.parametrize( + 'field', + [ + 'new_identity_key_id', + 'new_encryption_key_id', + 'old_identity_key_id', + 'old_encryption_key_id', + ], +) async def test_stored_key_id_cannot_override_the_signed_challenge(bound_history, field): store, challenge, details = bound_history changed = dict(details, **{field: 'foreign-key'}) @@ -84,8 +127,11 @@ async def test_stored_key_id_cannot_override_the_signed_challenge(bound_history, async def test_retired_new_target_blocks_in_progress_migration(bound_history): store, challenge, details = bound_history async with store.transaction(write=True) as tx: - tx.execute('UPDATE encryption_subkeys SET retired_at=?,is_primary=0 WHERE key_id=?', - (wire(NOW), details['new_encryption_key_id']), write=True) + tx.execute( + 'UPDATE encryption_subkeys SET retired_at=?,is_primary=0 WHERE key_id=?', + (wire(NOW), details['new_encryption_key_id']), + write=True, + ) async with store.transaction(write=False) as tx: with pytest.raises(Failure, match='^custodial_encryption_target_changed$'): await snapshot(tx, 'u_owner', 'pending_rewrap', challenge, details) diff --git a/tests/test_task_a_preflight.py b/tests/test_task_a_preflight.py index 8dea6af8..214fef61 100644 --- a/tests/test_task_a_preflight.py +++ b/tests/test_task_a_preflight.py @@ -10,17 +10,23 @@ def tree(path): - return {str(file.relative_to(path)): file.read_bytes() for file in path.rglob('*') if file.is_file()} + return { + str(file.relative_to(path)): file.read_bytes() for file in path.rglob('*') if file.is_file() + } def test_preflight_does_not_initialize_an_empty_database(pg_dsn, tmp_path): - settings = write_example(tmp_path / 'etc', tmp_path / 'data', 'http://testserver', postgres_dsn=pg_dsn) + settings = write_example( + tmp_path / 'etc', tmp_path / 'data', 'http://testserver', postgres_dsn=pg_dsn + ) before = tree(tmp_path) result = preflight(settings.config_dir) assert result['decision'] == 'blocked' and result['mutation_performed'] is False assert result['database']['state'] == 'empty' with psycopg.connect(pg_dsn) as conn: - assert conn.execute("SELECT count(*) FROM pg_tables WHERE schemaname='public'").fetchone() == (0,) + assert conn.execute( + "SELECT count(*) FROM pg_tables WHERE schemaname='public'" + ).fetchone() == (0,) assert tree(tmp_path) == before assert pg_dsn not in str(result) assert all(item['status'] == 'missing' for item in result['field_evidence']) @@ -28,7 +34,9 @@ def test_preflight_does_not_initialize_an_empty_database(pg_dsn, tmp_path): def test_existing_schema_is_checked_without_touching_sequences_or_files(pg_dsn, tmp_path): PostgresMetadataStore(pg_dsn) - settings = write_example(tmp_path / 'etc', tmp_path / 'data', 'http://testserver', postgres_dsn=pg_dsn) + settings = write_example( + tmp_path / 'etc', tmp_path / 'data', 'http://testserver', postgres_dsn=pg_dsn + ) before = tree(tmp_path) with psycopg.connect(pg_dsn) as conn: seq = conn.execute('SELECT last_value,is_called FROM money_ledger_seq_seq').fetchone() @@ -38,12 +46,17 @@ def test_existing_schema_is_checked_without_touching_sequences_or_files(pg_dsn, assert result['decision'] == 'blocked' # A consistent test DB is not a field attestation. assert result['database']['ca_inventory'] == [] with psycopg.connect(pg_dsn) as conn: - assert conn.execute('SELECT last_value,is_called FROM money_ledger_seq_seq').fetchone() == seq + assert ( + conn.execute('SELECT last_value,is_called FROM money_ledger_seq_seq').fetchone() == seq + ) assert conn.execute('SELECT COUNT(*) FROM audit').fetchone() == (0,) assert conn.execute('SELECT COUNT(*) FROM jobs').fetchone() == (0,) assert tree(tmp_path) == before def test_inspection_refuses_a_writable_transaction(pg_dsn): - with psycopg.connect(pg_dsn) as conn, pytest.raises(Failure, match='^preflight_read_only_required$'): + with ( + psycopg.connect(pg_dsn) as conn, + pytest.raises(Failure, match='^preflight_read_only_required$'), + ): inspect_database(conn) diff --git a/tests/test_task_a_recovery_checkpoint.py b/tests/test_task_a_recovery_checkpoint.py index 77e3fa32..1202f46b 100644 --- a/tests/test_task_a_recovery_checkpoint.py +++ b/tests/test_task_a_recovery_checkpoint.py @@ -1,4 +1,5 @@ """Independent checkpoint pins, not database flags, authorize partial authority replay.""" + import asyncio from copy import deepcopy from dataclasses import replace @@ -16,36 +17,69 @@ async def replay(store, packet, *, pin): - return await _replay(store, packet, pin=pin, operator="isolated-test-usecase") + return await _replay(store, packet, pin=pin, operator='isolated-test-usecase') @pytest.fixture def checkpoint(): signer = Ed25519Signer.generate() - body = {'format': 'msg-revocation-checkpoint-v1', 'service': 'http://testserver', - 'source_backup_sha256': 'a' * 64, 'sequence': 1, - 'entries': [{'sequence': 1, 'kind': 'credential.revoke', 'subject': 'u_owner', - 'target': 'token_one', 'at': wire(NOW)}]} - pin = TrustedCheckpointPin(service=body['service'], public_key=signer.public_key, - digest=digest(body), sequence=1) + body = { + 'format': 'msg-revocation-checkpoint-v1', + 'service': 'http://testserver', + 'source_backup_sha256': 'a' * 64, + 'sequence': 1, + 'entries': [ + { + 'sequence': 1, + 'kind': 'credential.revoke', + 'subject': 'u_owner', + 'target': 'token_one', + 'at': wire(NOW), + } + ], + } + pin = TrustedCheckpointPin( + service=body['service'], public_key=signer.public_key, digest=digest(body), sequence=1 + ) + def signed(value): - return {'checkpoint': value, - 'signature': wire(signer.sign(canonical(value), purpose='recovery-checkpoint-v1'))} + return { + 'checkpoint': value, + 'signature': wire(signer.sign(canonical(value), purpose='recovery-checkpoint-v1')), + } + return body, pin, signed @pytest.fixture async def restored(pg_dsn): store = PostgresMetadataStore(pg_dsn) - token = Credential(id='token_one', subject_id='u_owner', kind='token', verifier=b'x' * 32, - ceiling=(), not_before=NOW, expires_at=None, revoked_at=None) + token = Credential( + id='token_one', + subject_id='u_owner', + kind='token', + verifier=b'x' * 32, + ceiling=(), + not_before=NOW, + expires_at=None, + revoked_at=None, + ) async with store.transaction(write=True) as tx: - tx.set_setting(SETTING, {'format': 'msg-recovery-quarantine-v1', - 'source_backup_sha256': 'a' * 64, - 'outbound_enabled': False, 'authority': 'health_only', - 'revocation_replay': 'required'}) - tx.execute('INSERT INTO credentials VALUES (?,?,?)', - (token.id, token.subject_id, canonical(token).decode()), write=True) + tx.set_setting( + SETTING, + { + 'format': 'msg-recovery-quarantine-v1', + 'source_backup_sha256': 'a' * 64, + 'outbound_enabled': False, + 'authority': 'health_only', + 'revocation_replay': 'required', + }, + ) + tx.execute( + 'INSERT INTO credentials VALUES (?,?,?)', + (token.id, token.subject_id, canonical(token).decode()), + write=True, + ) yield store await store.close() @@ -89,10 +123,17 @@ async def test_replay_is_atomic_monotonic_idempotent_and_never_promotes(restored body, pin, signed = checkpoint results = await asyncio.gather(*(replay(restored, signed(body), pin=pin) for _ in range(3))) assert sum(result['changed'] for result in results) == 1 - assert all(result['promotion'] == 'blocked' and result['backup_retired'] is False and - result['promotion_blocked_reasons'] == [ - 'quarantine_remains', 'supported_fact_inventory_incomplete', - 'backup_retirement_not_attested'] for result in results) + assert all( + result['promotion'] == 'blocked' + and result['backup_retired'] is False + and result['promotion_blocked_reasons'] + == [ + 'quarantine_remains', + 'supported_fact_inventory_incomplete', + 'backup_retirement_not_attested', + ] + for result in results + ) async with restored.transaction(write=False) as tx: assert active(tx) credential = await tx.credential('token_one') @@ -113,8 +154,13 @@ async def test_replay_is_atomic_monotonic_idempotent_and_never_promotes(restored async def test_missing_or_wrong_subject_fact_rolls_back_earlier_revocation(restored, checkpoint): body, pin, signed = checkpoint body = deepcopy(body) - body['entries'].append({'sequence': 2, 'kind': 'credential.revoke', - 'subject': 'u_owner', 'target': 'missing', 'at': wire(NOW)}) + body['entries'].append({ + 'sequence': 2, + 'kind': 'credential.revoke', + 'subject': 'u_owner', + 'target': 'missing', + 'at': wire(NOW), + }) body['sequence'] = 2 pin = replace(pin, digest=digest(body), sequence=2) with pytest.raises(Failure, match='^recovery_fact_missing$'): @@ -145,7 +191,9 @@ async def test_wrong_backup_and_live_database_are_not_replay_targets(restored, c @pytest.mark.asyncio -async def test_cached_receipt_cannot_authorize_regression_or_skip_restored_authority(restored, checkpoint): +async def test_cached_receipt_cannot_authorize_regression_or_skip_restored_authority( + restored, checkpoint +): body, pin, signed = checkpoint await replay(restored, signed(body), pin=pin) # A coherent current DB receipt is only a guard, never an external freshness source. @@ -158,8 +206,11 @@ async def test_cached_receipt_cannot_authorize_regression_or_skip_restored_autho await replay(restored, signed(corrupt), pin=replace(pin, digest=digest(corrupt))) async with restored.transaction(write=True) as tx: token = await tx.credential('token_one') - tx.execute('UPDATE credentials SET body=? WHERE id=?', - (canonical(replace(token, revoked_at=None)).decode(), token.id), write=True) + tx.execute( + 'UPDATE credentials SET body=? WHERE id=?', + (canonical(replace(token, revoked_at=None)).decode(), token.id), + write=True, + ) # Re-applying the pinned full log re-revokes the row, even with an identical cached receipt. assert (await replay(restored, signed(body), pin=pin))['changed'] == 1 async with restored.transaction(write=False) as tx: @@ -170,6 +221,7 @@ async def test_cached_receipt_cannot_authorize_regression_or_skip_restored_autho @pytest.mark.asyncio async def test_actual_backend_termination_rolls_back_and_replay_can_resume(restored, checkpoint): import psycopg + body, pin, signed = checkpoint # Kill the actual PostgreSQL session after its revoke but before its COMMIT. # No patched driver or mock transaction supplies the rollback evidence. @@ -178,7 +230,9 @@ async def test_actual_backend_termination_rolls_back_and_replay_can_resume(resto assert (await replay(restored, signed(body), pin=pin))['changed'] == 1 pid = tx.one('SELECT pg_backend_pid()')[0] with psycopg.connect(restored.dsn) as killer: - assert killer.execute('SELECT pg_terminate_backend(%s)', (pid,)).fetchone() == (True,) + assert killer.execute('SELECT pg_terminate_backend(%s)', (pid,)).fetchone() == ( + True, + ) tx.one('SELECT 1') async with restored.transaction(write=False) as tx: assert (await tx.credential('token_one')).revoked_at is None @@ -188,14 +242,17 @@ async def test_actual_backend_termination_rolls_back_and_replay_can_resume(resto @pytest.mark.asyncio -async def test_public_replay_entry_rejects_nonconsole_before_database_access(restored, checkpoint, tmp_path): +async def test_public_replay_entry_rejects_nonconsole_before_database_access( + restored, checkpoint, tmp_path +): import os import subprocess import sys from pathlib import Path from msg.core.codec import loads - script = ''' + + script = """ import asyncio from pathlib import Path from msg.admin.recovery_replay import replay @@ -205,15 +262,24 @@ async def test_public_replay_entry_rejects_nonconsole_before_database_access(res asyncio.run(replay(None, {}, pin=None, config_dir=Path('.'))) except Failure as exc: print(canonical({'code': exc.code}).decode()) -''' +""" # Capture pipes are not a physical console. On the non-root CI runner the # real OS-admin check must reject even earlier; do not mock uid or accept an # arbitrary error to disguise the order of these independent safety gates. - environment = {key: value for key, value in os.environ.items() - if key not in {'SSH_CONNECTION', 'SSH_CLIENT', 'SSH_TTY'}} + environment = { + key: value + for key, value in os.environ.items() + if key not in {'SSH_CONNECTION', 'SSH_CLIENT', 'SSH_TTY'} + } environment['PYTHONPATH'] = str(Path(__file__).resolve().parents[1] / 'src') - result = await asyncio.to_thread(subprocess.run, [sys.executable, '-c', script], - capture_output=True, text=True, check=True, env=environment) + result = await asyncio.to_thread( + subprocess.run, + [sys.executable, '-c', script], + capture_output=True, + text=True, + check=True, + env=environment, + ) expected = 'local_console_required' if os.geteuid() == 0 else 'local_os_administrator_required' assert loads(result.stdout) == {'code': expected} async with restored.transaction(write=False) as tx: @@ -223,14 +289,22 @@ async def test_public_replay_entry_rejects_nonconsole_before_database_access(res def test_offline_selftest_is_not_field_or_database_acceptance(): from msg.admin.recovery_replay import selftest - assert selftest() == {'format': 'msg-revocation-checkpoint-v1', - 'scope': 'isolated_signature_contract_only', 'checks': 3, - 'status': 'passed', 'database_used': False, - 'production_evidence': False, 'promotion': 'blocked'} + + assert selftest() == { + 'format': 'msg-revocation-checkpoint-v1', + 'scope': 'isolated_signature_contract_only', + 'checks': 3, + 'status': 'passed', + 'database_used': False, + 'production_evidence': False, + 'promotion': 'blocked', + } @pytest.mark.asyncio -async def test_all_supported_owned_revocations_preserve_signed_bytes_and_never_import_keys(restored, checkpoint): +async def test_all_supported_owned_revocations_preserve_signed_bytes_and_never_import_keys( + restored, checkpoint +): from datetime import timedelta from msg.admin.recovery_replay import SUPPORTED_FACTS @@ -239,93 +313,232 @@ async def test_all_supported_owned_revocations_preserve_signed_bytes_and_never_i from msg.security.age_keys import encryption_key_id, generate_age_key, public_from_recipient from msg.security.certificates import sign_certificate from msg.security.crypto import key_id + body, pin, signed = checkpoint owner = 'u_owner' signer = Ed25519Signer.generate() recipient = generate_age_key()[1] age_id = encryption_key_id(public_from_recipient(recipient)) - credential = Credential(id=signer.key_id, subject_id=owner, kind='signing_key', - verifier=signer.public_key, ceiling=(), not_before=NOW, expires_at=None, revoked_at=None) - certificate = sign_certificate(Certificate(resource_id='cert_one', serial='serial-one', - subject_id=owner, key_id=key_id(signer.public_key), issuer_id=owner, - parent_certificate_id=None, authority_sources=(), kind='capability', grants=(), - not_before=NOW, expires_at=NOW+timedelta(days=1), target_service=pin.service, - delegation_depth=0, issuance=None, - signature=Signature(key_id=signer.key_id, algorithm='ed25519', value=b'')), signer) + credential = Credential( + id=signer.key_id, + subject_id=owner, + kind='signing_key', + verifier=signer.public_key, + ceiling=(), + not_before=NOW, + expires_at=None, + revoked_at=None, + ) + certificate = sign_certificate( + Certificate( + resource_id='cert_one', + serial='serial-one', + subject_id=owner, + key_id=key_id(signer.public_key), + issuer_id=owner, + parent_certificate_id=None, + authority_sources=(), + kind='capability', + grants=(), + not_before=NOW, + expires_at=NOW + timedelta(days=1), + target_service=pin.service, + delegation_depth=0, + issuance=None, + signature=Signature(key_id=signer.key_id, algorithm='ed25519', value=b''), + ), + signer, + ) original = canonical(certificate).decode() member = Membership(organization_id='g_test', subject_id=owner, role='member', version=1) - resource = Resource(id='t_test', type='topic', type_version=1, name='test', parent=None, - owner=owner, group='g_public', mode=0o700, generation=1, revision=None, state='active', - created_at=NOW, created_by=owner, modified_at=NOW, modified_by=owner) + resource = Resource( + id='t_test', + type='topic', + type_version=1, + name='test', + parent=None, + owner=owner, + group='g_public', + mode=0o700, + generation=1, + revision=None, + state='active', + created_at=NOW, + created_by=owner, + modified_at=NOW, + modified_by=owner, + ) async with restored.transaction(write=True) as tx: await tx.insert(resource) - tx.execute('INSERT INTO credentials VALUES (?,?,?)', - (credential.id, owner, canonical(credential).decode()), write=True) - tx.execute('INSERT INTO certificates VALUES (?,?,NULL,0,?)', - (certificate.resource_id, owner, original), write=True) - tx.execute('INSERT INTO memberships VALUES (?,?,?,?)', - (member.organization_id, owner, member.version, canonical(member).decode()), write=True) - tx.execute('INSERT INTO topic_memberships VALUES (?,?,?,?,?,NULL)', - (resource.id, owner, 'member', 'active', wire(NOW)), write=True) - tx.execute('INSERT INTO identity_keys VALUES (?,?,?,?,NULL,1)', - (signer.key_id, owner, b64(signer.public_key), wire(NOW)), write=True) - tx.execute('INSERT INTO encryption_subkeys VALUES (?,?,?,?,?,NULL,1)', - (age_id, owner, recipient, b64(public_from_recipient(recipient)), wire(NOW)), write=True) - tx.execute('INSERT INTO custodial_vault VALUES (?,?,?,?,?,?,?,?,?,NULL)', - (owner, signer.key_id, age_id, 'signing-nonce', 'sealed-signing', - 'age-nonce', 'sealed-age', 'active', wire(NOW)), write=True) - tx.execute('INSERT INTO token_deliveries VALUES (?,?,?,?,?,?,NULL,NULL)', - ('token_one', owner, 'original-create', 'digest', 'verifier', - wire(NOW+timedelta(hours=1))), write=True) - tx.execute('INSERT INTO share_grants VALUES (?,?,?,?,?,?,NULL)', - ('sg_one', resource.id, owner, 'u_reader', wire(NOW), - wire(NOW+timedelta(days=1))), write=True) - tx.execute('INSERT INTO share_grants_v2 VALUES (?,?,?,?,?,NULL,?,?,0,?,?,NULL)', - ('sg_two', resource.id, owner, 'u_reader', 'user', '["read"]', '{}', - wire(NOW), wire(NOW+timedelta(days=1))), write=True) - tx.execute('INSERT INTO share_links VALUES (?,?,?,?,?,?,?,NULL)', - ('link_one', resource.id, owner, signer.key_id, 'link-verifier', - wire(NOW), wire(NOW+timedelta(days=1))), write=True) - tx.execute('''INSERT INTO topic_bans (topic,subject,actor,created_at,expires_at,reason,status) - VALUES (?,?,?,?,NULL,?,?)''', - (resource.id, owner, owner, wire(NOW), 'restored-ban', 'active'), write=True) + tx.execute( + 'INSERT INTO credentials VALUES (?,?,?)', + (credential.id, owner, canonical(credential).decode()), + write=True, + ) + tx.execute( + 'INSERT INTO certificates VALUES (?,?,NULL,0,?)', + (certificate.resource_id, owner, original), + write=True, + ) + tx.execute( + 'INSERT INTO memberships VALUES (?,?,?,?)', + (member.organization_id, owner, member.version, canonical(member).decode()), + write=True, + ) + tx.execute( + 'INSERT INTO topic_memberships VALUES (?,?,?,?,?,NULL)', + (resource.id, owner, 'member', 'active', wire(NOW)), + write=True, + ) + tx.execute( + 'INSERT INTO identity_keys VALUES (?,?,?,?,NULL,1)', + (signer.key_id, owner, b64(signer.public_key), wire(NOW)), + write=True, + ) + tx.execute( + 'INSERT INTO encryption_subkeys VALUES (?,?,?,?,?,NULL,1)', + (age_id, owner, recipient, b64(public_from_recipient(recipient)), wire(NOW)), + write=True, + ) + tx.execute( + 'INSERT INTO custodial_vault VALUES (?,?,?,?,?,?,?,?,?,NULL)', + ( + owner, + signer.key_id, + age_id, + 'signing-nonce', + 'sealed-signing', + 'age-nonce', + 'sealed-age', + 'active', + wire(NOW), + ), + write=True, + ) + tx.execute( + 'INSERT INTO token_deliveries VALUES (?,?,?,?,?,?,NULL,NULL)', + ( + 'token_one', + owner, + 'original-create', + 'digest', + 'verifier', + wire(NOW + timedelta(hours=1)), + ), + write=True, + ) + tx.execute( + 'INSERT INTO share_grants VALUES (?,?,?,?,?,?,NULL)', + ('sg_one', resource.id, owner, 'u_reader', wire(NOW), wire(NOW + timedelta(days=1))), + write=True, + ) + tx.execute( + 'INSERT INTO share_grants_v2 VALUES (?,?,?,?,?,NULL,?,?,0,?,?,NULL)', + ( + 'sg_two', + resource.id, + owner, + 'u_reader', + 'user', + '["read"]', + '{}', + wire(NOW), + wire(NOW + timedelta(days=1)), + ), + write=True, + ) + tx.execute( + 'INSERT INTO share_links VALUES (?,?,?,?,?,?,?,NULL)', + ( + 'link_one', + resource.id, + owner, + signer.key_id, + 'link-verifier', + wire(NOW), + wire(NOW + timedelta(days=1)), + ), + write=True, + ) + tx.execute( + """INSERT INTO topic_bans (topic,subject,actor,created_at,expires_at,reason,status) + VALUES (?,?,?,?,NULL,?,?)""", + (resource.id, owner, owner, wire(NOW), 'restored-ban', 'active'), + write=True, + ) banned = replace(resource, id='t_ban', name='ban', parent=resource.id) await tx.insert(banned) - targets = [('credential.revoke', 'token_one'), ('certificate.revoke', 'cert_one'), - ('share_grant.revoke', 'sg_one'), ('share_grant_v2.revoke', 'sg_two'), - ('share_link.revoke', 'link_one'), ('membership.remove', 'g_test'), - ('topic_membership.remove', 't_test'), ('topic_ban.apply', 't_ban'), - ('topic_ban.lift', resource.id), - ('identity_key.retire', signer.key_id), - ('encryption_key.retire', age_id), ('vault.destroy', age_id)] + targets = [ + ('credential.revoke', 'token_one'), + ('certificate.revoke', 'cert_one'), + ('share_grant.revoke', 'sg_one'), + ('share_grant_v2.revoke', 'sg_two'), + ('share_link.revoke', 'link_one'), + ('membership.remove', 'g_test'), + ('topic_membership.remove', 't_test'), + ('topic_ban.apply', 't_ban'), + ('topic_ban.lift', resource.id), + ('identity_key.retire', signer.key_id), + ('encryption_key.retire', age_id), + ('vault.destroy', age_id), + ] assert {kind for kind, _ in targets} == SUPPORTED_FACTS - facts = [{'sequence': i, 'kind': kind, 'subject': owner, 'target': target, 'at': wire(NOW)} - for i, (kind, target) in enumerate(targets, 1)] + facts = [ + {'sequence': i, 'kind': kind, 'subject': owner, 'target': target, 'at': wire(NOW)} + for i, (kind, target) in enumerate(targets, 1) + ] body = dict(body, entries=facts, sequence=len(facts)) pin = replace(pin, sequence=len(facts), digest=digest(body)) result = await replay(restored, signed(body), pin=pin) assert result['changed'] == len(facts) and result['backup_retired'] is False assert result['promotion_blocked_reasons'] == [ - 'quarantine_remains', 'supported_fact_inventory_incomplete', 'backup_retirement_not_attested'] + 'quarantine_remains', + 'supported_fact_inventory_incomplete', + 'backup_retirement_not_attested', + ] async with restored.transaction(write=False) as tx: - assert tx.one('SELECT body,revoked FROM certificates WHERE id=?', ('cert_one',)) == (original, 1) + assert tx.one('SELECT body,revoked FROM certificates WHERE id=?', ('cert_one',)) == ( + original, + 1, + ) assert (await tx.credential(signer.key_id)).revoked_at == NOW - assert tx.one('SELECT consumed_at FROM token_deliveries WHERE credential_id=?', ('token_one',)) == (wire(NOW),) + assert tx.one( + 'SELECT consumed_at FROM token_deliveries WHERE credential_id=?', ('token_one',) + ) == (wire(NOW),) for table in ('identity_keys', 'encryption_subkeys'): - assert tx.one(f'SELECT retired_at,is_primary FROM {table} WHERE subject=?', (owner,)) == (wire(NOW), 0) - for table, target in (('share_grants', 'sg_one'), ('share_grants_v2', 'sg_two'), ('share_links', 'link_one')): + assert tx.one( + f'SELECT retired_at,is_primary FROM {table} WHERE subject=?', (owner,) + ) == (wire(NOW), 0) + for table, target in ( + ('share_grants', 'sg_one'), + ('share_grants_v2', 'sg_two'), + ('share_links', 'link_one'), + ): assert tx.one(f'SELECT revoked_at FROM {table} WHERE id=?', (target,)) == (wire(NOW),) - current = decode(Membership, loads(tx.one('SELECT body FROM memberships WHERE org=? AND subject=?', - ('g_test', owner))[0])) + current = decode( + Membership, + loads( + tx.one('SELECT body FROM memberships WHERE org=? AND subject=?', ('g_test', owner))[ + 0 + ] + ), + ) assert current.version == 2 and current.status == 'rejected' - assert tx.one('SELECT status FROM topic_memberships WHERE topic=? AND subject=?', - ('t_test', owner)) == ('removed',) - assert tx.one('SELECT status FROM topic_bans WHERE topic=? AND subject=?', - ('t_test', owner)) == ('lifted',) - assert tx.one('SELECT status,actor,reason FROM topic_bans WHERE topic=? AND subject=?', - ('t_ban', owner)) == ('active', 'u_root', 'recovery-replay') - assert tx.one('SELECT status,signing_nonce,signing_ciphertext,age_nonce,age_ciphertext ' - 'FROM custodial_vault WHERE subject=?', (owner,)) == ('destroyed', None, None, None, None) + assert tx.one( + 'SELECT status FROM topic_memberships WHERE topic=? AND subject=?', ('t_test', owner) + ) == ('removed',) + assert tx.one( + 'SELECT status FROM topic_bans WHERE topic=? AND subject=?', ('t_test', owner) + ) == ('lifted',) + assert tx.one( + 'SELECT status,actor,reason FROM topic_bans WHERE topic=? AND subject=?', + ('t_ban', owner), + ) == ('active', 'u_root', 'recovery-replay') + assert tx.one( + 'SELECT status,signing_nonce,signing_ciphertext,age_nonce,age_ciphertext ' + 'FROM custodial_vault WHERE subject=?', + (owner,), + ) == ('destroyed', None, None, None, None) assert tx.one('SELECT count(*) FROM jobs')[0] == 0 and active(tx) assert canonical(await tx.resource(resource.id)).decode() == canonical(resource).decode() assert (await replay(restored, signed(body), pin=pin))['changed'] == 0 @@ -338,6 +551,7 @@ def test_packaged_schema_and_example_match_the_replay_contract(): from msg.admin.recovery_replay import SUPPORTED_FACTS from msg.core.codec import loads, unb64 + root = files('msg').joinpath('data') schema = loads(root.joinpath('recovery-checkpoint.schema.json').read_bytes()) example = loads(root.joinpath('recovery-checkpoint.example.json').read_bytes()) @@ -360,18 +574,37 @@ def test_packaged_schema_and_example_match_the_replay_contract(): @pytest.mark.parametrize('lift', [False, True]) async def test_replayed_ban_removes_restored_admin_even_after_lift(restored, checkpoint, lift): from msg.core.models import Resource + body, pin, signed = checkpoint - resource = Resource(id='t_old_admin', type='topic', type_version=1, name='old-admin', - parent=None, owner='u_owner', group='g_public', mode=0o700, generation=1, - revision=None, state='active', created_at=NOW, created_by='u_owner', - modified_at=NOW, modified_by='u_owner') + resource = Resource( + id='t_old_admin', + type='topic', + type_version=1, + name='old-admin', + parent=None, + owner='u_owner', + group='g_public', + mode=0o700, + generation=1, + revision=None, + state='active', + created_at=NOW, + created_by='u_owner', + modified_at=NOW, + modified_by='u_owner', + ) async with restored.transaction(write=True) as tx: await tx.insert(resource) - tx.execute('INSERT INTO topic_memberships VALUES (?,?,?,?,?,NULL)', - (resource.id, 'u_owner', 'admin', 'active', wire(NOW)), write=True) + tx.execute( + 'INSERT INTO topic_memberships VALUES (?,?,?,?,?,NULL)', + (resource.id, 'u_owner', 'admin', 'active', wire(NOW)), + write=True, + ) kinds = ['topic_ban.apply'] + (['topic_ban.lift'] if lift else []) - facts = [{'sequence': i, 'kind': kind, 'subject': 'u_owner', - 'target': resource.id, 'at': wire(NOW)} for i, kind in enumerate(kinds, 1)] + facts = [ + {'sequence': i, 'kind': kind, 'subject': 'u_owner', 'target': resource.id, 'at': wire(NOW)} + for i, kind in enumerate(kinds, 1) + ] body = dict(body, entries=facts, sequence=len(facts)) pin = replace(pin, digest=digest(body), sequence=len(facts)) for _ in range(2): @@ -379,8 +612,12 @@ async def test_replayed_ban_removes_restored_admin_even_after_lift(restored, che assert receipt['scope'] == 'supported_authority_reconciliation_facts' assert 'supported_fact_inventory_incomplete' in receipt['promotion_blocked_reasons'] async with restored.transaction(write=False) as tx: - assert tx.one('SELECT role,status FROM topic_memberships WHERE topic=? AND subject=?', - (resource.id, 'u_owner')) == ('member', 'removed') - assert tx.one('SELECT status FROM topic_bans WHERE topic=? AND subject=?', - (resource.id, 'u_owner')) == ('lifted' if lift else 'active',) + assert tx.one( + 'SELECT role,status FROM topic_memberships WHERE topic=? AND subject=?', + (resource.id, 'u_owner'), + ) == ('member', 'removed') + assert tx.one( + 'SELECT status FROM topic_bans WHERE topic=? AND subject=?', + (resource.id, 'u_owner'), + ) == ('lifted' if lift else 'active',) assert active(tx) diff --git a/tests/test_task_a_root_files.py b/tests/test_task_a_root_files.py index e653dabb..23777a60 100644 --- a/tests/test_task_a_root_files.py +++ b/tests/test_task_a_root_files.py @@ -1,4 +1,5 @@ """Real disposable Root storage; these tests do not claim physical-console evidence.""" + from concurrent.futures import ThreadPoolExecutor import pytest @@ -38,6 +39,7 @@ async def test_history_validation_precedes_database_root_transition(installed, t history.chmod(0o644) elif unsafe == 'hardlink': import os + os.link(history, tmp_path / 'duplicate-key') else: history.unlink() @@ -58,16 +60,25 @@ async def test_history_validation_precedes_database_root_transition(installed, t async def test_concurrent_prepare_does_not_replace_an_approved_journal(installed): app, root = installed old_envelope(app, root) + def attempt(_): try: - return prepare(app, Ed25519Signer.generate(), PIN, - old_signer=root, operator='isolated-concurrent-test') + return prepare( + app, + Ed25519Signer.generate(), + PIN, + old_signer=root, + operator='isolated-concurrent-test', + ) except Failure as exc: return exc.code + with ThreadPoolExecutor(max_workers=2) as pool: results = list(pool.map(attempt, range(2))) journals = [result for result in results if isinstance(result, dict)] assert len(journals) == 1 assert next(result for result in results if isinstance(result, str)) in { - 'root_rotation_busy', 'root_rotation_pending'} + 'root_rotation_busy', + 'root_rotation_pending', + } assert canonical(loads(journal_path(app).read_bytes())) == canonical(journals[0]) diff --git a/tests/test_temporary_dual_keys.py b/tests/test_temporary_dual_keys.py index 0b5023bb..9cfe8c41 100644 --- a/tests/test_temporary_dual_keys.py +++ b/tests/test_temporary_dual_keys.py @@ -1,129 +1,167 @@ """A new temporary subject binds independent client-held keys at creation.""" + import hashlib import os import pytest +from test_service import call, temporary_v3_args from msg.core.codec import b64, canonical, unb64, wire from msg.security.age_keys import generate_age_key, public_from_recipient from msg.security.crypto import Ed25519Signer -from test_service import call, temporary_v3_args def temporary_arguments(signer, recipient, *, nonce, request_id): - subject='u_tmp_'+hashlib.sha256(unb64(nonce)).hexdigest()[:32] - proof=signer.sign(canonical({'subject_id':subject,'nonce':nonce, - 'public_key':b64(signer.public_key), - 'encryption_recipient':recipient,'request_id':request_id}), - purpose='temporary-key-possession-v1') - return {'nonce':nonce,'recovery_secret':b64(os.urandom(32)), - 'public_key':b64(signer.public_key), - 'encryption_recipient':recipient,'possession_proof':wire(proof)} + subject = 'u_tmp_' + hashlib.sha256(unb64(nonce)).hexdigest()[:32] + proof = signer.sign( + canonical({ + 'subject_id': subject, + 'nonce': nonce, + 'public_key': b64(signer.public_key), + 'encryption_recipient': recipient, + 'request_id': request_id, + }), + purpose='temporary-key-possession-v1', + ) + return { + 'nonce': nonce, + 'recovery_secret': b64(os.urandom(32)), + 'public_key': b64(signer.public_key), + 'encryption_recipient': recipient, + 'possession_proof': wire(proof), + } @pytest.mark.asyncio async def test_temporary_v3_creates_two_client_keys_and_upgrade_reuses_them(installed): - app,_=installed - signer=Ed25519Signer.generate() - _,recipient=generate_age_key() - assert signer.public_key!=public_from_recipient(recipient) - request_id='temporary-dual-key' - args=temporary_arguments(signer,recipient,nonce=b64(os.urandom(32)), - request_id=request_id) - created=await call(app,'identity.temporary',args,rid=request_id, - contract_version=3) - assert created.status=='ok',wire(created) - subject=created.data['subject_id'] - assert created.data['encryption_recipient']==recipient + app, _ = installed + signer = Ed25519Signer.generate() + _, recipient = generate_age_key() + assert signer.public_key != public_from_recipient(recipient) + request_id = 'temporary-dual-key' + args = temporary_arguments(signer, recipient, nonce=b64(os.urandom(32)), request_id=request_id) + created = await call(app, 'identity.temporary', args, rid=request_id, contract_version=3) + assert created.status == 'ok', wire(created) + subject = created.data['subject_id'] + assert created.data['encryption_recipient'] == recipient assert created.data['server_signable'] is False assert created.data['server_decryptable'] is False async with app.metadata.transaction(write=False) as tx: - signing=tx.one('SELECT key_id,public_key FROM identity_keys WHERE subject=?', - (subject,)) - encryption=tx.one('SELECT key_id,recipient,public_key FROM encryption_subkeys WHERE subject=?', - (subject,)) - assert signing[0]==created.data['identity_key_id'] - assert signing[1]==b64(signer.public_key) - assert encryption[0]==created.data['encryption_key_id'] - assert encryption[1]==recipient - assert encryption[2]==b64(public_from_recipient(recipient)) - assert tx.one('SELECT COUNT(*) FROM custodial_vault WHERE subject=?',(subject,))[0]==0 - replay=await call(app,'identity.temporary',args,rid=request_id,contract_version=3) - assert replay.error.code=='token_delivery_unavailable' - signed={'subject_id':subject,'handle':'temporary-promoted', - 'public_key':args['public_key'],'encryption_recipient':recipient} - upgraded=await call(app,'identity.upgrade',{ - 'handle':'temporary-promoted','public_key':args['public_key'], - 'encryption_recipient':recipient, - 'possession_proof':wire(signer.sign(canonical(signed),purpose='upgrade'))}, + signing = tx.one('SELECT key_id,public_key FROM identity_keys WHERE subject=?', (subject,)) + encryption = tx.one( + 'SELECT key_id,recipient,public_key FROM encryption_subkeys WHERE subject=?', (subject,) + ) + assert signing[0] == created.data['identity_key_id'] + assert signing[1] == b64(signer.public_key) + assert encryption[0] == created.data['encryption_key_id'] + assert encryption[1] == recipient + assert encryption[2] == b64(public_from_recipient(recipient)) + assert tx.one('SELECT COUNT(*) FROM custodial_vault WHERE subject=?', (subject,))[0] == 0 + replay = await call(app, 'identity.temporary', args, rid=request_id, contract_version=3) + assert replay.error.code == 'token_delivery_unavailable' + signed = { + 'subject_id': subject, + 'handle': 'temporary-promoted', + 'public_key': args['public_key'], + 'encryption_recipient': recipient, + } + upgraded = await call( + app, + 'identity.upgrade', + { + 'handle': 'temporary-promoted', + 'public_key': args['public_key'], + 'encryption_recipient': recipient, + 'possession_proof': wire(signer.sign(canonical(signed), purpose='upgrade')), + }, + subject=subject, + token=(created.data['credential_id'], unb64(created.data['token'])), + contract_version=2, + ) + assert upgraded.status == 'ok', wire(upgraded) + assert upgraded.data['subject_id'] == subject + assert upgraded.data['key_id'] == created.data['identity_key_id'] + assert upgraded.data['encryption_key_id'] == created.data['encryption_key_id'] + old_token = (created.data['credential_id'], unb64(created.data['token'])) + denied = await call( + app, + 'content.post_create', + {'parent': '/main', 'body': 'old token'}, + subject=subject, + token=old_token, + ) + assert denied.error.code == 'credential_revoked' + signed = await call( + app, + 'content.post_create', + {'parent': '/main', 'body': 'new signing key'}, + key=signer, subject=subject, - token=(created.data['credential_id'],unb64(created.data['token'])), - contract_version=2) - assert upgraded.status=='ok',wire(upgraded) - assert upgraded.data['subject_id']==subject - assert upgraded.data['key_id']==created.data['identity_key_id'] - assert upgraded.data['encryption_key_id']==created.data['encryption_key_id'] - old_token=(created.data['credential_id'],unb64(created.data['token'])) - denied=await call(app,'content.post_create',{'parent':'/main','body':'old token'}, - subject=subject,token=old_token) - assert denied.error.code=='credential_revoked' - signed=await call(app,'content.post_create', - {'parent':'/main','body':'new signing key'}, - key=signer,subject=subject) - assert signed.status=='ok',wire(signed) + ) + assert signed.status == 'ok', wire(signed) async with app.metadata.transaction(write=False) as tx: - assert tx.one('SELECT COUNT(*) FROM identity_keys WHERE subject=?',(subject,))[0]==1 - assert tx.one('SELECT COUNT(*) FROM encryption_subkeys WHERE subject=?',(subject,))[0]==1 + assert tx.one('SELECT COUNT(*) FROM identity_keys WHERE subject=?', (subject,))[0] == 1 + assert tx.one('SELECT COUNT(*) FROM encryption_subkeys WHERE subject=?', (subject,))[0] == 1 @pytest.mark.asyncio async def test_legacy_temporary_versions_and_false_key_proof_create_no_subject(installed): - app,_=installed + app, _ = installed async with app.metadata.transaction(write=False) as tx: - before=tx.one("SELECT COUNT(*) FROM identities WHERE kind='temporary'")[0] - nonce=b64(os.urandom(32)) - for version,args in ((1,{'nonce':nonce}), - (2,{'nonce':nonce,'recovery_secret':b64(os.urandom(32))})): - denied=await call(app,'identity.temporary',args,contract_version=version) - assert denied.error.code=='temporary_dual_keys_required' - signer=Ed25519Signer.generate() - _,recipient=generate_age_key() - args=temporary_arguments(signer,recipient,nonce=b64(os.urandom(32)), - request_id='bad-temporary-proof') - args['public_key']=b64(Ed25519Signer.generate().public_key) - denied=await call(app,'identity.temporary',args,rid='bad-temporary-proof', - contract_version=3) - assert denied.status=='error' + before = tx.one("SELECT COUNT(*) FROM identities WHERE kind='temporary'")[0] + nonce = b64(os.urandom(32)) + for version, args in ( + (1, {'nonce': nonce}), + (2, {'nonce': nonce, 'recovery_secret': b64(os.urandom(32))}), + ): + denied = await call(app, 'identity.temporary', args, contract_version=version) + assert denied.error.code == 'temporary_dual_keys_required' + signer = Ed25519Signer.generate() + _, recipient = generate_age_key() + args = temporary_arguments( + signer, recipient, nonce=b64(os.urandom(32)), request_id='bad-temporary-proof' + ) + args['public_key'] = b64(Ed25519Signer.generate().public_key) + denied = await call( + app, 'identity.temporary', args, rid='bad-temporary-proof', contract_version=3 + ) + assert denied.status == 'error' async with app.metadata.transaction(write=False) as tx: - assert tx.one("SELECT COUNT(*) FROM identities WHERE kind='temporary'")[0]==before + assert tx.one("SELECT COUNT(*) FROM identities WHERE kind='temporary'")[0] == before @pytest.mark.asyncio async def test_temporary_upgrade_cannot_silently_replace_creation_keys(installed): - app,_=installed - original=Ed25519Signer.generate() - _,recipient=generate_age_key() - args,request_id,_,_=temporary_v3_args( - signer=original,recipient=recipient) - created=await call(app,'identity.temporary',args,rid=request_id, - contract_version=3) - assert created.status=='ok',wire(created) - subject=created.data['subject_id'] - replacement=Ed25519Signer.generate() - _,new_recipient=generate_age_key() - signed={'subject_id':subject,'handle':'changed-temporary', - 'public_key':b64(replacement.public_key), - 'encryption_recipient':new_recipient} - denied=await call(app,'identity.upgrade',{ - 'handle':signed['handle'],'public_key':signed['public_key'], - 'encryption_recipient':new_recipient, - 'possession_proof':wire(replacement.sign(canonical(signed),purpose='upgrade'))}, + app, _ = installed + original = Ed25519Signer.generate() + _, recipient = generate_age_key() + args, request_id, _, _ = temporary_v3_args(signer=original, recipient=recipient) + created = await call(app, 'identity.temporary', args, rid=request_id, contract_version=3) + assert created.status == 'ok', wire(created) + subject = created.data['subject_id'] + replacement = Ed25519Signer.generate() + _, new_recipient = generate_age_key() + signed = { + 'subject_id': subject, + 'handle': 'changed-temporary', + 'public_key': b64(replacement.public_key), + 'encryption_recipient': new_recipient, + } + denied = await call( + app, + 'identity.upgrade', + { + 'handle': signed['handle'], + 'public_key': signed['public_key'], + 'encryption_recipient': new_recipient, + 'possession_proof': wire(replacement.sign(canonical(signed), purpose='upgrade')), + }, subject=subject, - token=(created.data['credential_id'],unb64(created.data['token'])), - contract_version=2) - assert denied.error.code=='temporary_identity_key_mismatch' + token=(created.data['credential_id'], unb64(created.data['token'])), + contract_version=2, + ) + assert denied.error.code == 'temporary_identity_key_mismatch' async with app.metadata.transaction(write=False) as tx: - assert (await tx.subject(subject)).kind=='temporary' - assert tx.one('SELECT COUNT(*) FROM identity_keys WHERE subject=?',(subject,))[0]==1 - assert tx.one('SELECT COUNT(*) FROM encryption_subkeys WHERE subject=?', - (subject,))[0]==1 + assert (await tx.subject(subject)).kind == 'temporary' + assert tx.one('SELECT COUNT(*) FROM identity_keys WHERE subject=?', (subject,))[0] == 1 + assert tx.one('SELECT COUNT(*) FROM encryption_subkeys WHERE subject=?', (subject,))[0] == 1 diff --git a/tests/test_temporary_legacy_migration.py b/tests/test_temporary_legacy_migration.py index ecd9d6b6..12eb4c8a 100644 --- a/tests/test_temporary_legacy_migration.py +++ b/tests/test_temporary_legacy_migration.py @@ -1,4 +1,5 @@ """A stored token-only temporary subject upgrades without losing ownership.""" + import hashlib import os from datetime import timedelta @@ -6,6 +7,7 @@ from uuid import uuid4 import pytest +from test_service import NOW, call from msg.core.codec import b64, canonical, wire from msg.core.errors import Failure @@ -13,81 +15,115 @@ from msg.plugins import identity from msg.security.age_keys import generate_age_key from msg.security.crypto import Ed25519Signer -from test_service import NOW, call @pytest.mark.asyncio -async def test_legacy_temporary_upgrade_is_atomic_and_preserves_subject(installed, - monkeypatch): - app,_=installed - subject='u_tmp_legacy_'+uuid4().hex[:20] - token_bytes=os.urandom(32) - token_id='t_'+subject[2:] +async def test_legacy_temporary_upgrade_is_atomic_and_preserves_subject(installed, monkeypatch): + app, _ = installed + subject = 'u_tmp_legacy_' + uuid4().hex[:20] + token_bytes = os.urandom(32) + token_id = 't_' + subject[2:] async with app.metadata.transaction(write=True) as tx: - await identity.make_user(app,tx,SimpleNamespace(now=NOW),subject, - 'tmp-'+subject[-20:],'temporary') - await tx.save_credential(Credential( - id=token_id,subject_id=subject,kind='token', - verifier=hashlib.sha256(token_bytes).digest(), - ceiling=app.temporary_ceiling(),not_before=NOW, - expires_at=NOW+timedelta(hours=1),revoked_at=None),0) - token=(token_id,token_bytes) - post=await call(app,'content.post_create', - {'parent':'/tmp','body':'legacy temporary work'}, - subject=subject,token=token) - assert post.status=='ok',wire(post) - owner_ref=post.resources[0].id - signer=Ed25519Signer.generate() - _,recipient=generate_age_key() - signed={'subject_id':subject,'public_key':b64(signer.public_key), - 'handle':'legacy-promoted','encryption_recipient':recipient} - args={'handle':'legacy-promoted','public_key':signed['public_key'], - 'encryption_recipient':recipient, - 'possession_proof':wire(signer.sign(canonical(signed),purpose='upgrade'))} - original=identity.issue_online + await identity.make_user( + app, tx, SimpleNamespace(now=NOW), subject, 'tmp-' + subject[-20:], 'temporary' + ) + await tx.save_credential( + Credential( + id=token_id, + subject_id=subject, + kind='token', + verifier=hashlib.sha256(token_bytes).digest(), + ceiling=app.temporary_ceiling(), + not_before=NOW, + expires_at=NOW + timedelta(hours=1), + revoked_at=None, + ), + 0, + ) + token = (token_id, token_bytes) + post = await call( + app, + 'content.post_create', + {'parent': '/tmp', 'body': 'legacy temporary work'}, + subject=subject, + token=token, + ) + assert post.status == 'ok', wire(post) + owner_ref = post.resources[0].id + signer = Ed25519Signer.generate() + _, recipient = generate_age_key() + signed = { + 'subject_id': subject, + 'public_key': b64(signer.public_key), + 'handle': 'legacy-promoted', + 'encryption_recipient': recipient, + } + args = { + 'handle': 'legacy-promoted', + 'public_key': signed['public_key'], + 'encryption_recipient': recipient, + 'possession_proof': wire(signer.sign(canonical(signed), purpose='upgrade')), + } + original = identity.issue_online - async def fail_after_identity_writes(*_args,**_kwargs): + async def fail_after_identity_writes(*_args, **_kwargs): raise Failure('forced_issue_failure') - monkeypatch.setattr(identity,'issue_online',fail_after_identity_writes) - failed=await call(app,'identity.upgrade',args,subject=subject,token=token, - contract_version=2) - assert failed.error.code=='forced_issue_failure' + monkeypatch.setattr(identity, 'issue_online', fail_after_identity_writes) + failed = await call( + app, 'identity.upgrade', args, subject=subject, token=token, contract_version=2 + ) + assert failed.error.code == 'forced_issue_failure' async with app.metadata.transaction(write=False) as tx: - assert (await tx.subject(subject)).kind=='temporary' - assert (await tx.resource(owner_ref)).owner==subject - assert tx.one('SELECT COUNT(*) FROM identity_keys WHERE subject=?',(subject,))[0]==0 - assert tx.one('SELECT COUNT(*) FROM encryption_subkeys WHERE subject=?', - (subject,))[0]==0 + assert (await tx.subject(subject)).kind == 'temporary' + assert (await tx.resource(owner_ref)).owner == subject + assert tx.one('SELECT COUNT(*) FROM identity_keys WHERE subject=?', (subject,))[0] == 0 + assert tx.one('SELECT COUNT(*) FROM encryption_subkeys WHERE subject=?', (subject,))[0] == 0 assert (await tx.credential(token_id)).revoked_at is None - monkeypatch.setattr(identity,'issue_online',original) - upgraded=await call(app,'identity.upgrade',args,subject=subject,token=token, - contract_version=2) - assert upgraded.status=='ok',wire(upgraded) - assert upgraded.data['subject_id']==subject - assert upgraded.data['encryption_recipient']==recipient + monkeypatch.setattr(identity, 'issue_online', original) + upgraded = await call( + app, 'identity.upgrade', args, subject=subject, token=token, contract_version=2 + ) + assert upgraded.status == 'ok', wire(upgraded) + assert upgraded.data['subject_id'] == subject + assert upgraded.data['encryption_recipient'] == recipient async with app.metadata.transaction(write=False) as tx: - assert (await tx.subject(subject)).kind=='registered' - assert (await tx.resource(owner_ref)).owner==subject - assert tx.one('SELECT COUNT(*) FROM identity_keys WHERE subject=?',(subject,))[0]==1 - key=tx.one('SELECT key_id,public_key FROM identity_keys WHERE subject=? AND is_primary=1', - (subject,)) - assert key==(upgraded.data['key_id'],b64(signer.public_key)) - assert tx.one('SELECT COUNT(*) FROM encryption_subkeys WHERE subject=?', - (subject,))[0]==1 - encryption=tx.one('SELECT key_id,recipient,public_key FROM encryption_subkeys WHERE subject=? AND is_primary=1', - (subject,)) - assert encryption==(upgraded.data['encryption_key_id'],recipient, - b64(identity.public_from_recipient(recipient))) + assert (await tx.subject(subject)).kind == 'registered' + assert (await tx.resource(owner_ref)).owner == subject + assert tx.one('SELECT COUNT(*) FROM identity_keys WHERE subject=?', (subject,))[0] == 1 + key = tx.one( + 'SELECT key_id,public_key FROM identity_keys WHERE subject=? AND is_primary=1', + (subject,), + ) + assert key == (upgraded.data['key_id'], b64(signer.public_key)) + assert tx.one('SELECT COUNT(*) FROM encryption_subkeys WHERE subject=?', (subject,))[0] == 1 + encryption = tx.one( + 'SELECT key_id,recipient,public_key FROM encryption_subkeys WHERE subject=? AND is_primary=1', + (subject,), + ) + assert encryption == ( + upgraded.data['encryption_key_id'], + recipient, + b64(identity.public_from_recipient(recipient)), + ) assert (await tx.credential(token_id)).revoked_at is not None - denied=await call(app,'content.post_create', - {'parent':'/tmp','body':'legacy token after upgrade'}, - subject=subject,token=token) - assert denied.error.code=='credential_revoked' - signed_post=await call(app,'content.post_create', - {'parent':'/main','body':'migrated signing key'}, - key=signer,subject=subject) - assert signed_post.status=='ok',wire(signed_post) - repeated=await call(app,'identity.upgrade',args,subject=subject,token=token, - contract_version=2) - assert repeated.error.code=='credential_revoked' + denied = await call( + app, + 'content.post_create', + {'parent': '/tmp', 'body': 'legacy token after upgrade'}, + subject=subject, + token=token, + ) + assert denied.error.code == 'credential_revoked' + signed_post = await call( + app, + 'content.post_create', + {'parent': '/main', 'body': 'migrated signing key'}, + key=signer, + subject=subject, + ) + assert signed_post.status == 'ok', wire(signed_post) + repeated = await call( + app, 'identity.upgrade', args, subject=subject, token=token, contract_version=2 + ) + assert repeated.error.code == 'credential_revoked' diff --git a/tests/test_temporary_signed_business.py b/tests/test_temporary_signed_business.py index 6c64c658..04dec054 100644 --- a/tests/test_temporary_signed_business.py +++ b/tests/test_temporary_signed_business.py @@ -1,22 +1,26 @@ """Signed temporary subjects and their bootstrap token have distinct limits.""" + import pytest +from test_service import call, temporary_v3_args from msg.core.codec import unb64, wire -from test_service import call, temporary_v3_args @pytest.mark.asyncio async def test_temporary_signing_key_can_create_low_risk_content(installed): app, _ = installed args, request_id, signer, _ = temporary_v3_args() - created = await call(app, 'identity.temporary', args, rid=request_id, - contract_version=3) + created = await call(app, 'identity.temporary', args, rid=request_id, contract_version=3) assert created.status == 'ok', wire(created) subject = created.data['subject_id'] - post = await call(app, 'content.post_create', - {'parent': '/tmp', 'body': 'signed temporary post'}, - key=signer, subject=subject) + post = await call( + app, + 'content.post_create', + {'parent': '/tmp', 'body': 'signed temporary post'}, + key=signer, + subject=subject, + ) assert post.status == 'ok', f'{post.error.code}: {wire(post)}' @@ -24,20 +28,27 @@ async def test_temporary_signing_key_can_create_low_risk_content(installed): async def test_temporary_token_allows_low_risk_write_but_not_permission_changes(installed): app, _ = installed args, request_id, _, _ = temporary_v3_args() - created = await call(app, 'identity.temporary', args, rid=request_id, - contract_version=3) + created = await call(app, 'identity.temporary', args, rid=request_id, contract_version=3) assert created.status == 'ok', wire(created) subject = created.data['subject_id'] token = (created.data['credential_id'], unb64(created.data['token'])) - post = await call(app, 'content.post_create', - {'parent': '/tmp', 'body': 'token temporary post'}, - subject=subject, token=token) + post = await call( + app, + 'content.post_create', + {'parent': '/tmp', 'body': 'token temporary post'}, + subject=subject, + token=token, + ) assert post.status == 'ok', wire(post) - chmod = await call(app, 'content.chmod', - {'id': post.resources[0].id, 'mode': '0777'}, - subject=subject, token=token, - expected=((post.resources[0].id, post.data['generation']),)) + chmod = await call( + app, + 'content.chmod', + {'id': post.resources[0].id, 'mode': '0777'}, + subject=subject, + token=token, + expected=((post.resources[0].id, post.data['generation']),), + ) assert chmod.status == 'error' assert chmod.error.code == 'credential_ceiling', wire(chmod) diff --git a/tests/test_text_patch.py b/tests/test_text_patch.py index ae9b3b82..5a2c3e20 100644 --- a/tests/test_text_patch.py +++ b/tests/test_text_patch.py @@ -1,101 +1,208 @@ """Exact text patches keep a single authorized Revision boundary.""" + import pytest +from test_service import call, register from msg.core.codec import b64, wire from msg.core.models import ResourceRef -from test_service import call, register -async def state(app,rid): +async def state(app, rid): async with app.metadata.transaction(write=False) as tx: - resource=await tx.resource(rid) - return (resource.generation,resource.revision, - tx.one('SELECT COUNT(*) FROM revisions')[0], - tx.one('SELECT COUNT(*) FROM events')[0]) + resource = await tx.resource(rid) + return ( + resource.generation, + resource.revision, + tx.one('SELECT COUNT(*) FROM revisions')[0], + tx.one('SELECT COUNT(*) FROM events')[0], + ) -async def content(app,rid): +async def content(app, rid): async with app.metadata.transaction(write=False) as tx: - revision=await tx.revision(ResourceRef(id=rid)) - return revision,(await app.contents.read_bytes(revision.content)).decode('utf-8') + revision = await tx.revision(ResourceRef(id=rid)) + return revision, (await app.contents.read_bytes(revision.content)).decode('utf-8') @pytest.mark.asyncio async def test_context_patch_post_and_file_preserve_revision_history(installed): - app,_=installed - key,owner,_=await register(app,'patch-owner') - post=await call(app,'content.post_create',{'parent':'/main','body':'甲 fish\n乙 fish\n'}, - key=key,subject=owner) - assert post.status=='ok',wire(post) - rid=post.resources[0].id - old=post.resources[0].revision - changed=await call(app,'content.text_patch',{'id':rid,'base_revision':old, - 'exact':'fish','replacement':'鱼','before':'乙 ','after':'\n'}, - key=key,subject=owner,expected=((rid,post.data['generation']),)) - assert changed.status=='ok',wire(changed) - revision,body=await content(app,rid) - assert body=='甲 fish\n乙 鱼\n' - assert revision.parents==(old,) - assert revision.content.media_type=='text/markdown' - assert revision.actor==owner and revision.subject==owner and revision.manifest_digest + app, _ = installed + key, owner, _ = await register(app, 'patch-owner') + post = await call( + app, + 'content.post_create', + {'parent': '/main', 'body': '甲 fish\n乙 fish\n'}, + key=key, + subject=owner, + ) + assert post.status == 'ok', wire(post) + rid = post.resources[0].id + old = post.resources[0].revision + changed = await call( + app, + 'content.text_patch', + { + 'id': rid, + 'base_revision': old, + 'exact': 'fish', + 'replacement': '鱼', + 'before': '乙 ', + 'after': '\n', + }, + key=key, + subject=owner, + expected=((rid, post.data['generation']),), + ) + assert changed.status == 'ok', wire(changed) + revision, body = await content(app, rid) + assert body == '甲 fish\n乙 鱼\n' + assert revision.parents == (old,) + assert revision.content.media_type == 'text/markdown' + assert revision.actor == owner and revision.subject == owner and revision.manifest_digest - file=await call(app,'content.file_put',{'parent':'/@patch-owner/files','name':'sample.txt', - 'data':b64('汉字 A'.encode()),'media_type':'text/plain'},key=key,subject=owner) - assert file.status=='ok',wire(file) - fid=file.resources[0].id - result=await call(app,'content.text_patch',{'id':fid,'base_revision':file.resources[0].revision, - 'exact':'汉字','replacement':'文字'},key=key,subject=owner, - expected=((fid,file.data['generation']),)) - assert result.status=='ok',wire(result) - revision,body=await content(app,fid) - assert body=='文字 A' and revision.content.media_type=='text/plain' + file = await call( + app, + 'content.file_put', + { + 'parent': '/@patch-owner/files', + 'name': 'sample.txt', + 'data': b64('汉字 A'.encode()), + 'media_type': 'text/plain', + }, + key=key, + subject=owner, + ) + assert file.status == 'ok', wire(file) + fid = file.resources[0].id + result = await call( + app, + 'content.text_patch', + { + 'id': fid, + 'base_revision': file.resources[0].revision, + 'exact': '汉字', + 'replacement': '文字', + }, + key=key, + subject=owner, + expected=((fid, file.data['generation']),), + ) + assert result.status == 'ok', wire(result) + revision, body = await content(app, fid) + assert body == '文字 A' and revision.content.media_type == 'text/plain' @pytest.mark.asyncio -async def test_patch_rejects_ambiguous_stale_missing_and_unauthorized_without_publication(installed): - app,_=installed - key,owner,_=await register(app,'patch-conflicts') - other,outsider,_=await register(app,'patch-outsider') - post=await call(app,'content.post_create',{'parent':'/main','body':'one one'}, - key=key,subject=owner) - rid=post.resources[0].id - base=post.resources[0].revision - generation=post.data['generation'] - original=await state(app,rid) - cases=[ - (key,owner,{'id':rid,'base_revision':base,'exact':'one','replacement':'two'},generation,'patch_ambiguous'), - (key,owner,{'id':rid,'base_revision':base,'exact':'absent','replacement':'two'},generation,'patch_no_match'), - (key,owner,{'id':rid,'base_revision':'v_stale','exact':'one','replacement':'two'},generation,'revision_conflict'), - (key,owner,{'id':rid,'base_revision':base,'exact':'one','replacement':'two'},generation+1,'generation_conflict'), - (other,outsider,{'id':rid,'base_revision':base,'exact':'one','replacement':'two'},generation,'permission_denied'), +async def test_patch_rejects_ambiguous_stale_missing_and_unauthorized_without_publication( + installed, +): + app, _ = installed + key, owner, _ = await register(app, 'patch-conflicts') + other, outsider, _ = await register(app, 'patch-outsider') + post = await call( + app, 'content.post_create', {'parent': '/main', 'body': 'one one'}, key=key, subject=owner + ) + rid = post.resources[0].id + base = post.resources[0].revision + generation = post.data['generation'] + original = await state(app, rid) + cases = [ + ( + key, + owner, + {'id': rid, 'base_revision': base, 'exact': 'one', 'replacement': 'two'}, + generation, + 'patch_ambiguous', + ), + ( + key, + owner, + {'id': rid, 'base_revision': base, 'exact': 'absent', 'replacement': 'two'}, + generation, + 'patch_no_match', + ), + ( + key, + owner, + {'id': rid, 'base_revision': 'v_stale', 'exact': 'one', 'replacement': 'two'}, + generation, + 'revision_conflict', + ), + ( + key, + owner, + {'id': rid, 'base_revision': base, 'exact': 'one', 'replacement': 'two'}, + generation + 1, + 'generation_conflict', + ), + ( + other, + outsider, + {'id': rid, 'base_revision': base, 'exact': 'one', 'replacement': 'two'}, + generation, + 'permission_denied', + ), ] - for signer,subject,args,expected_generation,code in cases: - failed=await call(app,'content.text_patch',args,key=signer,subject=subject, - expected=((rid,expected_generation),)) - assert failed.status=='error' and failed.error.code==code,wire(failed) - assert await state(app,rid)==original + for signer, subject, args, expected_generation, code in cases: + failed = await call( + app, + 'content.text_patch', + args, + key=signer, + subject=subject, + expected=((rid, expected_generation),), + ) + assert failed.status == 'error' and failed.error.code == code, wire(failed) + assert await state(app, rid) == original @pytest.mark.asyncio async def test_patch_rejects_binary_and_bounded_complexity(installed): - app,_=installed - key,owner,_=await register(app,'patch-bounds') - file=await call(app,'content.file_put',{'parent':'/@patch-bounds/files','name':'binary.bin', - 'data':b64(b'\xff\x00'),'media_type':'application/octet-stream'},key=key,subject=owner) - fid=file.resources[0].id - before=await state(app,fid) - failed=await call(app,'content.text_patch',{'id':fid,'base_revision':file.resources[0].revision, - 'exact':'a','replacement':'b'},key=key,subject=owner, - expected=((fid,file.data['generation']),)) - assert failed.status=='error' and failed.error.code=='text_patch_required',wire(failed) - assert await state(app,fid)==before + app, _ = installed + key, owner, _ = await register(app, 'patch-bounds') + file = await call( + app, + 'content.file_put', + { + 'parent': '/@patch-bounds/files', + 'name': 'binary.bin', + 'data': b64(b'\xff\x00'), + 'media_type': 'application/octet-stream', + }, + key=key, + subject=owner, + ) + fid = file.resources[0].id + before = await state(app, fid) + failed = await call( + app, + 'content.text_patch', + {'id': fid, 'base_revision': file.resources[0].revision, 'exact': 'a', 'replacement': 'b'}, + key=key, + subject=owner, + expected=((fid, file.data['generation']),), + ) + assert failed.status == 'error' and failed.error.code == 'text_patch_required', wire(failed) + assert await state(app, fid) == before - post=await call(app,'content.post_create',{'parent':'/main','body':'a'*5000}, - key=key,subject=owner) - rid=post.resources[0].id - before=await state(app,rid) - failed=await call(app,'content.text_patch',{'id':rid,'base_revision':post.resources[0].revision, - 'exact':'a','replacement':'b','before':'z'},key=key,subject=owner, - expected=((rid,post.data['generation']),)) - assert failed.status=='error' and failed.error.code=='patch_too_complex',wire(failed) - assert await state(app,rid)==before + post = await call( + app, 'content.post_create', {'parent': '/main', 'body': 'a' * 5000}, key=key, subject=owner + ) + rid = post.resources[0].id + before = await state(app, rid) + failed = await call( + app, + 'content.text_patch', + { + 'id': rid, + 'base_revision': post.resources[0].revision, + 'exact': 'a', + 'replacement': 'b', + 'before': 'z', + }, + key=key, + subject=owner, + expected=((rid, post.data['generation']),), + ) + assert failed.status == 'error' and failed.error.code == 'patch_too_complex', wire(failed) + assert await state(app, rid) == before diff --git a/tests/test_text_patch_line_boundaries.py b/tests/test_text_patch_line_boundaries.py index 9998be02..dd1d1ee6 100644 --- a/tests/test_text_patch_line_boundaries.py +++ b/tests/test_text_patch_line_boundaries.py @@ -1,4 +1,5 @@ """Git diff lines are LF-delimited; Unicode separators are signed content bytes.""" + import subprocess import pytest @@ -14,15 +15,20 @@ def test_unified_patch_preserves_non_lf_bytes_and_agrees_with_git(tmp_path, separator): before = f'left{separator}old\nunchanged\n' after = f'left{separator}new\nunchanged\n' - diff = ('--- a/document.txt\n+++ b/document.txt\n@@ -1,2 +1,2 @@\n' - f'-left{separator}old\n+left{separator}new\n unchanged\n') + diff = ( + '--- a/document.txt\n+++ b/document.txt\n@@ -1,2 +1,2 @@\n' + f'-left{separator}old\n+left{separator}new\n unchanged\n' + ) target = tmp_path / 'document.txt' target.write_bytes(before.encode()) - subprocess.run(['git', 'apply', '-'], cwd=tmp_path, input=diff.encode(), - check=True, capture_output=True) + subprocess.run( + ['git', 'apply', '-'], cwd=tmp_path, input=diff.encode(), check=True, capture_output=True + ) assert target.read_bytes() == after.encode() assert apply_patch(before, {'kind': 'unified', 'diff': diff}).encode() == target.read_bytes() - rebased = apply_patch('prefix\n' + before, {'kind': 'unified', 'diff': diff}, base_source=before) + rebased = apply_patch( + 'prefix\n' + before, {'kind': 'unified', 'diff': diff}, base_source=before + ) assert rebased.encode() == ('prefix\n' + after).encode() @@ -31,24 +37,45 @@ def test_inline_unicode_separator_cannot_forge_a_markdown_heading(separator): source = f'inline{separator}# Not a heading\nparagraph\n' fake_section = '# Not a heading\nparagraph\n' with pytest.raises(Failure) as caught: - apply_patch(source, {'kind': 'heading', 'heading': 'Not a heading', - 'digest': digest(fake_section.encode()), 'replacement': 'tampered\n'}) + apply_patch( + source, + { + 'kind': 'heading', + 'heading': 'Not a heading', + 'digest': digest(fake_section.encode()), + 'replacement': 'tampered\n', + }, + ) assert caught.value.code == 'patch_no_match' - assert apply_patch(source, {'kind': 'block', 'digest': digest(source.encode()), - 'replacement': 'replaced\n'}) == 'replaced\n' + assert ( + apply_patch( + source, + {'kind': 'block', 'digest': digest(source.encode()), 'replacement': 'replaced\n'}, + ) + == 'replaced\n' + ) @pytest.mark.parametrize('ending', ['\n', '\r\n', '\r']) def test_markdown_keeps_real_line_endings_and_signed_section_digest(ending): section = '# Title' + ending + 'text' + ending source = section + '# Other' + ending + 'untouched' + ending - patch = {'kind': 'heading', 'heading': 'Title', 'digest': digest(section.encode()), - 'replacement': '# New' + ending} - assert apply_patch(source, patch) == '# New' + ending + '# Other' + ending + 'untouched' + ending + patch = { + 'kind': 'heading', + 'heading': 'Title', + 'digest': digest(section.encode()), + 'replacement': '# New' + ending, + } + assert ( + apply_patch(source, patch) == '# New' + ending + '# Other' + ending + 'untouched' + ending + ) def test_unified_no_newline_markers_preserve_inline_unicode(): before, after = 'same\u2028old', 'same\u2028new' - patch = {'kind': 'unified', 'diff': '@@ -1 +1 @@\n-same\u2028old\n' - '\\ No newline at end of file\n+same\u2028new\n\\ No newline at end of file\n'} + patch = { + 'kind': 'unified', + 'diff': '@@ -1 +1 @@\n-same\u2028old\n' + '\\ No newline at end of file\n+same\u2028new\n\\ No newline at end of file\n', + } assert apply_patch(before, patch) == after diff --git a/tests/test_text_patch_rebase_batch.py b/tests/test_text_patch_rebase_batch.py index 49d6f78a..7f47fe5e 100644 --- a/tests/test_text_patch_rebase_batch.py +++ b/tests/test_text_patch_rebase_batch.py @@ -1,104 +1,202 @@ """Context rebase and multi-file patches use one PostgreSQL commit boundary.""" + import asyncio import pytest +from test_service import call, register from msg.core.codec import wire from msg.core.models import ResourceRef -from test_service import call, register async def snapshot(app, ids): async with app.metadata.transaction(write=False) as tx: - resources=[await tx.resource(rid) for rid in ids] - return (tuple((resource.generation,resource.revision) for resource in resources), - tx.one('SELECT COUNT(*) FROM revisions')[0], - tx.one('SELECT COUNT(*) FROM events')[0]) + resources = [await tx.resource(rid) for rid in ids] + return ( + tuple((resource.generation, resource.revision) for resource in resources), + tx.one('SELECT COUNT(*) FROM revisions')[0], + tx.one('SELECT COUNT(*) FROM events')[0], + ) async def body(app, rid): async with app.metadata.transaction(write=False) as tx: - revision=await tx.revision(ResourceRef(id=rid)) + revision = await tx.revision(ResourceRef(id=rid)) return (await app.contents.read_bytes(revision.content)).decode('utf-8') @pytest.mark.asyncio async def test_rebase_unique_context_and_reject_changed_or_ambiguous(installed): - app,_=installed - key,subject,_=await register(app,'rebase-owner') - created=await call(app,'content.post_create',{'parent':'/main','body':'alpha one\nbeta two\n'},key=key,subject=subject) - rid=created.resources[0].id - base=created.resources[0].revision - first=await call(app,'content.text_patch',{'id':rid,'base_revision':base, - 'exact':'alpha','replacement':'ALPHA'},key=key,subject=subject, - expected=((rid,created.data['generation']),)) - assert first.status=='ok',wire(first) - stale={'id':rid,'base_revision':base,'base_generation':created.data['generation'], - 'rebase':True,'exact':'beta','replacement':'BETA','after':' two'} - rebased=await call(app,'content.text_patch',stale,key=key,subject=subject, - expected=((rid,first.data['generation']),),contract_version=2) - assert rebased.status=='ok',wire(rebased) - assert await body(app,rid)=='ALPHA one\nBETA two\n' - before=await snapshot(app,(rid,)) - changed=await call(app,'content.text_patch',{**stale,'exact':'alpha'},key=key,subject=subject, - expected=((rid,rebased.data['generation']),),contract_version=2) - assert changed.error.code=='patch_no_match',wire(changed) - bad_generation=await call(app,'content.text_patch',{**stale,'base_generation':rebased.data['generation']}, - key=key,subject=subject,expected=((rid,rebased.data['generation']),),contract_version=2) - assert bad_generation.error.code=='base_generation_conflict',wire(bad_generation) - assert await snapshot(app,(rid,))==before + app, _ = installed + key, subject, _ = await register(app, 'rebase-owner') + created = await call( + app, + 'content.post_create', + {'parent': '/main', 'body': 'alpha one\nbeta two\n'}, + key=key, + subject=subject, + ) + rid = created.resources[0].id + base = created.resources[0].revision + first = await call( + app, + 'content.text_patch', + {'id': rid, 'base_revision': base, 'exact': 'alpha', 'replacement': 'ALPHA'}, + key=key, + subject=subject, + expected=((rid, created.data['generation']),), + ) + assert first.status == 'ok', wire(first) + stale = { + 'id': rid, + 'base_revision': base, + 'base_generation': created.data['generation'], + 'rebase': True, + 'exact': 'beta', + 'replacement': 'BETA', + 'after': ' two', + } + rebased = await call( + app, + 'content.text_patch', + stale, + key=key, + subject=subject, + expected=((rid, first.data['generation']),), + contract_version=2, + ) + assert rebased.status == 'ok', wire(rebased) + assert await body(app, rid) == 'ALPHA one\nBETA two\n' + before = await snapshot(app, (rid,)) + changed = await call( + app, + 'content.text_patch', + {**stale, 'exact': 'alpha'}, + key=key, + subject=subject, + expected=((rid, rebased.data['generation']),), + contract_version=2, + ) + assert changed.error.code == 'patch_no_match', wire(changed) + bad_generation = await call( + app, + 'content.text_patch', + {**stale, 'base_generation': rebased.data['generation']}, + key=key, + subject=subject, + expected=((rid, rebased.data['generation']),), + contract_version=2, + ) + assert bad_generation.error.code == 'base_generation_conflict', wire(bad_generation) + assert await snapshot(app, (rid,)) == before @pytest.mark.asyncio async def test_batch_preflight_failure_and_concurrent_writers_publish_nothing_partial(installed): - app,_=installed - key,subject,_=await register(app,'batch-patcher') - a=await call(app,'content.post_create',{'parent':'/main','body':'first A'},key=key,subject=subject) - b=await call(app,'content.post_create',{'parent':'/main','body':'second B'},key=key,subject=subject) - ids=(a.resources[0].id,b.resources[0].id) - expected=((ids[0],a.data['generation']),(ids[1],b.data['generation'])) - patches=[{'id':ids[0],'base_revision':a.resources[0].revision,'base_generation':a.data['generation'], - 'exact':'A','replacement':'AA'}, - {'id':ids[1],'base_revision':b.resources[0].revision,'base_generation':b.data['generation'], - 'exact':'absent','replacement':'BB'}] - before=await snapshot(app,ids) - rejected=await call(app,'content.text_patch_batch',{'patches':patches},key=key,subject=subject,expected=expected) - assert rejected.error.code=='patch_no_match',wire(rejected) - assert await snapshot(app,ids)==before - patches[1]['exact']='B' - first,second=await asyncio.gather(*( - call(app,'content.text_patch_batch',{'patches':patches},key=key,subject=subject, - expected=expected) for _ in range(2))) - assert sorted((first.status,second.status))==['error','ok'],(wire(first),wire(second)) - assert (first.error or second.error).code=='generation_conflict' - assert await body(app,ids[0])=='first AA' - assert await body(app,ids[1])=='second BB' + app, _ = installed + key, subject, _ = await register(app, 'batch-patcher') + a = await call( + app, 'content.post_create', {'parent': '/main', 'body': 'first A'}, key=key, subject=subject + ) + b = await call( + app, + 'content.post_create', + {'parent': '/main', 'body': 'second B'}, + key=key, + subject=subject, + ) + ids = (a.resources[0].id, b.resources[0].id) + expected = ((ids[0], a.data['generation']), (ids[1], b.data['generation'])) + patches = [ + { + 'id': ids[0], + 'base_revision': a.resources[0].revision, + 'base_generation': a.data['generation'], + 'exact': 'A', + 'replacement': 'AA', + }, + { + 'id': ids[1], + 'base_revision': b.resources[0].revision, + 'base_generation': b.data['generation'], + 'exact': 'absent', + 'replacement': 'BB', + }, + ] + before = await snapshot(app, ids) + rejected = await call( + app, + 'content.text_patch_batch', + {'patches': patches}, + key=key, + subject=subject, + expected=expected, + ) + assert rejected.error.code == 'patch_no_match', wire(rejected) + assert await snapshot(app, ids) == before + patches[1]['exact'] = 'B' + first, second = await asyncio.gather( + *( + call( + app, + 'content.text_patch_batch', + {'patches': patches}, + key=key, + subject=subject, + expected=expected, + ) + for _ in range(2) + ) + ) + assert sorted((first.status, second.status)) == ['error', 'ok'], (wire(first), wire(second)) + assert (first.error or second.error).code == 'generation_conflict' + assert await body(app, ids[0]) == 'first AA' + assert await body(app, ids[1]) == 'second BB' async with app.metadata.transaction(write=False) as tx: - assert tx.one('SELECT COUNT(*) FROM revisions WHERE resource_id IN (?,?)',ids)[0]==4 + assert tx.one('SELECT COUNT(*) FROM revisions WHERE resource_id IN (?,?)', ids)[0] == 4 @pytest.mark.asyncio -async def test_batch_second_revision_failure_rolls_back_pg(monkeypatch,installed): - app,_=installed - key,subject,_=await register(app,'batch-rollback') - created=[await call(app,'content.post_create',{'parent':'/main','body':body}, - key=key,subject=subject) for body in ('one X','two Y')] - ids=tuple(item.resources[0].id for item in created) - before=await snapshot(app,ids) - patches=[{'id':item.resources[0].id,'base_revision':item.resources[0].revision, - 'base_generation':item.data['generation'],'exact':needle,'replacement':'changed'} - for item,needle in zip(created,('X','Y'))] - original=app.contents.commit_revision - calls=0 - async def fail_second(topic,revision): +async def test_batch_second_revision_failure_rolls_back_pg(monkeypatch, installed): + app, _ = installed + key, subject, _ = await register(app, 'batch-rollback') + created = [ + await call( + app, 'content.post_create', {'parent': '/main', 'body': body}, key=key, subject=subject + ) + for body in ('one X', 'two Y') + ] + ids = tuple(item.resources[0].id for item in created) + before = await snapshot(app, ids) + patches = [ + { + 'id': item.resources[0].id, + 'base_revision': item.resources[0].revision, + 'base_generation': item.data['generation'], + 'exact': needle, + 'replacement': 'changed', + } + for item, needle in zip(created, ('X', 'Y'), strict=True) + ] + original = app.contents.commit_revision + calls = 0 + + async def fail_second(topic, revision): nonlocal calls - calls+=1 - if calls==2: + calls += 1 + if calls == 2: raise RuntimeError('injected second revision failure') - return await original(topic,revision) - monkeypatch.setattr(app.contents,'commit_revision',fail_second) - result=await call(app,'content.text_patch_batch',{'patches':patches},key=key,subject=subject, - expected=tuple((item.resources[0].id,item.data['generation']) for item in created)) - assert result.error.code=='internal_error',wire(result) - assert await snapshot(app,ids)==before - assert await body(app,ids[0])=='one X' and await body(app,ids[1])=='two Y' + return await original(topic, revision) + + monkeypatch.setattr(app.contents, 'commit_revision', fail_second) + result = await call( + app, + 'content.text_patch_batch', + {'patches': patches}, + key=key, + subject=subject, + expected=tuple((item.resources[0].id, item.data['generation']) for item in created), + ) + assert result.error.code == 'internal_error', wire(result) + assert await snapshot(app, ids) == before + assert await body(app, ids[0]) == 'one X' and await body(app, ids[1]) == 'two Y' diff --git a/tests/test_todo_due_inbox.py b/tests/test_todo_due_inbox.py index 784426f1..8df87e36 100644 --- a/tests/test_todo_due_inbox.py +++ b/tests/test_todo_due_inbox.py @@ -1,11 +1,12 @@ """Due reminders are explicit worker writes, private and idempotent.""" + from datetime import timedelta import pytest +from test_service import NOW, call, register from msg.core.codec import wire from msg.workers.maintenance import run_maintenance -from test_service import NOW, call, register @pytest.mark.asyncio @@ -13,37 +14,55 @@ async def test_due_todo_is_only_in_owner_inbox_and_repeated_maintenance_is_noop( app, _ = installed owner_key, owner, _ = await register(app, 'due-owner') other_key, other, _ = await register(app, 'due-other') - todo = await call(app, 'identity.todo_put', + todo = await call( + app, + 'identity.todo_put', {'name': 'due', 'title': 'Private due title', 'due_at': wire(NOW - timedelta(hours=1))}, - key=owner_key, subject=owner) + key=owner_key, + subject=owner, + ) assert todo.status == 'ok', wire(todo) rid = todo.resources[0].id async with app.metadata.transaction(write=False) as tx: before = tx.one('SELECT COUNT(*) FROM audit')[0] - assert tx.one("SELECT COUNT(*) FROM messages WHERE resource=?", (rid,))[0] == 0 + assert tx.one('SELECT COUNT(*) FROM messages WHERE resource=?', (rid,))[0] == 0 # Reads neither materialize the due reminder nor add an audit record. assert (await call(app, 'communication.inbox', {}, key=owner_key, subject=owner)).status == 'ok' - assert (await call(app, 'identity.todo_get', {'name': 'due'}, key=owner_key, subject=owner)).status == 'ok' + assert ( + await call(app, 'identity.todo_get', {'name': 'due'}, key=owner_key, subject=owner) + ).status == 'ok' async with app.metadata.transaction(write=False) as tx: assert tx.one('SELECT COUNT(*) FROM audit')[0] == before - assert tx.one("SELECT COUNT(*) FROM messages WHERE resource=?", (rid,))[0] == 0 + assert tx.one('SELECT COUNT(*) FROM messages WHERE resource=?', (rid,))[0] == 0 - assert (await run_maintenance(app, 'deliver_due_todos', scheduled=True)) == {'delivered_todo_reminders': 1} + assert (await run_maintenance(app, 'deliver_due_todos', scheduled=True)) == { + 'delivered_todo_reminders': 1 + } async with app.metadata.transaction(write=False) as tx: assert tx.one('SELECT COUNT(*) FROM audit')[0] == before + 1 - assert (await run_maintenance(app, 'deliver_due_todos', scheduled=True)) == {'delivered_todo_reminders': 0} + assert (await run_maintenance(app, 'deliver_due_todos', scheduled=True)) == { + 'delivered_todo_reminders': 0 + } async with app.metadata.transaction(write=False) as tx: assert tx.one('SELECT COUNT(*) FROM audit')[0] == before + 1 - assert tx.one("SELECT COUNT(*) FROM messages WHERE resource=?", (rid,))[0] == 1 + assert tx.one('SELECT COUNT(*) FROM messages WHERE resource=?', (rid,))[0] == 1 own = await call(app, 'communication.inbox', {}, key=owner_key, subject=owner) notices = [item for item in own.data['items'] if item['resource']['id'] == rid] assert len(notices) == 1 and notices[0]['source'] == 'todo_due' assert notices[0]['recipient'] == owner assert notices[0]['todo_revision_at_delivery'] == todo.resources[0].revision - assert not [item for item in (await call(app, 'communication.inbox', {}, key=other_key, - subject=other)).data['items'] if item['resource']['id'] == rid] - assert not (await call(app, 'discovery.search', {'query': 'Private due title'}, - key=other_key, subject=other)).data.get('results', []) + assert not [ + item + for item in (await call(app, 'communication.inbox', {}, key=other_key, subject=other)).data[ + 'items' + ] + if item['resource']['id'] == rid + ] + assert not ( + await call( + app, 'discovery.search', {'query': 'Private due title'}, key=other_key, subject=other + ) + ).data.get('results', []) @pytest.mark.asyncio @@ -52,33 +71,72 @@ async def test_only_current_pending_due_revision_can_be_delivered(installed): key, owner, _ = await register(app, 'due-revisions') past = wire(NOW - timedelta(hours=1)) future = wire(NOW + timedelta(days=2)) - original = await call(app, 'identity.todo_put', - {'name': 'rescheduled', 'title': 'Rescheduled', 'due_at': past}, key=key, subject=owner) + original = await call( + app, + 'identity.todo_put', + {'name': 'rescheduled', 'title': 'Rescheduled', 'due_at': past}, + key=key, + subject=owner, + ) rid = original.resources[0].id - revised = await call(app, 'identity.todo_put', - {'name': 'rescheduled', 'title': 'Rescheduled', 'due_at': future, - 'expected_revision': original.resources[0].revision}, key=key, subject=owner, - expected=((rid, original.data['generation']),)) + revised = await call( + app, + 'identity.todo_put', + { + 'name': 'rescheduled', + 'title': 'Rescheduled', + 'due_at': future, + 'expected_revision': original.resources[0].revision, + }, + key=key, + subject=owner, + expected=((rid, original.data['generation']),), + ) assert revised.status == 'ok', wire(revised) - completed = await call(app, 'identity.todo_put', + completed = await call( + app, + 'identity.todo_put', {'name': 'done', 'title': 'Completed', 'due_at': past, 'status': 'done'}, - key=key, subject=owner) - archived = await call(app, 'identity.todo_put', - {'name': 'archived', 'title': 'Archived', 'due_at': past}, key=key, subject=owner) + key=key, + subject=owner, + ) + archived = await call( + app, + 'identity.todo_put', + {'name': 'archived', 'title': 'Archived', 'due_at': past}, + key=key, + subject=owner, + ) archived_id = archived.resources[0].id - result = await call(app, 'identity.todo_archive', + result = await call( + app, + 'identity.todo_archive', {'name': 'archived', 'expected_revision': archived.resources[0].revision}, - key=key, subject=owner, expected=((archived_id, archived.data['generation']),)) + key=key, + subject=owner, + expected=((archived_id, archived.data['generation']),), + ) assert result.status == 'ok', wire(result) - assert (await run_maintenance(app, 'deliver_due_todos', scheduled=True)) == {'delivered_todo_reminders': 0} + assert (await run_maintenance(app, 'deliver_due_todos', scheduled=True)) == { + 'delivered_todo_reminders': 0 + } async with app.metadata.transaction(write=False) as tx: - assert tx.one("SELECT COUNT(*) FROM messages WHERE resource IN (?,?,?)", - (rid, completed.resources[0].id, archived_id))[0] == 0 + assert ( + tx.one( + 'SELECT COUNT(*) FROM messages WHERE resource IN (?,?,?)', + (rid, completed.resources[0].id, archived_id), + )[0] + == 0 + ) # A later due instant is a distinct reminder, delivered only once then. app.clock = lambda: NOW + timedelta(days=3) - assert (await run_maintenance(app, 'deliver_due_todos', scheduled=True)) == {'delivered_todo_reminders': 1} - assert (await run_maintenance(app, 'deliver_due_todos', scheduled=True)) == {'delivered_todo_reminders': 0} + assert (await run_maintenance(app, 'deliver_due_todos', scheduled=True)) == { + 'delivered_todo_reminders': 1 + } + assert (await run_maintenance(app, 'deliver_due_todos', scheduled=True)) == { + 'delivered_todo_reminders': 0 + } async with app.metadata.transaction(write=False) as tx: row = tx.one('SELECT body FROM messages WHERE resource=?', (rid,)) assert row is not None and future in row[0] and past not in row[0] diff --git a/tests/test_token_delivery.py b/tests/test_token_delivery.py index e78e9edf..5637d300 100644 --- a/tests/test_token_delivery.py +++ b/tests/test_token_delivery.py @@ -1,15 +1,16 @@ """Strict token delivery is opt-in at issuance version 2.""" + import asyncio import os from datetime import timedelta import pytest +from test_service import NOW, call, register, temporary_v3_args from msg.application import Application -from msg.core.codec import b64,unb64,wire -from msg.core.requests import receipt_bytes,request_for +from msg.core.codec import b64, unb64, wire +from msg.core.requests import receipt_bytes, request_for from msg.security.crypto import verify -from test_service import NOW,call,register,temporary_v3_args def secret(): @@ -18,149 +19,263 @@ def secret(): @pytest.mark.asyncio async def test_v2_bootstrap_claim_once_and_recover_lost_response(installed): - app,_=installed - original,_,_,_=temporary_v3_args(request_id='strict-bootstrap') - created=await call(app,'identity.temporary',original,rid='strict-bootstrap',contract_version=3) - assert created.status=='ok' and created.data['token'] - uid=created.data['subject_id'] - old_id=created.data['credential_id'] + app, _ = installed + original, _, _, _ = temporary_v3_args(request_id='strict-bootstrap') + created = await call( + app, 'identity.temporary', original, rid='strict-bootstrap', contract_version=3 + ) + assert created.status == 'ok' and created.data['token'] + uid = created.data['subject_id'] + old_id = created.data['credential_id'] async with app.metadata.transaction(write=False) as tx: - row=tx.one('SELECT recovery_verifier,claimed_at,consumed_at FROM token_deliveries WHERE credential_id=?',(old_id,)) + row = tx.one( + 'SELECT recovery_verifier,claimed_at,consumed_at FROM token_deliveries WHERE credential_id=?', + (old_id,), + ) assert row[1] and row[2] is None assert original['recovery_secret'] not in str(row) assert created.data['token'] not in str(row) - replay=await call(app,'identity.temporary',original,rid='strict-bootstrap',contract_version=3) - assert replay.error.code=='token_delivery_unavailable' - args={'credential_id':old_id,'original_request_id':'strict-bootstrap', - 'recovery_secret':original['recovery_secret'], - 'nonce':secret(),'new_recovery_secret':secret()} - wrong=await call(app,'identity.token_recover',{**args,'recovery_secret':secret()}, - subject=uid,rid='bad-recovery') - assert wrong.error.code=='recovery_unavailable' - recovered=await call(app,'identity.token_recover',args,subject=uid,rid='recover-once') - assert recovered.status=='ok' and recovered.data['token'],recovered.error.code - verify(app.receipt_signer.public_key,receipt_bytes(recovered),recovered.receipt,purpose='receipt') - new=(recovered.data['credential_id'],unb64(recovered.data['token'])) - old=(old_id,unb64(created.data['token'])) - assert new[0]!=old[0] - repeated=await call(app,'identity.token_recover',args,subject=uid,rid='recover-once') - assert repeated.error.code=='token_delivery_unavailable' - reuse=await call(app,'identity.token_recover',{**args,'nonce':secret()}, - subject=uid,rid='another-recovery') - assert reuse.error.code=='recovery_unavailable' - blocked=await call(app,'content.post_create',{'parent':'/main','body':'old'},subject=uid,token=old) - assert blocked.error.code=='credential_revoked' - allowed=await call(app,'content.post_create',{'parent':'/main','body':'new'},subject=uid,token=new) - assert allowed.status=='ok',wire(allowed) + replay = await call( + app, 'identity.temporary', original, rid='strict-bootstrap', contract_version=3 + ) + assert replay.error.code == 'token_delivery_unavailable' + args = { + 'credential_id': old_id, + 'original_request_id': 'strict-bootstrap', + 'recovery_secret': original['recovery_secret'], + 'nonce': secret(), + 'new_recovery_secret': secret(), + } + wrong = await call( + app, + 'identity.token_recover', + {**args, 'recovery_secret': secret()}, + subject=uid, + rid='bad-recovery', + ) + assert wrong.error.code == 'recovery_unavailable' + recovered = await call(app, 'identity.token_recover', args, subject=uid, rid='recover-once') + assert recovered.status == 'ok' and recovered.data['token'], recovered.error.code + verify( + app.receipt_signer.public_key, + receipt_bytes(recovered), + recovered.receipt, + purpose='receipt', + ) + new = (recovered.data['credential_id'], unb64(recovered.data['token'])) + old = (old_id, unb64(created.data['token'])) + assert new[0] != old[0] + repeated = await call(app, 'identity.token_recover', args, subject=uid, rid='recover-once') + assert repeated.error.code == 'token_delivery_unavailable' + reuse = await call( + app, + 'identity.token_recover', + {**args, 'nonce': secret()}, + subject=uid, + rid='another-recovery', + ) + assert reuse.error.code == 'recovery_unavailable' + blocked = await call( + app, 'content.post_create', {'parent': '/main', 'body': 'old'}, subject=uid, token=old + ) + assert blocked.error.code == 'credential_revoked' + allowed = await call( + app, 'content.post_create', {'parent': '/main', 'body': 'new'}, subject=uid, token=new + ) + assert allowed.status == 'ok', wire(allowed) @pytest.mark.asyncio async def test_v2_concurrent_claim_and_cross_subject_denial(installed): - app,_=installed - args={'handle':'delivery-race','nonce':secret(),'recovery_secret':secret()} - first,second=await asyncio.gather(*( - call(app,'identity.custodial_create',args,rid='race',contract_version=2) - for _ in range(2))) - assert sorted((first.status,second.status))==['error','ok'] - assert next(r for r in (first,second) if r.status=='error').error.code=='token_delivery_unavailable' - winner=next(r for r in (first,second) if r.status=='ok') - other_args,other_rid,_,_=temporary_v3_args() - other=await call(app,'identity.temporary',other_args, - rid=other_rid,contract_version=3) - denied=await call(app,'identity.token_recover',{ - 'credential_id':winner.data['credential_id'],'original_request_id':'race', - 'recovery_secret':args['recovery_secret'], - 'nonce':secret(),'new_recovery_secret':secret()},subject=other.data['subject_id']) - assert denied.error.code=='recovery_unavailable' + app, _ = installed + args = {'handle': 'delivery-race', 'nonce': secret(), 'recovery_secret': secret()} + first, second = await asyncio.gather( + *( + call(app, 'identity.custodial_create', args, rid='race', contract_version=2) + for _ in range(2) + ) + ) + assert sorted((first.status, second.status)) == ['error', 'ok'] + assert ( + next(r for r in (first, second) if r.status == 'error').error.code + == 'token_delivery_unavailable' + ) + winner = next(r for r in (first, second) if r.status == 'ok') + other_args, other_rid, _, _ = temporary_v3_args() + other = await call(app, 'identity.temporary', other_args, rid=other_rid, contract_version=3) + denied = await call( + app, + 'identity.token_recover', + { + 'credential_id': winner.data['credential_id'], + 'original_request_id': 'race', + 'recovery_secret': args['recovery_secret'], + 'nonce': secret(), + 'new_recovery_secret': secret(), + }, + subject=other.data['subject_id'], + ) + assert denied.error.code == 'recovery_unavailable' @pytest.mark.asyncio async def test_v2_unclaimed_commit_survives_restart_and_expiry(installed): - app,_=installed - args,_,_,_=temporary_v3_args(request_id='restart-before-claim') - original_hook=app.executor.response_hook - app.executor.response_hook=None # simulates commit followed by process loss before claim - committed=await call(app,'identity.temporary',args,rid='restart-before-claim',contract_version=3) - assert committed.status=='ok' and 'token' not in committed.data - second=Application(app.settings,clock=lambda:NOW) + app, _ = installed + args, _, _, _ = temporary_v3_args(request_id='restart-before-claim') + original_hook = app.executor.response_hook + app.executor.response_hook = None # simulates commit followed by process loss before claim + committed = await call( + app, 'identity.temporary', args, rid='restart-before-claim', contract_version=3 + ) + assert committed.status == 'ok' and 'token' not in committed.data + second = Application(app.settings, clock=lambda: NOW) await second.load() try: - claimed=await call(second,'identity.temporary',args,rid='restart-before-claim',contract_version=3) - assert claimed.status=='ok' and claimed.replayed and claimed.data['token'] - exhausted=await call(second,'identity.temporary',args,rid='restart-before-claim',contract_version=3) - assert exhausted.error.code=='token_delivery_unavailable' - uid=claimed.data['subject_id'] - recovery={'credential_id':claimed.data['credential_id'], - 'original_request_id':'restart-before-claim', - 'recovery_secret':args['recovery_secret'],'nonce':secret(), - 'new_recovery_secret':secret()} - second.clock=lambda:NOW+timedelta(minutes=16) - second.authenticator.clock=second.clock - packet=request_for('identity.token_recover',recovery,second.settings.service_url, - subject=uid,expires_at=NOW+timedelta(minutes=18)) - expired=await second.executor.execute(packet) - assert expired.error.code=='recovery_unavailable' + claimed = await call( + second, 'identity.temporary', args, rid='restart-before-claim', contract_version=3 + ) + assert claimed.status == 'ok' and claimed.replayed and claimed.data['token'] + exhausted = await call( + second, 'identity.temporary', args, rid='restart-before-claim', contract_version=3 + ) + assert exhausted.error.code == 'token_delivery_unavailable' + uid = claimed.data['subject_id'] + recovery = { + 'credential_id': claimed.data['credential_id'], + 'original_request_id': 'restart-before-claim', + 'recovery_secret': args['recovery_secret'], + 'nonce': secret(), + 'new_recovery_secret': secret(), + } + second.clock = lambda: NOW + timedelta(minutes=16) + second.authenticator.clock = second.clock + packet = request_for( + 'identity.token_recover', + recovery, + second.settings.service_url, + subject=uid, + expires_at=NOW + timedelta(minutes=18), + ) + expired = await second.executor.execute(packet) + assert expired.error.code == 'recovery_unavailable' finally: await second.close() - app.executor.response_hook=original_hook + app.executor.response_hook = original_hook @pytest.mark.asyncio async def test_v2_rotate_and_signed_create_bind_separate_recovery(installed): - app,_=installed - bootstrap_args,_,_,_=temporary_v3_args(request_id='rotate-bootstrap') - bootstrap=await call(app,'identity.temporary',bootstrap_args, - rid='rotate-bootstrap',contract_version=3) - uid=bootstrap.data['subject_id'] - token=(bootstrap.data['credential_id'],unb64(bootstrap.data['token'])) - rotate_args={'nonce':secret(),'recovery_secret':secret()} - rotated=await call(app,'identity.token_rotate',rotate_args,subject=uid,token=token, - rid='rotate-v2',contract_version=2) - assert rotated.status=='ok' and rotated.data['token'],rotated.error - replay=await call(app,'identity.token_rotate',rotate_args,subject=uid,token=token, - rid='rotate-v2',contract_version=2) - assert replay.error.code=='token_delivery_unavailable' - recovered=await call(app,'identity.token_recover',{ - 'credential_id':rotated.data['credential_id'], - 'original_request_id':'rotate-v2', - 'recovery_secret':rotate_args['recovery_secret'], - 'nonce':secret(),'new_recovery_secret':secret()},subject=uid) - assert recovered.status=='ok' and recovered.data['token'],recovered.error - - signer,signed_uid,_=await register(app,'signed-delivery') - created_args={'nonce':secret(),'recovery_secret':secret(),'ceiling':[],'ttl':900} - issued=await call(app,'identity.token_create',created_args,key=signer, - subject=signed_uid,contract_version=2) - assert issued.status=='ok' and issued.data['token'],issued.error - again=await call(app,'identity.token_create',created_args,key=signer, - subject=signed_uid,rid=issued.request_id,contract_version=2) - assert again.error.code=='token_delivery_unavailable' - restored=await call(app,'identity.token_recover',{ - 'credential_id':issued.data['credential_id'], - 'original_request_id':issued.request_id, - 'recovery_secret':created_args['recovery_secret'], - 'nonce':secret(),'new_recovery_secret':secret()},subject=signed_uid) - assert restored.status=='ok' and restored.data['token'],restored.error + app, _ = installed + bootstrap_args, _, _, _ = temporary_v3_args(request_id='rotate-bootstrap') + bootstrap = await call( + app, 'identity.temporary', bootstrap_args, rid='rotate-bootstrap', contract_version=3 + ) + uid = bootstrap.data['subject_id'] + token = (bootstrap.data['credential_id'], unb64(bootstrap.data['token'])) + rotate_args = {'nonce': secret(), 'recovery_secret': secret()} + rotated = await call( + app, + 'identity.token_rotate', + rotate_args, + subject=uid, + token=token, + rid='rotate-v2', + contract_version=2, + ) + assert rotated.status == 'ok' and rotated.data['token'], rotated.error + replay = await call( + app, + 'identity.token_rotate', + rotate_args, + subject=uid, + token=token, + rid='rotate-v2', + contract_version=2, + ) + assert replay.error.code == 'token_delivery_unavailable' + recovered = await call( + app, + 'identity.token_recover', + { + 'credential_id': rotated.data['credential_id'], + 'original_request_id': 'rotate-v2', + 'recovery_secret': rotate_args['recovery_secret'], + 'nonce': secret(), + 'new_recovery_secret': secret(), + }, + subject=uid, + ) + assert recovered.status == 'ok' and recovered.data['token'], recovered.error + + signer, signed_uid, _ = await register(app, 'signed-delivery') + created_args = {'nonce': secret(), 'recovery_secret': secret(), 'ceiling': [], 'ttl': 900} + issued = await call( + app, + 'identity.token_create', + created_args, + key=signer, + subject=signed_uid, + contract_version=2, + ) + assert issued.status == 'ok' and issued.data['token'], issued.error + again = await call( + app, + 'identity.token_create', + created_args, + key=signer, + subject=signed_uid, + rid=issued.request_id, + contract_version=2, + ) + assert again.error.code == 'token_delivery_unavailable' + restored = await call( + app, + 'identity.token_recover', + { + 'credential_id': issued.data['credential_id'], + 'original_request_id': issued.request_id, + 'recovery_secret': created_args['recovery_secret'], + 'nonce': secret(), + 'new_recovery_secret': secret(), + }, + subject=signed_uid, + ) + assert restored.status == 'ok' and restored.data['token'], restored.error @pytest.mark.asyncio async def test_v2_recovery_secret_is_consumed_once_under_race(installed): - app,_=installed - args,request_id,_,_=temporary_v3_args() - created=await call(app,'identity.temporary',args,rid=request_id,contract_version=3) - uid=created.data['subject_id'] - base={'credential_id':created.data['credential_id'], - 'original_request_id':created.request_id, - 'recovery_secret':args['recovery_secret']} - first,second=await asyncio.gather( - call(app,'identity.token_recover',{ - **base,'nonce':secret(),'new_recovery_secret':secret()}, - subject=uid,rid='recover-race-a'), - call(app,'identity.token_recover',{ - **base,'nonce':secret(),'new_recovery_secret':secret()}, - subject=uid,rid='recover-race-b')) - assert sorted((first.status,second.status))==['error','ok'] - assert next(r for r in (first,second) if r.status=='error').error.code=='recovery_unavailable' + app, _ = installed + args, request_id, _, _ = temporary_v3_args() + created = await call(app, 'identity.temporary', args, rid=request_id, contract_version=3) + uid = created.data['subject_id'] + base = { + 'credential_id': created.data['credential_id'], + 'original_request_id': created.request_id, + 'recovery_secret': args['recovery_secret'], + } + first, second = await asyncio.gather( + call( + app, + 'identity.token_recover', + {**base, 'nonce': secret(), 'new_recovery_secret': secret()}, + subject=uid, + rid='recover-race-a', + ), + call( + app, + 'identity.token_recover', + {**base, 'nonce': secret(), 'new_recovery_secret': secret()}, + subject=uid, + rid='recover-race-b', + ), + ) + assert sorted((first.status, second.status)) == ['error', 'ok'] + assert ( + next(r for r in (first, second) if r.status == 'error').error.code == 'recovery_unavailable' + ) async with app.metadata.transaction(write=False) as tx: - assert tx.one('SELECT consumed_at FROM token_deliveries WHERE credential_id=?', - (created.data['credential_id'],))[0] + assert tx.one( + 'SELECT consumed_at FROM token_deliveries WHERE credential_id=?', + (created.data['credential_id'],), + )[0] diff --git a/tests/test_token_delivery_boundary.py b/tests/test_token_delivery_boundary.py index e8e4743f..df333300 100644 --- a/tests/test_token_delivery_boundary.py +++ b/tests/test_token_delivery_boundary.py @@ -1,4 +1,5 @@ """Credential delivery has one owner, separate from service composition.""" + import ast import asyncio import hashlib @@ -6,17 +7,17 @@ import importlib import importlib.util import inspect -from pathlib import Path import subprocess import sys +from pathlib import Path import pytest +from test_service import NOW, temporary_v3_args from msg.core.codec import b64, canonical, loads, unb64, wire from msg.core.errors import Failure -from msg.core.requests import request_for, receipt_bytes +from msg.core.requests import receipt_bytes, request_for from msg.security.crypto import verify -from test_service import NOW, temporary_v3_args def delivery_module(): @@ -26,10 +27,14 @@ def delivery_module(): def test_composition_only_delegates_the_existing_delivery_api(): from msg.application import Application + module = delivery_module() tree = ast.parse(inspect.getsource(Application)) - methods = {node.name: node for node in tree.body[0].body - if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef))} + methods = { + node.name: node + for node in tree.body[0].body + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) + } for name in ('issued_token', 'record_token_delivery', '_secrets_for_caller'): method = methods[name] assert len(method.body) == 1 and isinstance(method.body[0], ast.Return) @@ -41,7 +46,11 @@ def test_composition_only_delegates_the_existing_delivery_api(): def test_domain_import_and_constructor_need_no_application_or_plugins(): module = delivery_module() assert set(inspect.signature(module.TokenDelivery).parameters) == { - 'metadata', 'token_secret', 'clock', 'recovery_window'} + 'metadata', + 'token_secret', + 'clock', + 'recovery_window', + } code = """ import importlib.abc import sys @@ -61,22 +70,39 @@ def find_spec(self, fullname, path=None, target=None): def test_derivation_and_recovery_verifier_keep_the_published_bytes(): module = delivery_module() - instance = module.TokenDelivery(metadata=None, token_secret=b'k'*32, - clock=lambda: NOW, recovery_window=lambda: 900) - request = request_for('identity.token_create', {'nonce': b64(b'n'*32)}, - 'https://example.invalid', request_id='stable-delivery', contract_version=2) - expected = hmac.digest(b'k'*32, b'issued-token-v1\0' + canonical({ - 'subject': 'u_subject', 'request_id': 'stable-delivery', 'nonce': b64(b'n'*32)}), 'sha256') + instance = module.TokenDelivery( + metadata=None, token_secret=b'k' * 32, clock=lambda: NOW, recovery_window=lambda: 900 + ) + request = request_for( + 'identity.token_create', + {'nonce': b64(b'n' * 32)}, + 'https://example.invalid', + request_id='stable-delivery', + contract_version=2, + ) + expected = hmac.digest( + b'k' * 32, + b'issued-token-v1\0' + + canonical({ + 'subject': 'u_subject', + 'request_id': 'stable-delivery', + 'nonce': b64(b'n' * 32), + }), + 'sha256', + ) assert instance.issued_token(request, 'u_subject') == expected assert instance.issued_token(request, 'u_other') != expected - assert module.recovery_verifier(b64(b'r'*32)) == hashlib.sha256( - b'token-recovery-v1\0' + b'r'*32).hexdigest() + assert ( + module.recovery_verifier(b64(b'r' * 32)) + == hashlib.sha256(b'token-recovery-v1\0' + b'r' * 32).hexdigest() + ) with pytest.raises(Failure, match='invalid_recovery_secret'): - module.recovery_verifier(b64(b'r'*31)) + module.recovery_verifier(b64(b'r' * 31)) def test_authentication_uses_the_same_verifier_owner(): from msg.security import authentication + module = delivery_module() assert authentication.recovery_verifier is module.recovery_verifier source = Path(authentication.__file__).read_text() @@ -95,16 +121,28 @@ async def test_standalone_delivery_claim_commits_once_without_persisting_secrets app, _ = installed module = delivery_module() args, rid, _, _ = temporary_v3_args(request_id='domain-claim-once') - request = request_for('identity.temporary', args, app.settings.service_url, - request_id=rid, contract_version=3, - expires_at=NOW.replace(minute=1)) + request = request_for( + 'identity.temporary', + args, + app.settings.service_url, + request_id=rid, + contract_version=3, + expires_at=NOW.replace(minute=1), + ) app.executor.response_hook = None committed = await app.executor.execute(request) assert committed.status == 'ok' and 'token' not in committed.data, wire(committed) - delivery = module.TokenDelivery(metadata=app.metadata, token_secret=app._token_secret, - clock=lambda: NOW, recovery_window=lambda: app.settings.credential_delivery_recovery_window) - answers = await asyncio.gather(delivery.release(request, committed), - delivery.release(request, committed), return_exceptions=True) + delivery = module.TokenDelivery( + metadata=app.metadata, + token_secret=app._token_secret, + clock=lambda: NOW, + recovery_window=lambda: app.settings.credential_delivery_recovery_window, + ) + answers = await asyncio.gather( + delivery.release(request, committed), + delivery.release(request, committed), + return_exceptions=True, + ) successes = [value for value in answers if not isinstance(value, BaseException)] failures = [value for value in answers if isinstance(value, Failure)] assert len(successes) == len(failures) == 1 @@ -112,13 +150,19 @@ async def test_standalone_delivery_claim_commits_once_without_persisting_secrets released = successes[0] assert unb64(released.data['token']) == app.issued_token(request, committed.subject) assert released.receipt == committed.receipt - verify(app.receipt_signer.public_key, receipt_bytes(released), released.receipt, purpose='receipt') + verify( + app.receipt_signer.public_key, receipt_bytes(released), released.receipt, purpose='receipt' + ) async with app.metadata.transaction(write=False) as tx: saved = await tx.request_result(committed.subject, rid, request.payload_digest) - assert saved.receipt == committed.receipt and receipt_bytes(saved) == receipt_bytes(committed) + assert saved.receipt == committed.receipt and receipt_bytes(saved) == receipt_bytes( + committed + ) assert saved.data == committed.data and 'token' not in saved.data - row = tx.one('SELECT claimed_at,recovery_verifier FROM token_deliveries WHERE credential_id=?', - (committed.data['credential_id'],)) + row = tx.one( + 'SELECT claimed_at,recovery_verifier FROM token_deliveries WHERE credential_id=?', + (committed.data['credential_id'],), + ) assert row[0] is not None and row[1] == module.recovery_verifier(args['recovery_secret']) assert released.data['token'] not in str(row) and args['recovery_secret'] not in str(row) # Event fields are stored in canonical JSON, not separate SQL columns. @@ -134,20 +178,36 @@ async def test_delivery_failure_returns_no_new_secret_or_claim(installed): app, _ = installed module = delivery_module() args, rid, _, _ = temporary_v3_args(request_id='domain-expired-claim') - request = request_for('identity.temporary', args, app.settings.service_url, - request_id=rid, contract_version=3, expires_at=NOW.replace(minute=1)) + request = request_for( + 'identity.temporary', + args, + app.settings.service_url, + request_id=rid, + contract_version=3, + expires_at=NOW.replace(minute=1), + ) app.executor.response_hook = None committed = await app.executor.execute(request) assert committed.status == 'ok', wire(committed) async with app.metadata.transaction(write=False) as tx: credential = await tx.credential(committed.data['credential_id']) - delivery = module.TokenDelivery(metadata=app.metadata, token_secret=app._token_secret, - clock=lambda: credential.expires_at, recovery_window=lambda: 900) + delivery = module.TokenDelivery( + metadata=app.metadata, + token_secret=app._token_secret, + clock=lambda: credential.expires_at, + recovery_window=lambda: 900, + ) with pytest.raises(Failure, match='credential_expired'): await delivery.release(request, committed) async with app.metadata.transaction(write=False) as tx: - assert tx.one('SELECT claimed_at FROM token_deliveries WHERE credential_id=?', - (credential.id,))[0] is None + assert ( + tx.one( + 'SELECT claimed_at FROM token_deliveries WHERE credential_id=?', (credential.id,) + )[0] + is None + ) saved = await tx.request_result(committed.subject, rid, request.payload_digest) - assert saved.receipt == committed.receipt and receipt_bytes(saved) == receipt_bytes(committed) + assert saved.receipt == committed.receipt and receipt_bytes(saved) == receipt_bytes( + committed + ) assert saved.data == committed.data and 'token' not in saved.data diff --git a/tests/test_tool_feature_selftest.py b/tests/test_tool_feature_selftest.py index 0bfdf162..3cec958e 100644 --- a/tests/test_tool_feature_selftest.py +++ b/tests/test_tool_feature_selftest.py @@ -10,14 +10,17 @@ @pytest.mark.asyncio async def test_doctor_exercises_real_bubblewrap_deny_path(tmp_path): - app = Application(write_example(tmp_path/'etc', tmp_path/'data')) + app = Application(write_example(tmp_path / 'etc', tmp_path / 'data')) assert await inspect_tool_sandbox(app) == { - 'runner': 'bubblewrap', 'policy_denial': True, 'external_requests': 0} + 'runner': 'bubblewrap', + 'policy_denial': True, + 'external_requests': 0, + } @pytest.mark.asyncio async def test_missing_sandbox_is_not_reported_ready(tmp_path, monkeypatch): - app = Application(write_example(tmp_path/'etc', tmp_path/'data')) + app = Application(write_example(tmp_path / 'etc', tmp_path / 'data')) monkeypatch.setattr('msg.workers.sandbox.shutil.which', lambda _: None) with pytest.raises(Failure, match='tool_isolation_unavailable'): await inspect_tool_sandbox(app) @@ -26,33 +29,46 @@ async def test_missing_sandbox_is_not_reported_ready(tmp_path, monkeypatch): @pytest.mark.asyncio async def test_tool_selftest_rejects_live_application(): from msg.admin.tool_check import check_tool_sandbox + with pytest.raises(Failure, match='selftest_namespace_required'): await check_tool_sandbox(SimpleNamespace(selftest_run_id=None), None, None, None, None) @pytest.mark.asyncio -async def test_tool_selftest_distinguishes_runner_output_from_operation_denial(installed, monkeypatch): +async def test_tool_selftest_distinguishes_runner_output_from_operation_denial( + installed, monkeypatch +): from test_authorization import approve from test_service import call, register from msg.admin.tool_check import check_tool_sandbox + app, root = installed monkeypatch.setattr(app, 'selftest_run_id', 'isolated-tool-fixture') key, subject, _ = await register(app, 'tool-selftest') + async def invoke(name, arguments, signer, uid, certificates): return await call(app, name, arguments, key=signer, subject=uid, certs=certificates) + async def certify(signer, uid, grants): return await approve(app, root, uid, signer, grants) + result = await check_tool_sandbox(app, invoke, certify, key, subject) - assert result == {'operation': 'private_target_denied', + assert result == { + 'operation': 'private_target_denied', 'runner': 'loopback_output_and_policy_denial', - 'normal_operation_pipeline': 'not_exercised', 'external_requests': 0} + 'normal_operation_pipeline': 'not_exercised', + 'external_requests': 0, + } def test_on_demand_feature_samples_are_explicitly_empty(): from msg.bootstrap import feature_manifest + rows = {row['feature_id']: row for row in feature_manifest()} for name in ('content_editing', 'private_dm', 'transfer', 'following', 'collaboration'): assert rows[name]['sample_resource'] is None assert rows[name]['default_config'] == { - 'sample_state': 'empty', 'sample_creation': 'isolated_selftest'} + 'sample_state': 'empty', + 'sample_creation': 'isolated_selftest', + } diff --git a/tests/test_tool_runner_contract.py b/tests/test_tool_runner_contract.py index c1d915a0..6f2f1196 100644 --- a/tests/test_tool_runner_contract.py +++ b/tests/test_tool_runner_contract.py @@ -1,20 +1,21 @@ """The runner owns local output; only the worker may publish a ResourceRef.""" + from __future__ import annotations import ast -from dataclasses import FrozenInstanceError, fields import inspect +from dataclasses import FrozenInstanceError, fields from pathlib import Path from typing import get_type_hints import pytest +from test_authorization import approve, scoped +from test_service import call, register from msg.core.errors import Failure from msg.core.models import JsonMap, NetworkPolicy, ResourceRef, ToolSpec from msg.workers.effects import EffectWorker from msg.workers.sandbox import BubblewrapRunner -from test_service import call, register -from test_authorization import approve, scoped def test_tool_result_has_one_neutral_owner_and_compatible_export(tmp_path): @@ -41,8 +42,13 @@ def test_actual_worker_dependency_is_the_declared_callable_port(): def test_default_runner_and_port_have_the_same_input_and_local_output_types(): from msg.core.tool_execution import ToolResult, ToolRunner - expected = {'tool': ToolSpec, 'arguments': JsonMap, - 'policies': tuple[NetworkPolicy, ...], 'directory': Path, 'return': ToolResult} + expected = { + 'tool': ToolSpec, + 'arguments': JsonMap, + 'policies': tuple[NetworkPolicy, ...], + 'directory': Path, + 'return': ToolResult, + } assert get_type_hints(ToolRunner.__call__) == expected assert get_type_hints(BubblewrapRunner.__call__) == expected assert inspect.iscoroutinefunction(BubblewrapRunner.__call__) @@ -69,8 +75,12 @@ def test_sandbox_does_not_import_its_worker_or_application_composition(): elif isinstance(node, ast.Import): dependencies.extend(alias.name for alias in node.names) assert 'msg.core.tool_execution' in dependencies - assert not any(name == 'msg.workers.effects' or name.startswith('msg.workers.effects.') - or name == 'msg.application' for name in dependencies) + assert not any( + name == 'msg.workers.effects' + or name.startswith('msg.workers.effects.') + or name == 'msg.application' + for name in dependencies + ) async def test_default_runner_still_refuses_missing_mandatory_sandbox(monkeypatch, tmp_path): @@ -91,14 +101,20 @@ async def test_injected_port_cannot_publish_a_resource_itself(installed, invalid cap = scoped(app, 'tool.use', 'tool_dns', app.registry.capability('tool.use').operations) certificate = await approve(app, root, subject, key, (cap,)) certs = (certificate.resource_id,) - request = await call(app, 'tool.run', + request = await call( + app, + 'tool.run', {'id': '/tools/dns', 'arguments': {'name': 'example.org', 'type': 'A'}}, - key=key, subject=subject, certs=certs) + key=key, + subject=subject, + certs=certs, + ) assert request.status == 'accepted' invocations = [] - async def runner(tool: ToolSpec, arguments: JsonMap, - policies: tuple[NetworkPolicy, ...], directory: Path) -> ToolResult: + async def runner( + tool: ToolSpec, arguments: JsonMap, policies: tuple[NetworkPolicy, ...], directory: Path + ) -> ToolResult: assert tool.executor_key == 'dns' assert arguments['name'] == 'example.org' assert isinstance(policies, tuple) and policies diff --git a/tests/test_tool_sandbox_acceptance.py b/tests/test_tool_sandbox_acceptance.py index adf4e622..27cf10de 100644 --- a/tests/test_tool_sandbox_acceptance.py +++ b/tests/test_tool_sandbox_acceptance.py @@ -1,4 +1,5 @@ """Real bubblewrap child enforces network policy without contacting a peer.""" + from types import SimpleNamespace import pytest @@ -9,16 +10,35 @@ @pytest.mark.asyncio -@pytest.mark.parametrize('url', ['http://127.0.0.1:9/', 'http://169.254.169.254/', - 'file:///etc/passwd', 'http://[::1]:9/']) +@pytest.mark.parametrize( + 'url', + ['http://127.0.0.1:9/', 'http://169.254.169.254/', 'file:///etc/passwd', 'http://[::1]:9/'], +) async def test_real_sandbox_denies_private_or_non_http_target(tmp_path, url): - app = SimpleNamespace(settings=SimpleNamespace(server=SimpleNamespace( - limits=SimpleNamespace(max_request_bytes=1024)))) + app = SimpleNamespace( + settings=SimpleNamespace( + server=SimpleNamespace(limits=SimpleNamespace(max_request_bytes=1024)) + ) + ) tool = SimpleNamespace(executor_key='curl') with pytest.raises(Failure) as caught: - await BubblewrapRunner(app)(tool, {'url': url}, (NetworkPolicy(schemes=frozenset({'http', 'https'}), hosts=(), - ports=frozenset({9, 80, 443}), methods=frozenset({'GET'}), allow_private=False, - timeout_ms=1000, max_response_bytes=1024, max_redirects=0),), tmp_path) + await BubblewrapRunner(app)( + tool, + {'url': url}, + ( + NetworkPolicy( + schemes=frozenset({'http', 'https'}), + hosts=(), + ports=frozenset({9, 80, 443}), + methods=frozenset({'GET'}), + allow_private=False, + timeout_ms=1000, + max_response_bytes=1024, + max_redirects=0, + ), + ), + tmp_path, + ) assert caught.value.code == 'network_policy_denied' assert not (tmp_path / 'output.bin').exists() @@ -32,8 +52,11 @@ async def test_real_sandbox_explicit_local_grant_returns_bounded_bytes(tmp_path, async def receive(reader, writer): requests.append(await reader.readuntil(b'\r\n\r\n')) - writer.write(f'HTTP/1.1 200 OK\r\nContent-Length: {len(body)}\r\n'.encode() + - b'Content-Type: text/plain\r\nConnection: close\r\n\r\n' + body) + writer.write( + f'HTTP/1.1 200 OK\r\nContent-Length: {len(body)}\r\n'.encode() + + b'Content-Type: text/plain\r\nConnection: close\r\n\r\n' + + body + ) await writer.drain() writer.close() await writer.wait_closed() @@ -41,13 +64,27 @@ async def receive(reader, writer): server = await asyncio.start_server(receive, '127.0.0.1', 0) async with server: port = server.sockets[0].getsockname()[1] - app = SimpleNamespace(settings=SimpleNamespace(server=SimpleNamespace( - limits=SimpleNamespace(max_request_bytes=1024)))) - policy = NetworkPolicy(schemes=frozenset({'http'}), hosts=('127.0.0.1',), - ports=frozenset({port}), methods=frozenset({'GET'}), allow_private=True, - timeout_ms=3000, max_response_bytes=4, max_redirects=0) - invocation = BubblewrapRunner(app)(SimpleNamespace(executor_key='curl'), - {'url': f'http://127.0.0.1:{port}/fixture'}, (policy,), tmp_path) + app = SimpleNamespace( + settings=SimpleNamespace( + server=SimpleNamespace(limits=SimpleNamespace(max_request_bytes=1024)) + ) + ) + policy = NetworkPolicy( + schemes=frozenset({'http'}), + hosts=('127.0.0.1',), + ports=frozenset({port}), + methods=frozenset({'GET'}), + allow_private=True, + timeout_ms=3000, + max_response_bytes=4, + max_redirects=0, + ) + invocation = BubblewrapRunner(app)( + SimpleNamespace(executor_key='curl'), + {'url': f'http://127.0.0.1:{port}/fixture'}, + (policy,), + tmp_path, + ) if len(body) > policy.max_response_bytes: with pytest.raises(Failure, match='tool_response_too_large'): await invocation diff --git a/tests/test_tools.py b/tests/test_tools.py index 25ea8ec7..fd47470c 100644 --- a/tests/test_tools.py +++ b/tests/test_tools.py @@ -1,272 +1,449 @@ from dataclasses import replace from datetime import timedelta -from pathlib import Path -import pytest + import httpx +import pytest +from test_authorization import approve, scoped +from test_service import NOW, call, register from msg.application import Application -from msg.core.codec import wire,canonical,digest,loads +from msg.core.codec import canonical, digest, loads, wire from msg.core.errors import Failure -from msg.core.models import Scope,NetworkPolicy,ResourceRef,Revision -from msg.plugins.common import new_id -from msg.security.network import validate_url,validate_addresses,intersect_policy -from msg.workers.effects import EffectWorker,ToolResult +from msg.core.models import NetworkPolicy, ResourceRef, Revision from msg.extensions.tools import descriptor, read_tool +from msg.plugins.common import new_id +from msg.security.network import intersect_policy, validate_addresses, validate_url from msg.transports.dictionary import build_dictionary from msg.transports.http import create_app -from test_service import register,call,NOW -from test_authorization import approve,scoped +from msg.workers.effects import EffectWorker, ToolResult def policy(**kw): - return NetworkPolicy(schemes=frozenset({'http','https'}),hosts=(),ports=frozenset({80,443}), - methods=frozenset({'GET','HEAD'}),allow_private=kw.get('allow_private',False),timeout_ms=1000, - max_response_bytes=10000,max_redirects=3) - - -@pytest.mark.parametrize('url',['file:///etc/passwd','http://user:pass@example.org','http://example.org\\@127.0.0.1', - 'http://example.org:22/','unix:///var/run/socket','http://example.org/\nX:test']) + return NetworkPolicy( + schemes=frozenset({'http', 'https'}), + hosts=(), + ports=frozenset({80, 443}), + methods=frozenset({'GET', 'HEAD'}), + allow_private=kw.get('allow_private', False), + timeout_ms=1000, + max_response_bytes=10000, + max_redirects=3, + ) + + +@pytest.mark.parametrize( + 'url', + [ + 'file:///etc/passwd', + 'http://user:pass@example.org', + 'http://example.org\\@127.0.0.1', + 'http://example.org:22/', + 'unix:///var/run/socket', + 'http://example.org/\nX:test', + ], +) def test_network_tool_rejects_non_http_or_ambiguous_targets(url): - with pytest.raises(Failure):validate_url(url,'GET',policy()) - - -@pytest.mark.parametrize('address',['127.0.0.1','10.2.3.4','169.254.169.254','::1','::ffff:127.0.0.1','fe80::1','0.0.0.0','100.64.0.1']) + with pytest.raises(Failure): + validate_url(url, 'GET', policy()) + + +@pytest.mark.parametrize( + 'address', + [ + '127.0.0.1', + '10.2.3.4', + '169.254.169.254', + '::1', + '::ffff:127.0.0.1', + 'fe80::1', + '0.0.0.0', + '100.64.0.1', + ], +) def test_public_tool_rejects_nonpublic_addresses(address): - with pytest.raises(Failure,match='private_target_forbidden'):validate_addresses([address],policy()) - if address=='0.0.0.0': - with pytest.raises(Failure,match='invalid_network_target'):validate_addresses([address],policy(allow_private=True)) + with pytest.raises(Failure, match='private_target_forbidden'): + validate_addresses([address], policy()) + if address == '0.0.0.0': + with pytest.raises(Failure, match='invalid_network_target'): + validate_addresses([address], policy(allow_private=True)) else: - assert validate_addresses([address],policy(allow_private=True))==[address] + assert validate_addresses([address], policy(allow_private=True)) == [address] def test_tool_policy_intersection_does_not_widen_allowlists_or_limits(): - restricted=intersect_policy(policy(),{'hosts':['example.org'],'methods':['GET'],'timeout_ms':500}) - assert validate_url('https://example.org/path','GET',restricted).hostname=='example.org' - assert restricted.timeout_ms==500 - with pytest.raises(Failure):validate_url('https://other.example','GET',restricted) - with pytest.raises(Failure):validate_url('https://example.org','HEAD',restricted) - with pytest.raises(Failure):intersect_policy(policy(),{'ports':[22]}) + restricted = intersect_policy( + policy(), {'hosts': ['example.org'], 'methods': ['GET'], 'timeout_ms': 500} + ) + assert validate_url('https://example.org/path', 'GET', restricted).hostname == 'example.org' + assert restricted.timeout_ms == 500 + with pytest.raises(Failure): + validate_url('https://other.example', 'GET', restricted) + with pytest.raises(Failure): + validate_url('https://example.org', 'HEAD', restricted) + with pytest.raises(Failure): + intersect_policy(policy(), {'ports': [22]}) @pytest.mark.asyncio -async def test_only_scoped_tool_is_visible_jobs_are_deduped_and_output_transfers(installed,tmp_path): - app,root=installed - key,uid,base=await register(app,'tool-agent') - denied=await call(app,'tool.run',{'id':'/tools/dns','arguments':{'name':'example.org','type':'A'}},key=key,subject=uid) - assert denied.error.code=='tool_certificate_required',wire(denied) - cap=scoped(app,'tool.use','tool_dns',app.registry.capability('tool.use').operations) - cert=await approve(app,root,uid,key,(cap,)) - tools=await call(app,'discovery.get',{'id':'/tools'},key=key,subject=uid,certs=(cert.resource_id,)) - assert [t['name'] for t in tools.data['items']]==['dns'] - wrong=await call(app,'tool.run',{'id':'/tools/curl','arguments':{'url':'https://example.org'}},key=key,subject=uid,certs=(cert.resource_id,)) - assert wrong.error.code=='tool_certificate_required' - args={'id':'/tools/dns','arguments':{'name':'example.org','type':'A'}} - first=await call(app,'tool.run',args,key=key,subject=uid,certs=(cert.resource_id,),rid='tool-once') - second=await call(app,'tool.run',args,key=key,subject=uid,certs=(cert.resource_id,),rid='tool-once') - assert first.status=='accepted' and second.replayed,wire(first) - assert first.data['job_id']==second.data['job_id'] - executions=[] - async def runner(tool,args,policy,outdir): +async def test_only_scoped_tool_is_visible_jobs_are_deduped_and_output_transfers( + installed, tmp_path +): + app, root = installed + key, uid, base = await register(app, 'tool-agent') + denied = await call( + app, + 'tool.run', + {'id': '/tools/dns', 'arguments': {'name': 'example.org', 'type': 'A'}}, + key=key, + subject=uid, + ) + assert denied.error.code == 'tool_certificate_required', wire(denied) + cap = scoped(app, 'tool.use', 'tool_dns', app.registry.capability('tool.use').operations) + cert = await approve(app, root, uid, key, (cap,)) + tools = await call( + app, 'discovery.get', {'id': '/tools'}, key=key, subject=uid, certs=(cert.resource_id,) + ) + assert [t['name'] for t in tools.data['items']] == ['dns'] + wrong = await call( + app, + 'tool.run', + {'id': '/tools/curl', 'arguments': {'url': 'https://example.org'}}, + key=key, + subject=uid, + certs=(cert.resource_id,), + ) + assert wrong.error.code == 'tool_certificate_required' + args = {'id': '/tools/dns', 'arguments': {'name': 'example.org', 'type': 'A'}} + first = await call( + app, 'tool.run', args, key=key, subject=uid, certs=(cert.resource_id,), rid='tool-once' + ) + second = await call( + app, 'tool.run', args, key=key, subject=uid, certs=(cert.resource_id,), rid='tool-once' + ) + assert first.status == 'accepted' and second.replayed, wire(first) + assert first.data['job_id'] == second.data['job_id'] + executions = [] + + async def runner(tool, args, policy, outdir): executions.append(tool.executor_key) - path=outdir/'output.bin';path.write_bytes(canonical({'answers':['93.184.216.34']})) - return ToolResult(path=path,media_type='application/json',metadata={'kind':'dns'}) - worker=EffectWorker(app,tool_runner=runner) + path = outdir / 'output.bin' + path.write_bytes(canonical({'answers': ['93.184.216.34']})) + return ToolResult(path=path, media_type='application/json', metadata={'kind': 'dns'}) + + worker = EffectWorker(app, tool_runner=runner) assert await worker.run_once() assert not await worker.run_once() - assert executions==['dns'] - state=await call(app,'job.get',{'id':first.data['job_id']},key=key,subject=uid,certs=(cert.resource_id,)) - assert state.data['state']=='done',wire(state) - ref=state.data['output'] - download=await call(app,'transfer.open',{'direction':'download','target':ref},key=key,subject=uid,certs=(cert.resource_id,)) - assert download.status=='ok',wire(download) + assert executions == ['dns'] + state = await call( + app, + 'job.get', + {'id': first.data['job_id']}, + key=key, + subject=uid, + certs=(cert.resource_id,), + ) + assert state.data['state'] == 'done', wire(state) + ref = state.data['output'] + download = await call( + app, + 'transfer.open', + {'direction': 'download', 'target': ref}, + key=key, + subject=uid, + certs=(cert.resource_id,), + ) + assert download.status == 'ok', wire(download) @pytest.mark.asyncio -async def test_pending_job_rechecks_revocation_and_expired_lease_is_uncertain(installed,tmp_path): - app,root=installed - key,uid,base=await register(app,'worker-agent') - cert=await approve(app,root,uid,key,(scoped(app,'tool.use','tool_dns',app.registry.capability('tool.use').operations),)) - result=await call(app,'tool.run',{'id':'/tools/dns','arguments':{'name':'example.org','type':'A'}},key=key,subject=uid,certs=(cert.resource_id,)) +async def test_pending_job_rechecks_revocation_and_expired_lease_is_uncertain(installed, tmp_path): + app, root = installed + key, uid, base = await register(app, 'worker-agent') + cert = await approve( + app, + root, + uid, + key, + (scoped(app, 'tool.use', 'tool_dns', app.registry.capability('tool.use').operations),), + ) + result = await call( + app, + 'tool.run', + {'id': '/tools/dns', 'arguments': {'name': 'example.org', 'type': 'A'}}, + key=key, + subject=uid, + certs=(cert.resource_id,), + ) async with app.metadata.transaction(write=True) as tx: - tx.execute('UPDATE certificates SET revoked=1 WHERE id=?',(cert.resource_id,),write=True) - ran=[] - async def runner(*args):ran.append(True) - worker=EffectWorker(app,tool_runner=runner) + tx.execute('UPDATE certificates SET revoked=1 WHERE id=?', (cert.resource_id,), write=True) + ran = [] + + async def runner(*args): + ran.append(True) + + worker = EffectWorker(app, tool_runner=runner) await worker.run_once() assert not ran async with app.metadata.transaction(write=False) as tx: - job=await tx.job(result.data['job_id']) - assert job.state=='failed' - cert2=await approve(app,root,uid,key,(scoped(app,'tool.use','tool_dns',app.registry.capability('tool.use').operations),)) - job_result=await call(app,'tool.run',{'id':'/tools/dns','arguments':{'name':'example.org','type':'A'}},key=key,subject=uid,certs=(cert2.resource_id,)) + job = await tx.job(result.data['job_id']) + assert job.state == 'failed' + cert2 = await approve( + app, + root, + uid, + key, + (scoped(app, 'tool.use', 'tool_dns', app.registry.capability('tool.use').operations),), + ) + job_result = await call( + app, + 'tool.run', + {'id': '/tools/dns', 'arguments': {'name': 'example.org', 'type': 'A'}}, + key=key, + subject=uid, + certs=(cert2.resource_id,), + ) async with app.metadata.transaction(write=True) as tx: - job=await tx.job(job_result.data['job_id']) - await tx.save_job(replace(job,state='running',lease_until=NOW-timedelta(seconds=1))) + job = await tx.job(job_result.data['job_id']) + await tx.save_job(replace(job, state='running', lease_until=NOW - timedelta(seconds=1))) await worker.run_once() async with app.metadata.transaction(write=False) as tx: - assert (await tx.job(job.id)).state=='uncertain' + assert (await tx.job(job.id)).state == 'uncertain' assert not ran + @pytest.mark.asyncio async def test_tool_external_success_then_revocation_records_uncertain(installed): - app,root=installed - key,uid,_=await register(app,'late-revocation') - cert=await approve(app,root,uid,key,(scoped(app,'tool.use','tool_dns',app.registry.capability('tool.use').operations),)) - result=await call(app,'tool.run',{'id':'/tools/dns','arguments':{'name':'example.org','type':'A'}},key=key,subject=uid,certs=(cert.resource_id,)) - async def runner(tool,args,policies,outdir): + app, root = installed + key, uid, _ = await register(app, 'late-revocation') + cert = await approve( + app, + root, + uid, + key, + (scoped(app, 'tool.use', 'tool_dns', app.registry.capability('tool.use').operations),), + ) + result = await call( + app, + 'tool.run', + {'id': '/tools/dns', 'arguments': {'name': 'example.org', 'type': 'A'}}, + key=key, + subject=uid, + certs=(cert.resource_id,), + ) + + async def runner(tool, args, policies, outdir): async with app.metadata.transaction(write=True) as tx: - tx.execute('UPDATE certificates SET revoked=1 WHERE id=?',(cert.resource_id,),write=True) - path=outdir/'result.bin';path.write_bytes(b'[]') - return ToolResult(path=path,media_type='application/json',metadata={}) - await EffectWorker(app,tool_runner=runner).run_once() + tx.execute( + 'UPDATE certificates SET revoked=1 WHERE id=?', (cert.resource_id,), write=True + ) + path = outdir / 'result.bin' + path.write_bytes(b'[]') + return ToolResult(path=path, media_type='application/json', metadata={}) + + await EffectWorker(app, tool_runner=runner).run_once() async with app.metadata.transaction(write=False) as tx: - assert (await tx.job(result.data['job_id'])).state=='uncertain' - - -async def _claimed_dns_job(app,root,handle): - key,uid,_=await register(app,handle) - cert=await approve(app,root,uid,key,(scoped(app,'tool.use','tool_dns',app.registry.capability('tool.use').operations),)) - result=await call(app,'tool.run',{'id':'/tools/dns','arguments':{'name':'example.org','type':'A'}}, - key=key,subject=uid,certs=(cert.resource_id,)) - assert result.status=='accepted',wire(result) + assert (await tx.job(result.data['job_id'])).state == 'uncertain' + + +async def _claimed_dns_job(app, root, handle): + key, uid, _ = await register(app, handle) + cert = await approve( + app, + root, + uid, + key, + (scoped(app, 'tool.use', 'tool_dns', app.registry.capability('tool.use').operations),), + ) + result = await call( + app, + 'tool.run', + {'id': '/tools/dns', 'arguments': {'name': 'example.org', 'type': 'A'}}, + key=key, + subject=uid, + certs=(cert.resource_id,), + ) + assert result.status == 'accepted', wire(result) async with app.metadata.transaction(write=False) as tx: return await tx.job(result.data['job_id']) -async def _enqueue_like(app,template,seconds_ago,**changes): - job_id=new_id('job') - job=replace(template,id=job_id,dedupe_key='test:'+job_id,state='pending',attempts=0,lease_until=None, - next_attempt_at=NOW-timedelta(seconds=seconds_ago),**changes) +async def _enqueue_like(app, template, seconds_ago, **changes): + job_id = new_id('job') + job = replace( + template, + id=job_id, + dedupe_key='test:' + job_id, + state='pending', + attempts=0, + lease_until=None, + next_attempt_at=NOW - timedelta(seconds=seconds_ago), + **changes, + ) async with app.metadata.transaction(write=True) as tx: await tx.enqueue(job) return job -async def _tool_ref(app,rid): +async def _tool_ref(app, rid): async with app.metadata.transaction(write=False) as tx: - return wire(ResourceRef(id=rid,revision=(await tx.resource(rid)).revision)) + return wire(ResourceRef(id=rid, revision=(await tx.resource(rid)).revision)) -async def _dns_revision_declaring(app,concurrency): +async def _dns_revision_declaring(app, concurrency): async with app.metadata.transaction(write=True) as tx: - resource=await tx.resource('tool_dns') - blob=await app.contents.put_bytes(canonical({**descriptor('dns'),'concurrency':concurrency}),'application/json') - revision=Revision(format_version=1,id=new_id('v'),resource_id=resource.id,parents=(resource.revision,), - content=blob,relations=(),actor=resource.owner,subject=resource.owner,author=resource.owner, - created_at=app.clock(),manifest_digest='') - revision=replace(revision,manifest_digest=digest({k:v for k,v in wire(revision).items() - if k not in {'manifest_digest','signature'}})) - await app.contents.pin(blob,revision.id) - await app.contents.commit_revision(resource.parent,revision) + resource = await tx.resource('tool_dns') + blob = await app.contents.put_bytes( + canonical({**descriptor('dns'), 'concurrency': concurrency}), 'application/json' + ) + revision = Revision( + format_version=1, + id=new_id('v'), + resource_id=resource.id, + parents=(resource.revision,), + content=blob, + relations=(), + actor=resource.owner, + subject=resource.owner, + author=resource.owner, + created_at=app.clock(), + manifest_digest='', + ) + revision = replace( + revision, + manifest_digest=digest({ + k: v for k, v in wire(revision).items() if k not in {'manifest_digest', 'signature'} + }), + ) + await app.contents.pin(blob, revision.id) + await app.contents.commit_revision(resource.parent, revision) await tx.append_revision(revision) - return wire(ResourceRef(id=resource.id,revision=revision.id)) + return wire(ResourceRef(id=resource.id, revision=revision.id)) -async def _state(app,job_id): +async def _state(app, job_id): async with app.metadata.transaction(write=False) as tx: return (await tx.job(job_id)).state @pytest.mark.asyncio async def test_historical_tool_revision_bytes_have_no_concurrency_and_mean_one(installed): - app,_=installed + app, _ = installed assert 'concurrency' not in descriptor('dns') async with app.metadata.transaction(write=False) as tx: - rev=await tx.revision(ResourceRef(id='tool_dns')) - stored=await app.contents.read_bytes(rev.content,limit=65536) - assert stored==canonical(descriptor('dns')) + rev = await tx.revision(ResourceRef(id='tool_dns')) + stored = await app.contents.read_bytes(rev.content, limit=65536) + assert stored == canonical(descriptor('dns')) assert 'concurrency' not in loads(stored) @pytest.mark.asyncio async def test_same_tool_is_not_claimed_by_a_second_worker_while_other_jobs_proceed(installed): - app,root=installed - held=await _claimed_dns_job(app,root,'concurrency-holder') - first=EffectWorker(app,tool_runner=lambda *a:None) - claimed,execute=await first._claim() - assert execute and claimed.id==held.id - same_tool=await _enqueue_like(app,held,3) - other_tool=await _enqueue_like(app,held,2,arguments={**held.arguments,'tool':await _tool_ref(app,'tool_curl')}) - non_tool=await _enqueue_like(app,held,1,kind='mail',operation='communication.send', - arguments={'recipient_subject':held.principal.subject}) - peer=await Application(app.settings,clock=lambda:NOW).load() + app, root = installed + held = await _claimed_dns_job(app, root, 'concurrency-holder') + first = EffectWorker(app, tool_runner=lambda *a: None) + claimed, execute = await first._claim() + assert execute and claimed.id == held.id + same_tool = await _enqueue_like(app, held, 3) + other_tool = await _enqueue_like( + app, held, 2, arguments={**held.arguments, 'tool': await _tool_ref(app, 'tool_curl')} + ) + non_tool = await _enqueue_like( + app, + held, + 1, + kind='mail', + operation='communication.send', + arguments={'recipient_subject': held.principal.subject}, + ) + peer = await Application(app.settings, clock=lambda: NOW).load() try: - second=EffectWorker(peer,tool_runner=lambda *a:None) - order=[] + second = EffectWorker(peer, tool_runner=lambda *a: None) + order = [] for _ in range(2): - job,execute=await second._claim() + job, execute = await second._claim() assert execute order.append(job.id) - assert order==[other_tool.id,non_tool.id] - assert await second._claim()==(None,False) - assert await _state(app,same_tool.id)=='pending' - await first._finish(claimed,'done','ok') - job,execute=await second._claim() - assert execute and job.id==same_tool.id + assert order == [other_tool.id, non_tool.id] + assert await second._claim() == (None, False) + assert await _state(app, same_tool.id) == 'pending' + await first._finish(claimed, 'done', 'ok') + job, execute = await second._claim() + assert execute and job.id == same_tool.id finally: await peer.close() @pytest.mark.asyncio async def test_explicit_tool_concurrency_admits_up_to_the_declared_limit(installed): - app,root=installed - template=await _claimed_dns_job(app,root,'concurrency-two') - ref=await _dns_revision_declaring(app,2) + app, root = installed + template = await _claimed_dns_job(app, root, 'concurrency-two') + ref = await _dns_revision_declaring(app, 2) async with app.metadata.transaction(write=False) as tx: - assert (await read_tool(app,tx,'tool_dns',ref['revision'])).resource.revision==ref['revision'] + assert (await read_tool(app, tx, 'tool_dns', ref['revision'])).resource.revision == ref[ + 'revision' + ] async with app.metadata.transaction(write=True) as tx: - await tx.save_job(replace(template,state='done')) - jobs=[await _enqueue_like(app,template,3-i,arguments={**template.arguments,'tool':ref}) for i in range(3)] - peer=await Application(app.settings,clock=lambda:NOW).load() + await tx.save_job(replace(template, state='done')) + jobs = [ + await _enqueue_like(app, template, 3 - i, arguments={**template.arguments, 'tool': ref}) + for i in range(3) + ] + peer = await Application(app.settings, clock=lambda: NOW).load() try: - workers=(EffectWorker(app,tool_runner=lambda *a:None),EffectWorker(peer,tool_runner=lambda *a:None)) - for worker,job in zip(workers,jobs): - claimed,execute=await worker._claim() - assert execute and claimed.id==job.id + workers = ( + EffectWorker(app, tool_runner=lambda *a: None), + EffectWorker(peer, tool_runner=lambda *a: None), + ) + for worker, job in zip(workers, jobs, strict=False): + claimed, execute = await worker._claim() + assert execute and claimed.id == job.id for worker in workers: - assert await worker._claim()==(None,False) - assert await _state(app,jobs[2].id)=='pending' + assert await worker._claim() == (None, False) + assert await _state(app, jobs[2].id) == 'pending' finally: await peer.close() @pytest.mark.asyncio async def test_historical_limit_counts_running_jobs_across_revisions_of_one_tool(installed): - app,root=installed - template=await _claimed_dns_job(app,root,'concurrency-mixed') - ref=await _dns_revision_declaring(app,2) - worker=EffectWorker(app,tool_runner=lambda *a:None) - claimed,_=await worker._claim() - assert claimed.id==template.id - historical=await _enqueue_like(app,template,2) - declared=await _enqueue_like(app,template,1,arguments={**template.arguments,'tool':ref}) - job,execute=await worker._claim() - assert execute and job.id==declared.id - assert await worker._claim()==(None,False) - assert await _state(app,historical.id)=='pending' + app, root = installed + template = await _claimed_dns_job(app, root, 'concurrency-mixed') + ref = await _dns_revision_declaring(app, 2) + worker = EffectWorker(app, tool_runner=lambda *a: None) + claimed, _ = await worker._claim() + assert claimed.id == template.id + historical = await _enqueue_like(app, template, 2) + declared = await _enqueue_like(app, template, 1, arguments={**template.arguments, 'tool': ref}) + job, execute = await worker._claim() + assert execute and job.id == declared.id + assert await worker._claim() == (None, False) + assert await _state(app, historical.id) == 'pending' @pytest.mark.asyncio -@pytest.mark.parametrize('declared',[0,33,-1,True,'2',1.5,None]) -async def test_invalid_declared_tool_concurrency_is_rejected_not_unbounded(installed,declared): - app,root=installed - template=await _claimed_dns_job(app,root,'concurrency-invalid') - ref=await _dns_revision_declaring(app,declared) +@pytest.mark.parametrize('declared', [0, 33, -1, True, '2', 1.5, None]) +async def test_invalid_declared_tool_concurrency_is_rejected_not_unbounded(installed, declared): + app, root = installed + template = await _claimed_dns_job(app, root, 'concurrency-invalid') + ref = await _dns_revision_declaring(app, declared) async with app.metadata.transaction(write=False) as tx: - with pytest.raises(Failure,match='invalid_tool_concurrency'): - await read_tool(app,tx,'tool_dns',ref['revision']) + with pytest.raises(Failure, match='invalid_tool_concurrency'): + await read_tool(app, tx, 'tool_dns', ref['revision']) async with app.metadata.transaction(write=True) as tx: - await tx.save_job(replace(template,state='done')) - bad=await _enqueue_like(app,template,1,arguments={**template.arguments,'tool':ref}) - ran=[] - async def runner(*args):ran.append(args) - worker=EffectWorker(app,tool_runner=runner) - job,execute=await worker._claim() - assert job.id==bad.id and not execute - assert await _state(app,bad.id)=='failed' and not ran + await tx.save_job(replace(template, state='done')) + bad = await _enqueue_like(app, template, 1, arguments={**template.arguments, 'tool': ref}) + ran = [] + + async def runner(*args): + ran.append(args) + + worker = EffectWorker(app, tool_runner=runner) + job, execute = await worker._claim() + assert job.id == bad.id and not execute + assert await _state(app, bad.id) == 'failed' and not ran async with app.metadata.transaction(write=False) as tx: - assert tx.setting('job_status:'+bad.id)=={'code':'invalid_tool_concurrency'} - assert await worker._claim()==(None,False) + assert tx.setting('job_status:' + bad.id) == {'code': 'invalid_tool_concurrency'} + assert await worker._claim() == (None, False) @pytest.mark.asyncio @@ -285,8 +462,9 @@ async def test_tool_run_is_the_only_public_operation_and_old_short_code_is_retir assert descriptor('dns')['operation'] == 'tool.run' async with app.metadata.transaction(write=False) as tx: assert (await read_tool(app, tx, 'tool_dns')).operation == 'tool.run' - async with httpx.AsyncClient(transport=httpx.ASGITransport(app=create_app(app)), - base_url='http://testserver') as http: + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=create_app(app)), base_url='http://testserver' + ) as http: current = await http.get('/-/d/tool.run') assert current.status_code == 200 assert current.json()['operations'][0]['code'] == new_code diff --git a/tests/test_tools_directory.py b/tests/test_tools_directory.py index 7a2f9d53..e9bd4c0f 100644 --- a/tests/test_tools_directory.py +++ b/tests/test_tools_directory.py @@ -1,4 +1,5 @@ """Canonical /tools path and the certificate boundary on a fresh install.""" + from dataclasses import replace import pytest @@ -29,11 +30,11 @@ async def test_new_install_tools_path_keeps_permissions_and_capability_gate(inst assert denied.error.code == 'tool_certificate_required', wire(denied) cap = scoped(app, 'tool.use', 'tool_dns', app.registry.capability('tool.use').operations) cert = await approve(app, root, uid, key, (cap,)) - listing = await call(app, 'discovery.get', {'id': '/tools'}, key=key, subject=uid, - certs=(cert.resource_id,)) + listing = await call( + app, 'discovery.get', {'id': '/tools'}, key=key, subject=uid, certs=(cert.resource_id,) + ) assert [tool['name'] for tool in listing.data['items']] == ['dns'] - accepted = await call(app, 'tool.run', args, key=key, subject=uid, - certs=(cert.resource_id,)) + accepted = await call(app, 'tool.run', args, key=key, subject=uid, certs=(cert.resource_id,)) assert accepted.status == 'accepted', wire(accepted) @@ -42,20 +43,22 @@ async def test_legacy_tools_path_is_read_only_and_new_alias_still_checks_certifi app, root = installed async with app.metadata.transaction(write=True) as tx: current = await tx.resource('t_tools') - await tx.replace(replace(current, name='_tools', generation=current.generation + 1), - current.generation) + await tx.replace( + replace(current, name='_tools', generation=current.generation + 1), current.generation + ) await bootstrap(app.metadata, app.contents, app.registry, NOW) key, uid, _ = await register(app, 'legacy-tools-directory') cap = scoped(app, 'tool.use', 'tool_dns', app.registry.capability('tool.use').operations) cert = await approve(app, root, uid, key, (cap,)) - legacy_read = await call(app, 'discovery.get', {'id': '/_tools'}, key=key, subject=uid, - certs=(cert.resource_id,)) + legacy_read = await call( + app, 'discovery.get', {'id': '/_tools'}, key=key, subject=uid, certs=(cert.resource_id,) + ) assert [tool['name'] for tool in legacy_read.data['items']] == ['dns'] args = {'id': '/_tools/dns', 'arguments': {'name': 'example.org', 'type': 'A'}} - rejected = await call(app, 'tool.run', args, key=key, subject=uid, - certs=(cert.resource_id,)) + rejected = await call(app, 'tool.run', args, key=key, subject=uid, certs=(cert.resource_id,)) assert rejected.error.code == 'legacy_tool_path_read_only', wire(rejected) canonical = {**args, 'id': '/tools/dns'} - allowed = await call(app, 'tool.run', canonical, key=key, subject=uid, - certs=(cert.resource_id,)) + allowed = await call( + app, 'tool.run', canonical, key=key, subject=uid, certs=(cert.resource_id,) + ) assert allowed.status == 'accepted', wire(allowed) diff --git a/tests/test_topic_governance.py b/tests/test_topic_governance.py index 2e6a7df4..9e6092fd 100644 --- a/tests/test_topic_governance.py +++ b/tests/test_topic_governance.py @@ -1,14 +1,14 @@ """Topic roles, bans and virtual event history share the write transaction.""" + from datetime import timedelta import httpx import pytest +from test_service import NOW, call, register -from msg.core.codec import wire -from msg.core.codec import b64, canonical +from msg.core.codec import b64, canonical, wire from msg.core.requests import request_for from msg.transports.http import create_app -from test_service import NOW, call, register @pytest.mark.asyncio @@ -16,43 +16,74 @@ async def test_topic_open_membership_ban_and_last_admin(installed): app, _ = installed owner_key, owner, _ = await register(app, 'topic-owner') member_key, member, _ = await register(app, 'topic-member') - created = await call(app, 'content.topic_create', {'parent': '/main', 'name': 'governed'}, - key=owner_key, subject=owner) + created = await call( + app, + 'content.topic_create', + {'parent': '/main', 'name': 'governed'}, + key=owner_key, + subject=owner, + ) assert created.status == 'ok', wire(created) topic = created.resources[0].id async with app.metadata.transaction(write=False) as tx: resource = await tx.resource(topic) assert resource.created_by == owner - assert tx.one('SELECT role,status FROM topic_memberships WHERE topic=? AND subject=?', - (topic, owner)) == ('admin', 'active') + assert tx.one( + 'SELECT role,status FROM topic_memberships WHERE topic=? AND subject=?', (topic, owner) + ) == ('admin', 'active') joined = await call(app, 'content.topic_join', {'id': topic}, key=member_key, subject=member) assert joined.status == 'ok' and joined.data['status'] == 'active', wire(joined) refused = await call(app, 'content.topic_leave', {'id': topic}, key=owner_key, subject=owner) assert refused.status == 'error' and refused.error.code == 'last_topic_admin' async with app.metadata.transaction(write=False) as tx: - after_failed_leave=tx.one('SELECT COUNT(*) FROM events')[0] - demote = await call(app, 'content.topic_demote', {'id': topic, 'subject_id': owner}, - key=owner_key, subject=owner) + after_failed_leave = tx.one('SELECT COUNT(*) FROM events')[0] + demote = await call( + app, + 'content.topic_demote', + {'id': topic, 'subject_id': owner}, + key=owner_key, + subject=owner, + ) assert demote.status == 'error' and demote.error.code == 'last_topic_admin' async with app.metadata.transaction(write=False) as tx: assert tx.one('SELECT COUNT(*) FROM events')[0] == after_failed_leave - ban = await call(app, 'content.topic_ban', {'id': topic, 'subject_id': member, - 'reason': 'moderation detail'}, key=owner_key, subject=owner) + ban = await call( + app, + 'content.topic_ban', + {'id': topic, 'subject_id': member, 'reason': 'moderation detail'}, + key=owner_key, + subject=owner, + ) assert ban.status == 'ok', wire(ban) - blocked = await call(app, 'content.post_create', {'parent': topic, 'body': 'should fail'}, - key=member_key, subject=member) + blocked = await call( + app, + 'content.post_create', + {'parent': topic, 'body': 'should fail'}, + key=member_key, + subject=member, + ) assert blocked.status == 'error' and blocked.error.code == 'topic_banned' - unban = await call(app, 'content.topic_unban', {'id': topic, 'subject_id': member}, - key=owner_key, subject=owner) + unban = await call( + app, + 'content.topic_unban', + {'id': topic, 'subject_id': member}, + key=owner_key, + subject=owner, + ) assert unban.status == 'ok' async with app.metadata.transaction(write=False) as tx: - assert tx.one('SELECT status FROM topic_memberships WHERE topic=? AND subject=?', - (topic, member))[0] != 'active' + assert ( + tx.one( + 'SELECT status FROM topic_memberships WHERE topic=? AND subject=?', (topic, member) + )[0] + != 'active' + ) assert (await tx.resource(topic)).created_by == owner rejoined = await call(app, 'content.topic_join', {'id': topic}, key=member_key, subject=member) assert rejoined.status == 'ok' - no_system_power = await call(app, 'discovery.get', {'id': '/private'}, - key=owner_key, subject=owner) + no_system_power = await call( + app, 'discovery.get', {'id': '/private'}, key=owner_key, subject=owner + ) assert no_system_power.status == 'error' @@ -62,43 +93,88 @@ async def test_topic_policy_approval_invite_closed_and_role_transfer(installed): owner_key, owner, _ = await register(app, 'policy-owner') alice_key, alice, _ = await register(app, 'policy-alice') bob_key, bob, _ = await register(app, 'policy-bob') - topic = (await call(app, 'content.topic_create', {'parent': '/main', 'name': 'policy-topic'}, - key=owner_key, subject=owner)).resources[0].id - approval = await call(app, 'content.topic_policy_set', {'id': topic, 'membership_policy': 'approval'}, - key=owner_key, subject=owner) + topic = ( + ( + await call( + app, + 'content.topic_create', + {'parent': '/main', 'name': 'policy-topic'}, + key=owner_key, + subject=owner, + ) + ) + .resources[0] + .id + ) + approval = await call( + app, + 'content.topic_policy_set', + {'id': topic, 'membership_policy': 'approval'}, + key=owner_key, + subject=owner, + ) assert approval.status == 'ok' pending = await call(app, 'content.topic_join', {'id': topic}, key=alice_key, subject=alice) assert pending.data['status'] == 'pending' - approved = await call(app, 'content.topic_approve', {'id': topic, 'subject_id': alice}, - key=owner_key, subject=owner) + approved = await call( + app, + 'content.topic_approve', + {'id': topic, 'subject_id': alice}, + key=owner_key, + subject=owner, + ) assert approved.data['status'] == 'active' member_sync = await call(app, 'communication.changes', {}, key=alice_key, subject=alice) - assert any(item['type'].startswith('topic.') and item['data'].get('topic_id') == topic - for item in member_sync.data['items']) - assert sum(item['request_id']==approved.request_id for item in member_sync.data['items'])==1 - promoted = await call(app, 'content.topic_promote', {'id': topic, 'subject_id': alice}, - key=owner_key, subject=owner) + assert any( + item['type'].startswith('topic.') and item['data'].get('topic_id') == topic + for item in member_sync.data['items'] + ) + assert sum(item['request_id'] == approved.request_id for item in member_sync.data['items']) == 1 + promoted = await call( + app, + 'content.topic_promote', + {'id': topic, 'subject_id': alice}, + key=owner_key, + subject=owner, + ) assert promoted.data['role'] == 'admin' left = await call(app, 'content.topic_leave', {'id': topic}, key=owner_key, subject=owner) assert left.status == 'ok' - changed = await call(app, 'content.topic_policy_set', {'id': topic, 'membership_policy': 'invite'}, - key=alice_key, subject=alice) + changed = await call( + app, + 'content.topic_policy_set', + {'id': topic, 'membership_policy': 'invite'}, + key=alice_key, + subject=alice, + ) assert changed.status == 'ok' denied = await call(app, 'content.topic_join', {'id': topic}, key=bob_key, subject=bob) assert denied.status == 'error' - invited = await call(app, 'content.topic_invite', {'id': topic, 'subject_id': bob}, - key=alice_key, subject=alice) + invited = await call( + app, 'content.topic_invite', {'id': topic, 'subject_id': bob}, key=alice_key, subject=alice + ) assert invited.status == 'ok' joined = await call(app, 'content.topic_join', {'id': topic}, key=bob_key, subject=bob) assert joined.status == 'ok' - closed = await call(app, 'content.topic_policy_set', {'id': topic, 'membership_policy': 'closed'}, - key=alice_key, subject=alice) + closed = await call( + app, + 'content.topic_policy_set', + {'id': topic, 'membership_policy': 'closed'}, + key=alice_key, + subject=alice, + ) assert closed.status == 'ok' - reopened = await call(app, 'content.topic_policy_set', {'id': topic, 'membership_policy': 'open'}, - key=alice_key, subject=alice) + reopened = await call( + app, + 'content.topic_policy_set', + {'id': topic, 'membership_policy': 'open'}, + key=alice_key, + subject=alice, + ) assert reopened.status == 'ok' - removed = await call(app, 'content.topic_remove', {'id': topic, 'subject_id': bob}, - key=alice_key, subject=alice) + removed = await call( + app, 'content.topic_remove', {'id': topic, 'subject_id': bob}, key=alice_key, subject=alice + ) assert removed.data['status'] == 'removed' rejoined = await call(app, 'content.topic_join', {'id': topic}, key=bob_key, subject=bob) assert rejoined.data['status'] == 'active' @@ -109,20 +185,45 @@ async def test_expired_ban_stops_blocking_without_restoring_membership(installed app, _ = installed owner_key, owner, _ = await register(app, 'expiry-owner') member_key, member, _ = await register(app, 'expiry-member') - topic = (await call(app, 'content.topic_create', {'parent': '/main', 'name': 'expiry-topic'}, - key=owner_key, subject=owner)).resources[0].id + topic = ( + ( + await call( + app, + 'content.topic_create', + {'parent': '/main', 'name': 'expiry-topic'}, + key=owner_key, + subject=owner, + ) + ) + .resources[0] + .id + ) await call(app, 'content.topic_join', {'id': topic}, key=member_key, subject=member) - banned = await call(app, 'content.topic_ban', {'id': topic, 'subject_id': member, - 'expires_at': wire(NOW + timedelta(seconds=60))}, key=owner_key, subject=owner) + banned = await call( + app, + 'content.topic_ban', + {'id': topic, 'subject_id': member, 'expires_at': wire(NOW + timedelta(seconds=60))}, + key=owner_key, + subject=owner, + ) assert banned.status == 'ok' app.executor.clock = lambda: NOW + timedelta(seconds=61) async with app.metadata.transaction(write=False) as tx: - assert tx.one('SELECT status FROM topic_memberships WHERE topic=? AND subject=?', - (topic, member))[0] != 'active' + assert ( + tx.one( + 'SELECT status FROM topic_memberships WHERE topic=? AND subject=?', (topic, member) + )[0] + != 'active' + ) joined = await call(app, 'content.topic_join', {'id': topic}, key=member_key, subject=member) assert joined.status == 'ok' - post = await call(app, 'content.post_create', {'parent': topic, 'body': 'after expiry'}, - key=member_key, subject=member) + post = await call( + app, + 'content.post_create', + {'parent': topic, 'body': 'after expiry'}, + key=member_key, + subject=member, + ) assert post.status == 'ok' @@ -131,68 +232,113 @@ async def test_virtual_events_compact_are_read_only_and_reason_is_admin_only(ins app, _ = installed owner_key, owner, _ = await register(app, 'events-owner') member_key, member, _ = await register(app, 'events-member') - topic = (await call(app, 'content.topic_create', {'parent': '/main', 'name': 'events-topic'}, - key=owner_key, subject=owner)).resources[0].id + topic = ( + ( + await call( + app, + 'content.topic_create', + {'parent': '/main', 'name': 'events-topic'}, + key=owner_key, + subject=owner, + ) + ) + .resources[0] + .id + ) await call(app, 'content.topic_join', {'id': topic}, key=member_key, subject=member) watched = await call(app, 'communication.watch', {'id': topic}, key=member_key, subject=member) assert watched.status == 'ok' before = None async with app.metadata.transaction(write=False) as tx: - before = (tx.one('SELECT COUNT(*) FROM resources WHERE parent=? AND type=?',(topic,'post'))[0], - tx.one('SELECT COUNT(*) FROM revisions')[0]) - await call(app, 'content.topic_ban', {'id': topic, 'subject_id': member, - 'reason': 'private moderation note'}, key=owner_key, subject=owner) + before = ( + tx.one('SELECT COUNT(*) FROM resources WHERE parent=? AND type=?', (topic, 'post'))[0], + tx.one('SELECT COUNT(*) FROM revisions')[0], + ) + await call( + app, + 'content.topic_ban', + {'id': topic, 'subject_id': member, 'reason': 'private moderation note'}, + key=owner_key, + subject=owner, + ) compact = await call(app, 'content.topic_events', {'id': topic}, key=owner_key, subject=owner) assert compact.status == 'ok' and compact.data['view'] == 'compact' assert len(compact.data['items']) <= 10 and compact.data['items'] - normal = await call(app, 'content.topic_events', {'id': topic, 'view': 'normal'}, - key=owner_key, subject=owner) + normal = await call( + app, 'content.topic_events', {'id': topic, 'view': 'normal'}, key=owner_key, subject=owner + ) assert any('private moderation note' in str(item) for item in normal.data['items']) - outsider = await call(app, 'content.topic_events', {'id': topic, 'view': 'normal'}, - key=member_key, subject=member) + outsider = await call( + app, 'content.topic_events', {'id': topic, 'view': 'normal'}, key=member_key, subject=member + ) assert outsider.status == 'error' or 'private moderation note' not in str(outsider.data) changes = await call(app, 'communication.changes', {}, key=member_key, subject=member) assert changes.status == 'ok' and 'private moderation note' not in str(changes.data) inbox = await call(app, 'communication.inbox', {}, key=member_key, subject=member) - assert any(item.get('source') == 'topic_governance' and item.get('action') == 'member.ban' - for item in inbox.data['items']) + assert any( + item.get('source') == 'topic_governance' and item.get('action') == 'member.ban' + for item in inbox.data['items'] + ) for index in range(12): policy = 'approval' if index % 2 else 'open' - changed = await call(app, 'content.topic_policy_set', {'id': topic, - 'membership_policy': policy}, key=owner_key, subject=owner) + changed = await call( + app, + 'content.topic_policy_set', + {'id': topic, 'membership_policy': policy}, + key=owner_key, + subject=owner, + ) assert changed.status == 'ok' recent = await call(app, 'content.topic_events', {'id': topic}, key=owner_key, subject=owner) assert len(recent.data['items']) == 10 and recent.data['cursor'] async with app.metadata.transaction(write=False) as tx: - topic_path=await tx.path(topic) - async with httpx.AsyncClient(transport=httpx.ASGITransport(app=create_app(app)), - base_url='http://testserver') as http: - compact_http=await http.get(topic_path+'/_events.md') - compact_path=await http.get(topic_path+'/_events.md/compact/10') - assert compact_http.status_code==compact_path.status_code==200 - assert compact_http.content==compact_path.content - assert compact_http.headers['etag']==compact_path.headers['etag'] - assert len(compact_http.json()['items'])==10 - normal_query=await http.get(topic_path+'/_events.md?view=normal&limit=10') - normal_path=await http.get(topic_path+'/_events.md/normal/10') - assert normal_query.status_code==normal_path.status_code==200 - assert normal_query.content==normal_path.content - assert (await http.head(topic_path+'/_events.md')).status_code==200 - assert (await http.post(topic_path+'/_events.md')).status_code==405 - assert (await http.get(topic_path+'/_events.md',headers={ - 'If-None-Match':compact_http.headers['etag']})).status_code==304 - older = await call(app, 'content.topic_events', {'id': topic, - 'cursor': recent.data['cursor']}, key=owner_key, subject=owner) + topic_path = await tx.path(topic) + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=create_app(app)), base_url='http://testserver' + ) as http: + compact_http = await http.get(topic_path + '/_events.md') + compact_path = await http.get(topic_path + '/_events.md/compact/10') + assert compact_http.status_code == compact_path.status_code == 200 + assert compact_http.content == compact_path.content + assert compact_http.headers['etag'] == compact_path.headers['etag'] + assert len(compact_http.json()['items']) == 10 + normal_query = await http.get(topic_path + '/_events.md?view=normal&limit=10') + normal_path = await http.get(topic_path + '/_events.md/normal/10') + assert normal_query.status_code == normal_path.status_code == 200 + assert normal_query.content == normal_path.content + assert (await http.head(topic_path + '/_events.md')).status_code == 200 + assert (await http.post(topic_path + '/_events.md')).status_code == 405 + assert ( + await http.get( + topic_path + '/_events.md', headers={'If-None-Match': compact_http.headers['etag']} + ) + ).status_code == 304 + older = await call( + app, + 'content.topic_events', + {'id': topic, 'cursor': recent.data['cursor']}, + key=owner_key, + subject=owner, + ) assert older.status == 'ok' and older.data['items'] assert {item['c'] for item in recent.data['items']} <= set(recent.data['event_codes'].values()) async with app.metadata.transaction(write=False) as tx: - after = (tx.one('SELECT COUNT(*) FROM resources WHERE parent=? AND type=?',(topic,'post'))[0], - tx.one('SELECT COUNT(*) FROM revisions')[0]) - assert tx.one('SELECT id FROM resources WHERE parent=? AND name=?', - (topic, '_events.md')) is None + after = ( + tx.one('SELECT COUNT(*) FROM resources WHERE parent=? AND type=?', (topic, 'post'))[0], + tx.one('SELECT COUNT(*) FROM revisions')[0], + ) + assert ( + tx.one('SELECT id FROM resources WHERE parent=? AND name=?', (topic, '_events.md')) + is None + ) assert after == before - reserved = await call(app, 'content.post_create', {'parent': topic, 'name': '_events.md', - 'body': 'fake'}, key=owner_key, subject=owner) + reserved = await call( + app, + 'content.post_create', + {'parent': topic, 'name': '_events.md', 'body': 'fake'}, + key=owner_key, + subject=owner, + ) assert reserved.status == 'error' @@ -200,23 +346,41 @@ async def test_virtual_events_compact_are_read_only_and_reason_is_admin_only(ins async def test_virtual_events_private_topic_authorizes_before_etag(installed): app, _ = installed owner_key, owner, _ = await register(app, 'events-private-owner') - created=await call(app,'content.topic_create',{'parent':'/main','name':'private-events'}, - key=owner_key,subject=owner) - topic=created.resources[0].id - private=await call(app,'content.chmod',{'id':topic,'mode':'0700'},key=owner_key, - subject=owner,expected=((topic,created.data['generation']),)) - assert private.status=='ok',wire(private) + created = await call( + app, + 'content.topic_create', + {'parent': '/main', 'name': 'private-events'}, + key=owner_key, + subject=owner, + ) + topic = created.resources[0].id + private = await call( + app, + 'content.chmod', + {'id': topic, 'mode': '0700'}, + key=owner_key, + subject=owner, + expected=((topic, created.data['generation']),), + ) + assert private.status == 'ok', wire(private) async with app.metadata.transaction(write=False) as tx: - path=await tx.path(topic) - packet=request_for('content.topic_events',{'id':path,'view':'compact','limit':10}, - app.settings.service_url,subject=owner,signer=owner_key, - expires_at=NOW+timedelta(seconds=120)) - async with httpx.AsyncClient(transport=httpx.ASGITransport(app=create_app(app)), - base_url='http://testserver') as http: - for method in (http.get,http.head): - denied=await method(path+'/_events.md',headers={'If-None-Match':'*'}) - assert denied.status_code==403 and 'etag' not in denied.headers - allowed=await http.get(path+'/_events.md',headers={ - 'X-Msg-Request':b64(canonical(packet))}) - assert allowed.status_code==200,allowed.text - assert allowed.json()['topic_id']==topic + path = await tx.path(topic) + packet = request_for( + 'content.topic_events', + {'id': path, 'view': 'compact', 'limit': 10}, + app.settings.service_url, + subject=owner, + signer=owner_key, + expires_at=NOW + timedelta(seconds=120), + ) + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=create_app(app)), base_url='http://testserver' + ) as http: + for method in (http.get, http.head): + denied = await method(path + '/_events.md', headers={'If-None-Match': '*'}) + assert denied.status_code == 403 and 'etag' not in denied.headers + allowed = await http.get( + path + '/_events.md', headers={'X-Msg-Request': b64(canonical(packet))} + ) + assert allowed.status_code == 200, allowed.text + assert allowed.json()['topic_id'] == topic diff --git a/tests/test_transfer.py b/tests/test_transfer.py index 7b472536..08792710 100644 --- a/tests/test_transfer.py +++ b/tests/test_transfer.py @@ -1,82 +1,124 @@ """One persistent transfer state machine, independently of the adapter.""" -from dataclasses import replace -from datetime import timedelta -import hashlib + import pytest +from test_service import call, register -from msg.core.codec import b64,unb64,digest,wire -from msg.core.models import ResourceRef -from test_service import call,register,NOW +from msg.core.codec import b64, digest, unb64, wire @pytest.mark.asyncio async def test_upload_out_of_order_conflicts_seal_and_download(installed): - app,_=installed - key,uid,cert=await register(app,'transfer-user') - async def invoke(op,a): - return await call(app,op,a,key=key,subject=uid,certs=(cert,)) - data=b'plain text\n'+bytes(range(256))*7 - opened=await invoke('transfer.open',{'direction':'upload','size':len(data),'digest':digest(data), - 'requested_part_bytes':512,'media_type':'application/octet-stream'}) - assert opened.status=='ok',wire(opened) - tid=opened.data['transfer_id'] - size=opened.data['part_bytes'] - assert 0