From f87ea8c0f72a774d98d002cfdb5aa95e29979b22 Mon Sep 17 00:00:00 2001 From: LIghtJUNction Date: Mon, 28 Sep 2026 23:02:46 +0800 Subject: [PATCH 001/136] fix: replay an active topic ban without promoting recovery A restored database can omit a ban that the pinned log still requires. Record that ban locally, keep lift deny-only, and state why promotion stays blocked. Co-authored-by: Cursor --- src/msg/admin/recovery_replay.py | 31 +++++++++++++++++--- src/msg/data/recovery-checkpoint.schema.json | 1 + tests/test_task_a_recovery_checkpoint.py | 14 +++++++-- 3 files changed, 40 insertions(+), 6 deletions(-) diff --git a/src/msg/admin/recovery_replay.py b/src/msg/admin/recovery_replay.py index 0df26631..f0a61b1e 100644 --- a/src/msg/admin/recovery_replay.py +++ b/src/msg/admin/recovery_replay.py @@ -23,8 +23,13 @@ FORMAT = 'msg-revocation-checkpoint-v1' SUPPORTED_FACTS = frozenset({'credential.revoke', 'certificate.revoke', 'share_grant.revoke', 'share_grant_v2.revoke', 'share_link.revoke', 'membership.remove', - 'topic_membership.remove', 'topic_ban.lift', 'identity_key.retire', + 'topic_membership.remove', 'topic_ban.apply', 'topic_ban.lift', 'identity_key.retire', 'encryption_key.retire', 'vault.destroy'}) +PROMOTION_BLOCKED_REASONS = ( + 'quarantine_remains', + 'supported_facts_are_deny_only', + 'backup_retirement_not_attested', +) _SHARE_TABLES = {'share_grant.revoke': 'share_grants', 'share_grant_v2.revoke': 'share_grants_v2', 'share_link.revoke': 'share_links'} _MAX_ENTRIES = 10000 @@ -157,6 +162,21 @@ async def _apply(tx, fact): (target, subject), write=True) return True return False + if kind == 'topic_ban.apply': + require(tx.one('SELECT id FROM resources WHERE id=?', (target,)) is not None, + 'recovery_fact_missing') + row = tx.one('SELECT status FROM topic_bans WHERE topic=? AND subject=?', (target, subject)) + if row is None: + tx.execute('''INSERT INTO topic_bans + (topic,subject,actor,created_at,expires_at,reason,status) + VALUES (?,?,?,?,NULL,?,?)''', + (target, subject, ROOT_SUBJECT, at, 'recovery-replay', 'active'), write=True) + return True + if row[0] != 'active': + tx.execute("UPDATE topic_bans SET status='active' WHERE topic=? AND subject=?", + (target, subject), write=True) + return True + return False if kind == 'topic_ban.lift': row = tx.one('SELECT status FROM topic_bans WHERE topic=? AND subject=?', (target, subject)) require(row is not None, 'recovery_fact_missing') @@ -209,8 +229,9 @@ async def _replay(store, packet, *, pin, operator): Always re-apply facts, including after a repeated invocation/restart: cached watermarks are not evidence that an authorization row has not been restored. - This neither imports keys, emits effects, nor clears quarantine. topic_ban.lift - is the only topic-ban fact; it does not reconcile ACLs or promote production. + This neither imports keys, emits effects, nor clears quarantine. topic_ban.apply + only records an active ban, and topic_ban.lift only marks an existing ban lifted. + Neither reconciles ACLs nor promotes production. """ require(isinstance(operator, str) and bool(operator), 'recovery_operator_required') body = verify_checkpoint(packet, pin=pin) @@ -233,7 +254,9 @@ async def _replay(store, packet, *, pin, operator): changed += bool(await _apply(tx, fact)) receipt = {'format': FORMAT, 'checkpoint_digest': pin.digest, 'sequence': pin.sequence, 'prefix_digest': digest(body['entries']), 'scope': 'supported_deny_only_facts', - 'promotion': 'blocked', 'backup_retired': False} + 'promotion': 'blocked', + 'promotion_blocked_reasons': list(PROMOTION_BLOCKED_REASONS), + 'backup_retired': False} if changed: tx.set_setting('authorization_epoch', tx.setting('authorization_epoch', 0) + 1) if changed or previous != receipt: diff --git a/src/msg/data/recovery-checkpoint.schema.json b/src/msg/data/recovery-checkpoint.schema.json index a0b06413..a2726df4 100644 --- a/src/msg/data/recovery-checkpoint.schema.json +++ b/src/msg/data/recovery-checkpoint.schema.json @@ -67,6 +67,7 @@ "share_link.revoke", "membership.remove", "topic_membership.remove", + "topic_ban.apply", "topic_ban.lift", "identity_key.retire", "encryption_key.retire", diff --git a/tests/test_task_a_recovery_checkpoint.py b/tests/test_task_a_recovery_checkpoint.py index 4fe1c0c4..c5cba9ca 100644 --- a/tests/test_task_a_recovery_checkpoint.py +++ b/tests/test_task_a_recovery_checkpoint.py @@ -89,7 +89,10 @@ async def test_replay_is_atomic_monotonic_idempotent_and_never_promotes(restored body, pin, signed = checkpoint results = await asyncio.gather(*(replay(restored, signed(body), pin=pin) for _ in range(3))) assert sum(result['changed'] for result in results) == 1 - assert all(result['promotion'] == 'blocked' and result['backup_retired'] is False for result in results) + assert all(result['promotion'] == 'blocked' and result['backup_retired'] is False and + result['promotion_blocked_reasons'] == [ + 'quarantine_remains', 'supported_facts_are_deny_only', + 'backup_retirement_not_attested'] for result in results) async with restored.transaction(write=False) as tx: assert active(tx) credential = await tx.credential('token_one') @@ -286,10 +289,13 @@ async def test_all_supported_owned_revocations_preserve_signed_bytes_and_never_i tx.execute('''INSERT INTO topic_bans (topic,subject,actor,created_at,expires_at,reason,status) VALUES (?,?,?,?,NULL,?,?)''', (resource.id, owner, owner, wire(NOW), 'restored-ban', 'active'), write=True) + banned = replace(resource, id='t_ban', name='ban') + await tx.insert(banned) targets = [('credential.revoke', 'token_one'), ('certificate.revoke', 'cert_one'), ('share_grant.revoke', 'sg_one'), ('share_grant_v2.revoke', 'sg_two'), ('share_link.revoke', 'link_one'), ('membership.remove', 'g_test'), - ('topic_membership.remove', 't_test'), ('topic_ban.lift', resource.id), + ('topic_membership.remove', 't_test'), ('topic_ban.apply', 't_ban'), + ('topic_ban.lift', resource.id), ('identity_key.retire', signer.key_id), ('encryption_key.retire', age_id), ('vault.destroy', age_id)] assert {kind for kind, _ in targets} == SUPPORTED_FACTS @@ -299,6 +305,8 @@ async def test_all_supported_owned_revocations_preserve_signed_bytes_and_never_i pin = replace(pin, sequence=len(facts), digest=digest(body)) result = await replay(restored, signed(body), pin=pin) assert result['changed'] == len(facts) and result['backup_retired'] is False + assert result['promotion_blocked_reasons'] == [ + 'quarantine_remains', 'supported_facts_are_deny_only', 'backup_retirement_not_attested'] async with restored.transaction(write=False) as tx: assert tx.one('SELECT body,revoked FROM certificates WHERE id=?', ('cert_one',)) == (original, 1) assert (await tx.credential(signer.key_id)).revoked_at == NOW @@ -314,6 +322,8 @@ async def test_all_supported_owned_revocations_preserve_signed_bytes_and_never_i ('t_test', owner)) == ('removed',) assert tx.one('SELECT status FROM topic_bans WHERE topic=? AND subject=?', ('t_test', owner)) == ('lifted',) + assert tx.one('SELECT status,actor,reason FROM topic_bans WHERE topic=? AND subject=?', + ('t_ban', owner)) == ('active', 'u_root', 'recovery-replay') assert tx.one('SELECT status,signing_nonce,signing_ciphertext,age_nonce,age_ciphertext ' 'FROM custodial_vault WHERE subject=?', (owner,)) == ('destroyed', None, None, None, None) assert tx.one('SELECT count(*) FROM jobs')[0] == 0 and active(tx) From ed12fe897ac451bb9b021856ccffbf33cab0cf8c Mon Sep 17 00:00:00 2001 From: LIghtJUNction Date: Mon, 28 Sep 2026 23:33:09 +0800 Subject: [PATCH 002/136] test: nest the banned topic so the fixture keeps a single root resource The one_root index allows only one parentless resource, so a second root topic made the topic_ban.apply fixture fail with constraint_conflict on PostgreSQL. Co-authored-by: Cursor --- tests/test_task_a_recovery_checkpoint.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_task_a_recovery_checkpoint.py b/tests/test_task_a_recovery_checkpoint.py index c5cba9ca..602159ea 100644 --- a/tests/test_task_a_recovery_checkpoint.py +++ b/tests/test_task_a_recovery_checkpoint.py @@ -289,7 +289,7 @@ async def test_all_supported_owned_revocations_preserve_signed_bytes_and_never_i tx.execute('''INSERT INTO topic_bans (topic,subject,actor,created_at,expires_at,reason,status) VALUES (?,?,?,?,NULL,?,?)''', (resource.id, owner, owner, wire(NOW), 'restored-ban', 'active'), write=True) - banned = replace(resource, id='t_ban', name='ban') + banned = replace(resource, id='t_ban', name='ban', parent=resource.id) await tx.insert(banned) targets = [('credential.revoke', 'token_one'), ('certificate.revoke', 'cert_one'), ('share_grant.revoke', 'sg_one'), ('share_grant_v2.revoke', 'sg_two'), From f2ebc093fb8333c2a0f196b6ebd86cfdf717affa Mon Sep 17 00:00:00 2001 From: LIghtJUNction Date: Tue, 29 Sep 2026 02:21:38 +0800 Subject: [PATCH 003/136] chore: mark the rewrite preview as 1.0.0a1 The published 0.x line is a different series. This source version does not claim a PyPI release or a finished production migration. Co-authored-by: Cursor --- CHANGELOG.md | 6 ++++++ pyproject.toml | 2 +- src/msg/__init__.py | 2 +- 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9f08c7c1..b2e93c1e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,11 @@ # 变更记录 +## 1.0.0a1 — 重构预览 + +以 1.x 预发布版本标识当前重构，与 PyPI 上的 0.x 系列区分。新版本使用 Python 3.15、PostgreSQL 权威状态、版本化操作与签名授权，并通过 `uv_build` 构建。 + +这只是源码版本调整，尚未发布到 PyPI，也不表示生产部署、旧数据迁移或恢复验收已经完成。 + ## 0.1.0a1 — 本地独立重写交付 从项目设计契约重新实现模块化单体。删除旧实现，不包含旧数据库迁移器或线上自动升级动作。 diff --git a/pyproject.toml b/pyproject.toml index dac7c944..48f96b6f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "uv_build" [project] name = "msg-lmm-best" -version = "0.1.0a1" +version = "1.0.0a1" description = "Atomic communication infrastructure for sandboxed agents" readme = "README.md" requires-python = ">=3.15" diff --git a/src/msg/__init__.py b/src/msg/__init__.py index 966a4f77..d72073ba 100644 --- a/src/msg/__init__.py +++ b/src/msg/__init__.py @@ -1,2 +1,2 @@ """Atomic communication for sandboxed agents.""" -__version__ = "0.1.0a1" +__version__ = "1.0.0a1" From 3490750dd8fd59c8070e9816a104292db0937cea Mon Sep 17 00:00:00 2001 From: LIghtJUNction Date: Tue, 29 Sep 2026 02:43:40 +0800 Subject: [PATCH 004/136] feat(security): accept only console-imported root-signed backup retirement records backup_retired now needs one exact root-signed record bound to the subject and both old key IDs, imported behind require_local_console and re-verified on every read; server_key_retired additionally needs online retirement. Missing, altered, expired, misbound or pre-rotation records fail closed. Slice of #68. Co-authored-by: Cursor --- docs/CUSTODIAL_HISTORY.md | 24 +- docs/ISSUES_68_70_PROGRESS.md | 6 +- src/msg/admin/backup_retirement.py | 56 ++++ src/msg/admin/root.py | 37 +++ src/msg/plugins/custodial_lifecycle.py | 7 +- src/msg/plugins/recovery.py | 3 +- src/msg/security/backup_retirement.py | 135 +++++++++ src/msg/security/custodial_migration.py | 32 +- src/msg/security/custody_history.py | 20 +- tests/test_backup_retirement_attestation.py | 318 ++++++++++++++++++++ 10 files changed, 612 insertions(+), 26 deletions(-) create mode 100644 src/msg/admin/backup_retirement.py create mode 100644 src/msg/security/backup_retirement.py create mode 100644 tests/test_backup_retirement_attestation.py diff --git a/docs/CUSTODIAL_HISTORY.md b/docs/CUSTODIAL_HISTORY.md index db25bc9e..c051f56c 100644 --- a/docs/CUSTODIAL_HISTORY.md +++ b/docs/CUSTODIAL_HISTORY.md @@ -90,11 +90,25 @@ Test Root, rewraps, locally decrypts, ACKs and finalizes it. It refuses a produc application without the isolated selftest namespace. `history_recoverable` covers **enumerated and client-verified revisions only**. -`online_key_retired` describes online vault material only. `backup_retired` and -`server_key_retired` remain false until a separate independently verified offline -retirement procedure exists. Neither a boolean supplied by a network caller nor a -green test run can establish that old backups lost a key. Restore/revocation replay -and real operator acceptance are tracked separately in issues #69 and #70. +`online_key_retired` describes online vault material only. `backup_retired` stays +false unless a root-signed backup retirement record was imported at the physical +console (`RootAdmin.sign_backup_retirement` / `RootAdmin.import_backup_retirement`, +both behind `require_local_console`). The record binds the subject, both old key +IDs, a non-empty list of named backup sets and a validity window of at most 400 +days; it is signed with purpose `custodial-backup-retirement-v1`. Every read +re-verifies it against the active, unrevoked root credential, so an altered, +expired, misbound or pre-rotation record reports `backup_retired=false` +(`backup_status=local_attestation_invalid|expired`). `server_key_retired` is true +only when that record and observable online retirement both hold. + +The record is an operator attestation for the **listed backup sets only**. It +does not prove that no other copy exists and it does not claim that production +backups were destroyed. No network operation writes, accepts or forwards it; a +`backup_retirement` value inside the network-writable migration body is ignored. +Neither a boolean supplied by a network caller nor a green test run can establish +that old backups lost a key. There is no `msg-admin` CLI wiring yet. Restore/ +revocation replay and real operator acceptance are tracked separately in issues +#69 and #70. ## Published contract compatibility diff --git a/docs/ISSUES_68_70_PROGRESS.md b/docs/ISSUES_68_70_PROGRESS.md index 94c5e9e7..1245a4f0 100644 --- a/docs/ISSUES_68_70_PROGRESS.md +++ b/docs/ISSUES_68_70_PROGRESS.md @@ -21,8 +21,10 @@ only the challenge's new signing key may inspect its completed result. The read reports `identity_switched`, `history_recoverable`, `online_retired`, `backup_retired`, `server_key_retired` and a separate `completion_status`. Online retirement requires the new primary keys, retired old keys, revoked old -signing credential and empty destroyed vault fields. No backup-destruction -attestation exists yet, so `backup_retired` and `server_key_retired` remain false. +signing credential and empty destroyed vault fields. `backup_retired` becomes true +only for one exact root-signed record imported at the physical console and +re-verified on each read (see `CUSTODIAL_HISTORY.md`); `server_key_retired` +additionally needs online retirement. Without that record both remain false. The legacy `status=completed` from an empty-inventory identity switch is not a claim that every server-held backup has been destroyed; use the independent retirement fields. diff --git a/src/msg/admin/backup_retirement.py b/src/msg/admin/backup_retirement.py new file mode 100644 index 00000000..adb6b164 --- /dev/null +++ b/src/msg/admin/backup_retirement.py @@ -0,0 +1,56 @@ +"""Local import of a root-signed custodial backup retirement record. + +Never imported by a network adapter, plugin or worker. RootAdmin calls these use +cases only after require_local_console(); tests call them against disposable +databases. The record is verified again on every read, so this import is not a +trusted cache of a success flag. +""" +from __future__ import annotations + +from msg.constants import ROOT_SUBJECT +from msg.core.codec import digest, loads +from msg.core.errors import require +from msg.core.models import AuditEvent, Event, ResourceRef +from msg.plugins.common import new_id +from msg.security.backup_retirement import FIELDS, DOMAIN, check, root_verifier, setting_key + + +def unsigned_statement(value): + require(isinstance(value, dict) and set(value) == FIELDS and value.get('domain') == DOMAIN, + 'backup_retirement_invalid') + return value + + +def completed_migration(tx, subject_id, old_identity_key_id): + """The one completed migration of this subject that retired exactly this old key.""" + matches = [loads(body) for (body,) in tx.rows( + "SELECT body FROM custodial_upgrades WHERE subject=? AND status='completed' ORDER BY id", + (subject_id,)) if loads(body).get('old_identity_key_id') == old_identity_key_id] + require(len(matches) == 1, 'backup_retirement_migration_not_found') + return matches[0] + + +async def import_record(tx, record, *, now, operator): + body = record.get('statement') if isinstance(record, dict) else None + require(isinstance(body, dict) and isinstance(body.get('subject_id'), str) and + isinstance(body.get('old_identity_key_id'), str), 'backup_retirement_invalid') + subject = body['subject_id'] + details = completed_migration(tx, subject, body['old_identity_key_id']) + attestation = check(record, await root_verifier(tx), subject, details, now=now) + key = setting_key(subject, attestation.old_identity_key_id) + previous = tx.setting(key) + tx.set_setting(key, record) + data = {'subject_id': subject, 'old_identity_key_id': attestation.old_identity_key_id, + 'old_encryption_key_id': attestation.old_encryption_key_id, + 'backup_sets': list(attestation.backup_sets), 'record_digest': attestation.record_digest, + 'signer_key_id': attestation.signer_key_id, 'operator': operator, + 'scope': 'listed_backup_sets_only'} + event = Event(id=new_id('audit'), type='root.custodial.backup_retirement', time=now, + request_id=new_id('local'), actor=ROOT_SUBJECT, subject=subject, + resources=(ResourceRef(id=subject),), data=data) + await tx.append_audit(AuditEvent(event=event, + authority=(ResourceRef(id=tx.setting('active_root_certificate')),), + before_digest=digest(previous) if previous is not None else None, + after_digest=attestation.record_digest, previous_digest=None, entry_digest='', + result='imported')) + return attestation diff --git a/src/msg/admin/root.py b/src/msg/admin/root.py index bf31d7d8..c77107fa 100644 --- a/src/msg/admin/root.py +++ b/src/msg/admin/root.py @@ -262,6 +262,43 @@ async def read(): return asyncio.run(_revoke(app,certificate_id,Ed25519Signer.from_bytes(private),reason=reason,operator=operator)) + def sign_backup_retirement(self,source,destination): + """Root-sign an operator's statement that listed backup sets lost one old key.""" + require_local_console(self.config_dir) + from msg.admin.backup_retirement import unsigned_statement + from msg.security.backup_retirement import sign_record + target=Path(destination) + require(not target.exists() and not target.is_symlink(),'backup_retirement_destination_exists') + body=unsigned_statement(loads(Path(source).read_bytes())) + print(canonical({'statement':body,'digest':digest(body), + 'claim':'listed_backup_sets_only'}).decode()) + require(input('Type ATTEST BACKUP RETIREMENT '+digest(body)+': ')== + 'ATTEST BACKUP RETIREMENT '+digest(body),'approval_cancelled') + private=open_private_key(loads(root_envelope(self.config_dir).read_bytes()),getpass.getpass('Root PIN/passphrase: ')) + record=sign_record(body,Ed25519Signer.from_bytes(private)) + durable_write(target,canonical(record),mode=0o600) + return {'status':'backup_retirement_signed','path':str(target),'digest':digest(record)} + + def import_backup_retirement(self,source): + """Only this console path persists a record; reads re-verify it every time.""" + operator=require_local_console(self.config_dir) + from msg.admin.backup_retirement import import_record + app=self._app() + record=loads(Path(source).read_bytes()) + print(canonical({'record':record,'digest':digest(record)}).decode()) + require(input('Type IMPORT BACKUP RETIREMENT '+digest(record)+': ')== + 'IMPORT BACKUP RETIREMENT '+digest(record),'approval_cancelled') + async def run(): + await app.load() + try: + async with app.metadata.transaction(write=True) as tx: + return await import_record(tx,record,now=app.clock(),operator=operator) + finally: + await app.close() + attestation=asyncio.run(run()) + return {'status':'backup_retirement_imported','scope':'listed_backup_sets_only', + **attestation.view()} + def backup(self,destination): require_local_console(self.config_dir) target=Path(destination) diff --git a/src/msg/plugins/custodial_lifecycle.py b/src/msg/plugins/custodial_lifecycle.py index 20e76017..c933cd88 100644 --- a/src/msg/plugins/custodial_lifecycle.py +++ b/src/msg/plugins/custodial_lifecycle.py @@ -66,7 +66,7 @@ async def transition(app, ctx, request, tx, subject): else: require(hmac.compare_digest(details['age_possession_digest'], digest(args['age_proof'])), 'age_possession_failed') - state = await snapshot(tx, subject.resource_id, row[1], challenge, details) + state = await snapshot(tx, subject.resource_id, row[1], challenge, details, now=ctx.now) state.require_observation(args) before = digest(details) if args['action'] == 'refresh': @@ -142,7 +142,8 @@ async def transition(app, ctx, request, tx, subject): await audit(tx, ctx, request, subject.resource_id, 'identity.custodial_' + args['action'], {'challenge_id': args['challenge_id'], 'before_digest': before, 'after_digest': digest(details), 'decision_signature': wire(args['migration_ack'])}) - return HandlerOutput(data=(await snapshot(tx, subject.resource_id, row[1], challenge, details)).data) + return HandlerOutput(data=(await snapshot(tx, subject.resource_id, row[1], challenge, details, + now=ctx.now)).data) async def switch_identity(app, ctx, request, tx, subject, challenge, details, *, retain_decrypt=False): @@ -204,7 +205,7 @@ async def switch_identity(app, ctx, request, tx, subject, challenge, details, *, tx.execute('''UPDATE custodial_vault SET status='destroyed',signing_nonce=NULL, signing_ciphertext=NULL,age_nonce=NULL,age_ciphertext=NULL,destroyed_at=? WHERE subject=?''', (wire(ctx.now), uid), write=True) - current = await snapshot(tx, uid, 'completed', challenge, details) + current = await snapshot(tx, uid, 'completed', challenge, details, now=ctx.now) completed.update(completion_scope='identity_switch_only', phase=current.data['phase'], history_recoverable=current.data['history_recoverable'], verification_scope=current.data['verification_scope'], diff --git a/src/msg/plugins/recovery.py b/src/msg/plugins/recovery.py index e1359f5d..1b76240a 100644 --- a/src/msg/plugins/recovery.py +++ b/src/msg/plugins/recovery.py @@ -40,7 +40,8 @@ async def migration(ctx, request, tx, *, pending=False): challenge, details = loads(row[2]), loads(row[3]) authorize_migration(ctx, subject, row[0], row[1], details) require(not pending or row[1] != 'pending', 'custodial_upgrade_not_pending_rewrap') - state = await migration_snapshot(tx, subject.resource_id, row[1], challenge, details) + state = await migration_snapshot(tx, subject.resource_id, row[1], challenge, details, + now=ctx.now) return subject, row[1], challenge, details, state @op('identity.custodial_upgrade_inventory',obj({'challenge_id':IDENTIFIER}, diff --git a/src/msg/security/backup_retirement.py b/src/msg/security/backup_retirement.py new file mode 100644 index 00000000..f0001e80 --- /dev/null +++ b/src/msg/security/backup_retirement.py @@ -0,0 +1,135 @@ +"""Root-signed attestation that listed backup sets no longer hold one old key. + +Only the physical-console import in msg.admin.backup_retirement persists a +record, under a settings key that no network operation writes. Every read +re-verifies the signature against the current, unrevoked root credential and the +exact subject and old key IDs; an absent, altered, expired or superseded-root +record is not evidence. The statement is an operator attestation for the listed +backup sets only, not a proof that no other copy of the key exists anywhere. +""" +from __future__ import annotations + +from dataclasses import dataclass +from datetime import datetime, timedelta + +from msg.constants import ROOT_SUBJECT +from msg.core.codec import canonical, decode, digest, parse_time, wire +from msg.core.errors import Failure, require +from msg.core.models import Signature +from msg.security.crypto import key_id, verify + +DOMAIN = 'msg-custodial-backup-retirement-v1' +PURPOSE = 'custodial-backup-retirement-v1' +PREFIX = 'custodial_backup_retirement:' +MAX_VALIDITY = timedelta(days=400) +FIELDS = frozenset({'domain', 'subject_id', 'old_identity_key_id', 'old_encryption_key_id', + 'backup_sets', 'attested_at', 'not_after', 'operator'}) + + +@dataclass(frozen=True) +class BackupRetirement: + """Constructed only by check(); a bare boolean or dict is never accepted.""" + subject_id: str + old_identity_key_id: str + old_encryption_key_id: str + backup_sets: tuple + attested_at: datetime + not_after: datetime + signer_key_id: str + record_digest: str + + def binds(self, subject_id, details): + return (self.subject_id == subject_id and + self.old_identity_key_id == details.get('old_identity_key_id') and + self.old_encryption_key_id == details.get('old_encryption_key_id')) + + def view(self): + return {'scope': 'listed_backup_sets_only', 'backup_sets': list(self.backup_sets), + 'attested_at': wire(self.attested_at), 'not_after': wire(self.not_after), + 'signer_key_id': self.signer_key_id, 'record_digest': self.record_digest} + + +def setting_key(subject_id, old_identity_key_id): + return PREFIX + subject_id + ':' + old_identity_key_id + + +def statement(subject_id, details, backup_sets, *, attested_at, not_after, operator): + return {'domain': DOMAIN, 'subject_id': subject_id, + 'old_identity_key_id': details['old_identity_key_id'], + 'old_encryption_key_id': details['old_encryption_key_id'], + 'backup_sets': sorted(set(backup_sets)), 'attested_at': wire(attested_at), + 'not_after': wire(not_after), 'operator': operator} + + +def sign_record(body, signer): + return {'statement': body, 'signature': wire(signer.sign(canonical(body), purpose=PURPOSE))} + + +async def root_verifier(tx): + """The key of the active, unrevoked root certificate; rotation retires old records.""" + try: + certificate_id = tx.setting('active_root_certificate') + certificate = await tx.certificate(certificate_id) + credential = await tx.credential(certificate.key_id) + require(certificate.subject_id == ROOT_SUBJECT and + not await tx.certificate_revoked(certificate_id) and + credential.subject_id == ROOT_SUBJECT and credential.kind == 'signing_key' and + credential.revoked_at is None and + key_id(credential.verifier) == credential.id == certificate.key_id, + 'backup_retirement_root_unavailable') + except (Failure, KeyError, TypeError, ValueError) as exc: + raise Failure('backup_retirement_root_unavailable') from exc + return credential.verifier + + +def _sets(value): + return (isinstance(value, list) and bool(value) and value == sorted(set(value)) and + all(isinstance(item, str) and 0 < len(item) <= 256 for item in value)) + + +def check(record, verifier, subject_id, details, *, now): + """Return the attestation only if this exact record is valid now for these old keys.""" + try: + require(isinstance(record, dict) and set(record) == {'statement', 'signature'}, + 'backup_retirement_invalid') + body = record['statement'] + require(isinstance(body, dict) and set(body) == FIELDS and body['domain'] == DOMAIN and + all(isinstance(body[name], str) and body[name] for name in + ('subject_id', 'old_identity_key_id', 'old_encryption_key_id', 'operator')) and + _sets(body['backup_sets']), 'backup_retirement_invalid') + attested, not_after = parse_time(body['attested_at']), parse_time(body['not_after']) + signature = decode(Signature, record['signature']) + verify(verifier, canonical(body), signature, purpose=PURPOSE) + require(attested < not_after <= attested + MAX_VALIDITY, 'backup_retirement_invalid') + except (Failure, KeyError, TypeError, ValueError) as exc: + raise Failure('backup_retirement_invalid') from exc + require(isinstance(details, dict) and body['subject_id'] == subject_id and + body['old_identity_key_id'] == details.get('old_identity_key_id') and + body['old_encryption_key_id'] == details.get('old_encryption_key_id'), + 'backup_retirement_binding_mismatch') + try: + current = attested <= now < not_after + except TypeError: + current = False + require(current, 'backup_retirement_expired') + return BackupRetirement(subject_id=subject_id, + old_identity_key_id=body['old_identity_key_id'], + old_encryption_key_id=body['old_encryption_key_id'], + backup_sets=tuple(body['backup_sets']), attested_at=attested, not_after=not_after, + signer_key_id=signature.key_id, record_digest=digest(record)) + + +async def verified(tx, subject_id, details, *, now): + """(status, attestation-or-None); any failure is reported as not retired.""" + old = details.get('old_identity_key_id') if isinstance(details, dict) else None + if not isinstance(subject_id, str) or not isinstance(old, str) or not old: + return 'unavailable', None + record = tx.setting(setting_key(subject_id, old)) + if record is None: + return 'unavailable', None + if now is None: + return 'clock_unavailable', None + try: + return 'verified', check(record, await root_verifier(tx), subject_id, details, now=now) + except Failure as exc: + return ('expired' if exc.code == 'backup_retirement_expired' else 'invalid'), None diff --git a/src/msg/security/custodial_migration.py b/src/msg/security/custodial_migration.py index 09f6ce58..bcff8257 100644 --- a/src/msg/security/custodial_migration.py +++ b/src/msg/security/custodial_migration.py @@ -14,6 +14,7 @@ from msg.core.models import ResourceRef, Signature from msg.security.crypto import key_id, verify from msg.security.age_keys import encryption_key_id, public_from_recipient +from msg.security.backup_retirement import BackupRetirement, verified as verified_backup class MigrationPhase(StrEnum): @@ -94,21 +95,25 @@ def output_refs(details): return result -def retirement_view(details, vault, *, identity_switched): +def retirement_view(details, vault, *, identity_switched, subject_id=None, backup=None, + backup_status='requires_offline_verification'): """An absent online key cannot prove that historical backups lost that key.""" signing_deleted = bool(vault and vault[1] is None and vault[2] is None) encryption_deleted = bool(vault and vault[3] is None and vault[4] is None) - # Only a separately validated, local retirement record may set this field. - # The network migration API deliberately never writes or accepts one. - backup = details.get('backup_retirement') + # Only a locally imported, root-signed record verified by + # msg.security.backup_retirement may set this field. The network migration + # API never writes or accepts one; details['backup_retirement'] is ignored. + retired = isinstance(backup, BackupRetirement) and backup.binds(subject_id, details) + online = bool(identity_switched and vault and vault[0] == 'destroyed' and + signing_deleted and encryption_deleted) return {'identity_switched': identity_switched, 'token_revoked': identity_switched, 'online_signing_key_deleted': signing_deleted, 'online_encryption_key_deleted': encryption_deleted, - 'backup_retired': False, - 'backup_status': 'requires_offline_verification', - 'backup_retirement_record': backup, - 'server_key_retired': False} + 'backup_retired': retired, + 'backup_status': 'local_attestation_verified' if retired else backup_status, + 'backup_retirement_record': backup.view() if retired else None, + 'server_key_retired': retired and online} @dataclass(frozen=True) @@ -166,7 +171,8 @@ async def verify_challenge_binding(tx, subject, status, challenge, details): 'custodial_encryption_target_changed') -async def snapshot(tx, subject, status, challenge, details): +async def snapshot(tx, subject, status, challenge, details, *, now=None): + """Without a trusted clock a backup attestation cannot be current, so it is not used.""" await verify_challenge_binding(tx, subject, status, challenge, details) frozen = details.get('age_inventory') require(isinstance(frozen, list), 'custodial_inventory_required') @@ -245,13 +251,19 @@ async def snapshot(tx, subject, status, challenge, details): vault = tx.one('''SELECT status,signing_nonce,signing_ciphertext,age_nonce,age_ciphertext FROM custodial_vault WHERE subject=?''', (subject,)) switched = status == 'completed' - retirement = retirement_view(details, vault, identity_switched=switched) + backup_status, backup = await verified_backup(tx, subject, details, now=now) if switched \ + else ('unavailable', None) + retirement = retirement_view(details, vault, identity_switched=switched, subject_id=subject, + backup=backup, backup_status='requires_offline_verification' if backup_status == 'unavailable' + else 'local_attestation_' + backup_status) complete = not drift and verified == set(sources) phase = MigrationPhase.CHALLENGE if status == 'pending' else MigrationPhase.MIGRATING if switched: phase = MigrationPhase.HISTORY_RECOVERABLE if complete else MigrationPhase.IDENTITY_SWITCHED if retirement['online_signing_key_deleted'] and retirement['online_encryption_key_deleted']: phase = MigrationPhase.ONLINE_KEY_RETIRED + if retirement['server_key_retired']: + phase = MigrationPhase.SERVER_KEY_RETIRED results = {'mappings': mappings, 'acks': acks, 'recovery_envelope': envelope, 'recovery_acks': recovery_acks, 'resolution': details.get('resolution')} return MigrationSnapshot({'challenge_id': challenge['challenge_id'], 'status': status, diff --git a/src/msg/security/custody_history.py b/src/msg/security/custody_history.py index 39297716..661504ae 100644 --- a/src/msg/security/custody_history.py +++ b/src/msg/security/custody_history.py @@ -9,6 +9,7 @@ from msg.core.errors import Failure from msg.core.models import Signature from msg.security.age_keys import encryption_key_id, public_from_recipient +from msg.security.backup_retirement import verified as verified_backup from msg.security.crypto import key_id, verify @@ -101,8 +102,12 @@ def history_evidence(subject_id, challenge_id, challenge, details, current): return result -async def retirement_evidence(tx, subject_id, details, *, history_recoverable): - """Separate observable online retirement from unproven backup destruction.""" +async def retirement_evidence(tx, subject_id, details, *, history_recoverable, now=None): + """Separate observable online retirement from locally attested backup retirement. + + backup_retired needs one exact root-signed record imported at the physical + console; server_key_retired additionally needs observable online retirement. + """ new_signing = tx.one('SELECT key_id FROM identity_keys WHERE subject=? AND is_primary=1', (subject_id,)) new_encryption = tx.one('SELECT key_id FROM encryption_subkeys WHERE subject=? AND is_primary=1', @@ -129,8 +134,13 @@ async def retirement_evidence(tx, subject_id, details, *, history_recoverable): online = (switched and destroyed and signing_revoked and all(row is not None and row[0] is not None and not row[1] for row in (old_signing, old_encryption))) + status, attestation = await verified_backup(tx, subject_id, details, now=now) + backup = attestation is not None and attestation.binds(subject_id, details) + server = online and backup return {'identity_switched': switched, 'history_recoverable': history_recoverable, - 'online_retired': online, 'backup_retired': False, 'server_key_retired': False, - 'backup_retirement_evidence': 'unavailable', - 'completion_status': ('pending_backup_retirement' if online and history_recoverable + 'online_retired': online, 'backup_retired': backup, 'server_key_retired': server, + 'backup_retirement_evidence': status if not backup else 'local_attestation_verified', + 'backup_retirement_record': attestation.view() if backup else None, + 'completion_status': ('server_key_retired' if server and history_recoverable + else 'pending_backup_retirement' if online and history_recoverable else 'pending_history' if switched else 'pending_identity_switch')} diff --git a/tests/test_backup_retirement_attestation.py b/tests/test_backup_retirement_attestation.py new file mode 100644 index 00000000..12cb14b8 --- /dev/null +++ b/tests/test_backup_retirement_attestation.py @@ -0,0 +1,318 @@ +"""Issue #68 slice: a physical-console, root-signed backup retirement record. + +Without that exact record backup_retired stays false; server_key_retired also +needs observable online retirement. Altered, expired, misbound, wrong-signer and +superseded-root records fail closed. No network path can write or supply one. +""" +from dataclasses import replace +from datetime import timedelta +import os +from pathlib import Path + +import pytest + +from msg.admin.backup_retirement import import_record +from msg.admin.root import RootAdmin +from msg.constants import ROOT_SPACE, ROOT_SUBJECT +from msg.core.codec import b64, canonical, loads, unb64, wire +from msg.core.errors import Failure +from msg.core.models import (Certificate, CapabilityGrant, Credential, IssuancePolicy, Scope, + Signature) +from msg.security.age_keys import generate_age_key, public_from_recipient +from msg.security.backup_retirement import (BackupRetirement, check, setting_key, sign_record, + statement) +from msg.security.certificates import sign_certificate +from msg.security.crypto import Ed25519Signer +from msg.security.custodial_migration import retirement_view +from msg.security.custody_history import retirement_evidence +from msg.storage.sqlite import FakeMetadataStore +from test_custodial_upgrade import begin, finish +from test_service import NOW, call + +SUBJECT = 'u_exit' +SETS = ['offsite-2026-09-weekly', 'pg-basebackup-2026-09-20'] + + +def root_certificate(root): + grant = CapabilityGrant(capability='cert.issue', version=1, + scope=Scope(resource_id=ROOT_SPACE, descendants=True), + operations=frozenset({'cert.publish@1'}), constraints={}) + return sign_certificate(Certificate(resource_id='cert_root', serial='cert_root', + subject_id=ROOT_SUBJECT, key_id=root.key_id, issuer_id=ROOT_SUBJECT, + parent_certificate_id=None, authority_sources=(), kind='ca', grants=(grant,), + not_before=NOW - timedelta(days=1), expires_at=NOW + timedelta(days=365), + target_service='http://testserver', delegation_depth=8, + issuance=IssuancePolicy(issue_grants=(grant,), max_cert_ttl_seconds=86400, + max_child_ca_depth=3, max_delegation_depth=8), + signature=Signature(key_id=root.key_id, algorithm='ed25519', value=b'')), root) + + +def credential(signer, subject, *, revoked=None): + return Credential(id=signer.key_id, subject_id=subject, kind='signing_key', + verifier=signer.public_key, ceiling=(), not_before=NOW - timedelta(days=1), + expires_at=None, revoked_at=revoked) + + +@pytest.fixture +async def exited(): + """An online-retired custodial exit plus an active root, all in a disposable store.""" + store = FakeMetadataStore() + root, old, new = (Ed25519Signer.generate() for _ in range(3)) + details = {'old_identity_key_id': old.key_id, 'new_identity_key_id': new.key_id, + 'old_encryption_key_id': 'e_old', 'new_encryption_key_id': 'e_new'} + async with store.transaction(write=True) as tx: + for key, primary in ((old, 0), (new, 1)): + tx.execute('INSERT INTO identity_keys VALUES(?,?,?,?,?,?)', + (key.key_id, SUBJECT, b64(key.public_key), wire(NOW), + None if primary else wire(NOW), primary), write=True) + for kid, primary in (('e_old', 0), ('e_new', 1)): + _, recipient = generate_age_key() + tx.execute('INSERT INTO encryption_subkeys VALUES(?,?,?,?,?,?,?)', + (kid, SUBJECT, recipient, b64(public_from_recipient(recipient)), wire(NOW), + None if primary else wire(NOW), primary), write=True) + for signer, subject, revoked in ((old, SUBJECT, NOW), (root, ROOT_SUBJECT, None)): + tx.execute('INSERT INTO credentials VALUES(?,?,?)', (signer.key_id, subject, + canonical(credential(signer, subject, revoked=revoked)).decode()), write=True) + tx.execute('''INSERT INTO custodial_vault(subject,signing_key_id,encryption_key_id, + status,created_at,destroyed_at) VALUES(?,?,?,?,?,?)''', + (SUBJECT, old.key_id, 'e_old', 'destroyed', wire(NOW), wire(NOW)), write=True) + tx.execute('''INSERT INTO custodial_upgrades(id,subject,credential_id,status,expires_at, + challenge,body) VALUES(?,?,?,?,?,?,?)''', ('cupg_exit', SUBJECT, 't_old', 'completed', + wire(NOW), '{}', canonical(details).decode()), write=True) + await tx.register_certificate(root_certificate(root), None, 0) + tx.set_setting('active_root_certificate', 'cert_root') + yield store, root, details + await store.close() + + +def signed(details, signer, *, subject=SUBJECT, attested=NOW - timedelta(hours=1), + not_after=NOW + timedelta(days=30), sets=SETS, **changes): + body = statement(subject, dict(details, **changes), sets, attested_at=attested, + not_after=not_after, operator='console-operator') + return sign_record(body, signer) + + +async def evidence(store, details, *, now=NOW): + async with store.transaction(write=False) as tx: + return await retirement_evidence(tx, SUBJECT, details, history_recoverable=True, now=now) + + +async def stored(store, details): + async with store.transaction(write=False) as tx: + return tx.setting(setting_key(SUBJECT, details['old_identity_key_id'])) + + +@pytest.mark.asyncio +async def test_without_record_backup_and_server_key_remain_false(exited): + store, _, details = exited + async with store.transaction(write=True) as tx: + tx.set_setting('backup_retired', True) # An arbitrary flag is not evidence. + result = await evidence(store, details) + assert result['online_retired'] and not result['backup_retired'] + assert result['server_key_retired'] is False + assert result['backup_retirement_evidence'] == 'unavailable' + assert result['completion_status'] == 'pending_backup_retirement' + + +@pytest.mark.asyncio +async def test_imported_root_signed_record_retires_exactly_that_backup_scope(exited): + store, root, details = exited + async with store.transaction(write=True) as tx: + attestation = await import_record(tx, signed(details, root), now=NOW, operator='uid:0:/dev/tty1') + assert attestation.backup_sets == tuple(SETS) + result = await evidence(store, details) + assert result['backup_retired'] is True and result['server_key_retired'] is True + assert result['completion_status'] == 'server_key_retired' + assert result['backup_retirement_record']['scope'] == 'listed_backup_sets_only' + assert result['backup_retirement_record']['backup_sets'] == SETS + assert not (await evidence(store, details, now=None))['backup_retired'] + other = dict(details, old_identity_key_id=Ed25519Signer.generate().key_id) + assert not (await evidence(store, other))['backup_retired'] + async with store.transaction(write=False) as tx: + audits = [loads(body) for (body,) in tx.rows('SELECT body FROM audit')] + assert [a['event']['type'] for a in audits] == ['root.custodial.backup_retirement'] + assert audits[0]['event']['data']['operator'] == 'uid:0:/dev/tty1' + + +@pytest.mark.asyncio +async def test_backup_attestation_alone_is_not_server_key_retirement(exited): + store, root, details = exited + async with store.transaction(write=True) as tx: + await import_record(tx, signed(details, root), now=NOW, operator='console') + tx.execute('UPDATE custodial_vault SET age_ciphertext=? WHERE subject=?', + ('surviving encrypted key', SUBJECT), write=True) + result = await evidence(store, details) + assert result['backup_retired'] is True and result['online_retired'] is False + assert result['server_key_retired'] is False + assert result['completion_status'] == 'pending_history' + + +def tampered(record): + body = dict(record['statement'], backup_sets=['a-different-set']) + return dict(record, statement=body) + + +@pytest.mark.asyncio +@pytest.mark.parametrize('make, code', [ + (lambda d, root: signed(d, Ed25519Signer.generate()), 'backup_retirement_invalid'), + (lambda d, root: tampered(signed(d, root)), 'backup_retirement_invalid'), + (lambda d, root: signed(d, root, not_after=NOW), 'backup_retirement_expired'), + (lambda d, root: signed(d, root, attested=NOW + timedelta(minutes=1), + not_after=NOW + timedelta(days=1)), 'backup_retirement_expired'), + (lambda d, root: signed(d, root, not_after=NOW + timedelta(days=500)), 'backup_retirement_invalid'), + (lambda d, root: signed(d, root, sets=[]), 'backup_retirement_invalid'), + (lambda d, root: signed(d, root, old_encryption_key_id='e_new'), 'backup_retirement_binding_mismatch'), + (lambda d, root: signed(d, root, subject='u_other'), 'backup_retirement_migration_not_found'), + (lambda d, root: dict(signed(d, root), backup_retired=True), 'backup_retirement_invalid'), + (lambda d, root: dict(signed(d, root), statement=dict(signed(d, root)['statement'], extra=1)), + 'backup_retirement_invalid'), + (lambda d, root: True, 'backup_retirement_invalid'), +]) +async def test_bad_records_are_rejected_at_import_and_never_stored(exited, make, code): + store, root, details = exited + with pytest.raises(Failure, match='^' + code + '$'): + async with store.transaction(write=True) as tx: + await import_record(tx, make(details, root), now=NOW, operator='console') + assert await stored(store, details) is None + assert not (await evidence(store, details))['backup_retired'] + + +@pytest.mark.asyncio +async def test_persisted_record_is_reverified_and_fails_closed(exited): + store, root, details = exited + good = signed(details, root) + async with store.transaction(write=True) as tx: + await import_record(tx, good, now=NOW, operator='console') + later = await evidence(store, details, now=NOW + timedelta(days=30)) + assert not later['backup_retired'] and later['backup_retirement_evidence'] == 'expired' + for bad in (tampered(good), signed(details, Ed25519Signer.generate()), + signed(details, root, old_encryption_key_id='e_new')): + async with store.transaction(write=True) as tx: + tx.set_setting(setting_key(SUBJECT, details['old_identity_key_id']), bad) + result = await evidence(store, details) + assert not result['backup_retired'] and not result['server_key_retired'] + assert result['backup_retirement_evidence'] == 'invalid' + async with store.transaction(write=True) as tx: + tx.set_setting(setting_key(SUBJECT, details['old_identity_key_id']), good) + assert (await evidence(store, details))['backup_retired'] + async with store.transaction(write=True) as tx: + root_credential = await tx.credential(root.key_id) + tx.execute('UPDATE credentials SET body=? WHERE id=?', (canonical( + replace(root_credential, revoked_at=NOW)).decode(), root.key_id), write=True) + rotated = await evidence(store, details) + assert not rotated['backup_retired'] and rotated['backup_retirement_evidence'] == 'invalid' + + +def test_retirement_view_ignores_network_body_and_caller_flags(): + root = Ed25519Signer.generate() + details = {'old_identity_key_id': 'k_old', 'old_encryption_key_id': 'e_old'} + record = signed(details, root) + attestation = check(record, root.public_key, SUBJECT, details, now=NOW) + destroyed = ('destroyed', None, None, None, None) + network = dict(details, backup_retirement=record) + for backup in (None, True, 'verified', {'backup_retired': True}, record): + view = retirement_view(network, destroyed, identity_switched=True, subject_id=SUBJECT, + backup=backup) + assert view['backup_retired'] is False and view['server_key_retired'] is False + assert view['backup_retirement_record'] is None + assert not retirement_view(details, destroyed, identity_switched=True, subject_id='u_other', + backup=attestation)['backup_retired'] + forged = BackupRetirement(**{**attestation.__dict__, 'old_encryption_key_id': 'e_other'}) + assert not retirement_view(details, destroyed, identity_switched=True, subject_id=SUBJECT, + backup=forged)['backup_retired'] + view = retirement_view(details, destroyed, identity_switched=True, subject_id=SUBJECT, + backup=attestation) + assert view['backup_retired'] and view['server_key_retired'] + assert view['backup_status'] == 'local_attestation_verified' + retained = retirement_view(details, ('decrypt_only', None, None, 'n', 'c'), + identity_switched=True, subject_id=SUBJECT, backup=attestation) + assert retained['backup_retired'] and not retained['server_key_retired'] + + +@pytest.mark.parametrize('euid, env, code', [ + (1000, {}, 'local_os_administrator_required'), + (0, {'SSH_CONNECTION': '203.0.113.1 5 198.51.100.2 22'}, 'remote_admin_forbidden'), +]) +def test_import_and_signing_require_the_physical_console(tmp_path, monkeypatch, euid, env, code): + import msg.admin.root as root_module + monkeypatch.setattr(root_module.os, 'geteuid', lambda: euid, raising=False) + for name in ('SSH_CONNECTION', 'SSH_CLIENT', 'SSH_TTY'): + monkeypatch.delenv(name, raising=False) + for name, value in env.items(): + monkeypatch.setenv(name, value) + source = tmp_path/'record.json' + source.write_bytes(b'{}') + admin = RootAdmin(tmp_path) + with pytest.raises(Failure, match='^' + code + '$'): + admin.import_backup_retirement(source) + with pytest.raises(Failure, match='^' + code + '$'): + admin.sign_backup_retirement(source, tmp_path/'signed.json') + assert not (tmp_path/'signed.json').exists() + + +def test_no_network_adapter_can_write_or_import_the_record(): + src = Path(__file__).resolve().parents[1]/'src'/'msg' + allowed = {src/'admin'/'backup_retirement.py', src/'admin'/'root.py', + src/'security'/'backup_retirement.py'} + for path in src.rglob('*.py'): + if path in allowed: + continue + text = path.read_text() + assert 'admin.backup_retirement' not in text, path + assert 'import_record' not in text, path + assert 'custodial_backup_retirement:' not in text, path + assert 'setting_key' not in text or 'backup_retirement' not in text, path + + +@pytest.mark.asyncio +async def test_network_inventory_reports_backup_retired_only_after_local_import(installed): + app, root = installed + created = await call(app, 'identity.custodial_create', {'handle': 'backup-exit', + 'nonce': b64(os.urandom(32)), 'recovery_secret': b64(os.urandom(32))}, contract_version=2) + subject = created.data['subject_id'] + token = (created.data['credential_id'], unb64(created.data['token'])) + signer = Ed25519Signer.generate() + age_key, recipient = generate_age_key() + challenge = await begin(app, subject, token, signer, recipient) + done = await finish(app, subject, token, signer, age_key, challenge.data) + assert done.status == 'ok' and done.data['status'] == 'completed' + args = {'challenge_id': challenge.data['challenge_id']} + + async def inventory(extra=None): + return await call(app, 'identity.custodial_upgrade_inventory', dict(args, **(extra or {})), + key=signer, subject=subject) + + before = await inventory() + assert before.status == 'ok', wire(before) + assert before.data['phase'] == 'online_key_retired' + assert before.data['retirement']['backup_retired'] is False + assert before.data['retirement']['server_key_retired'] is False + async with app.metadata.transaction(write=False) as tx: + details = loads(tx.one('SELECT body FROM custodial_upgrades WHERE id=?', + (args['challenge_id'],))[0]) + supplied = await inventory({'backup_retirement': signed(details, root, subject=subject)}) + assert supplied.status == 'error' + with pytest.raises(Failure, match='^backup_retirement_invalid$'): + async with app.metadata.transaction(write=True) as tx: + await import_record(tx, signed(details, Ed25519Signer.generate(), subject=subject), + now=NOW, operator='console') + assert (await inventory()).data['retirement']['backup_retired'] is False + record = signed(details, root, subject=subject) + async with app.metadata.transaction(write=True) as tx: + await import_record(tx, record, now=NOW, operator='console') + after = await inventory() + assert after.status == 'ok', wire(after) + assert after.data['retirement']['backup_retired'] is True + assert after.data['retirement']['server_key_retired'] is True + assert list(after.data['retirement']['backup_retirement_record']['backup_sets']) == SETS + assert after.data['phase'] == 'server_key_retired' + async with app.metadata.transaction(write=False) as tx: + result = await retirement_evidence(tx, subject, details, history_recoverable=True, now=NOW) + assert result['online_retired'] and result['backup_retired'] and result['server_key_retired'] + async with app.metadata.transaction(write=True) as tx: + tx.set_setting(setting_key(subject, details['old_identity_key_id']), tampered(record)) + damaged = await inventory() + assert damaged.status == 'ok', wire(damaged) + assert damaged.data['retirement']['backup_retired'] is False + assert damaged.data['retirement']['backup_status'] == 'local_attestation_invalid' + assert damaged.data['phase'] == 'online_key_retired' From 1ca13fdf5ff71e0cbdca598a80f1cafb2561cbaa Mon Sep 17 00:00:00 2001 From: LIghtJUNction Date: Tue, 29 Sep 2026 03:30:29 +0800 Subject: [PATCH 005/136] fix: reject stale Root authority and replay bans without inherited expiry Fix the 15 failures reproduced by cloud run 36471956362 without changing any regression assertions. Keep current-authority reads mutation-free and recovery quarantined. Use time differences to avoid datetime overflow. Refs #68 #69 #70 #83 #85. --- docs/ISSUE_PROGRESS.md | 23 ++++++++++++++++++----- src/msg/admin/backup_retirement.py | 2 +- src/msg/admin/recovery_replay.py | 8 +++++--- src/msg/security/backup_retirement.py | 22 +++++++++++++++------- 4 files changed, 39 insertions(+), 16 deletions(-) diff --git a/docs/ISSUE_PROGRESS.md b/docs/ISSUE_PROGRESS.md index 66aca2eb..ee4980c2 100644 --- a/docs/ISSUE_PROGRESS.md +++ b/docs/ISSUE_PROGRESS.md @@ -1,14 +1,27 @@ # 当前 issue 推进笔记 -源请求：2026-09-28T19:16Z。此文件只记录本轮代码与验收证据；权威设计和已完成归档仍有效，入口见 issue #83。 +源请求：2026-09-28T19:16Z。此文件记录本轮代码与验收证据；权威设计和已完成归档仍有效，入口见 issue #83。只更新当前笔记，不重复堆叠状态文件。 ## 基线及集成 - 开始 main：`01e39345c755d6ec8f584534de20d9cd265ca922`，tree `f51cadf931d32c53c64501af8db33dbde43eacbb`。 -- 主线 CI `36465610565`、Recovery safety `36465610588` 已读为 success。 -- 保留 #145 `f2ebc093fb8333c2a0f196b6ebd86cfdf717affa` 与 #154 `3490750dd8fd59c8070e9816a104292db0937cea` 的提交历史，不修改原分支。 -- 本轮先加入失败回归：旧禁言过期时间、当前 Root 证书/凭据有效期与结构/签名、日期上溢。新工作流在云端真实 PostgreSQL 运行；原完整四分片、逐 node-ID gate、conformance、构建与恢复专项保持不变。 -- 当前为复现阶段，未声称测试通过或合并完成。运行的 source.json 绑定实际 checkout SHA/tree、原 PR head、run/attempt；PR 合并候选与原分支 head 分开记录。 +- 主线 Rewrite CI `36465610565`、Recovery safety `36465610588` 已读为 success。 +- 集成分支：`integrate/issue-recovery-cloud-20260928`。以 main 为第一父提交，完整保留 #145 `f2ebc093fb8333c2a0f196b6ebd86cfdf717affa` 与 #154 `3490750dd8fd59c8070e9816a104292db0937cea` 的历史，不修改原分支。 +- 完整读过权威设计及完成归档、两份 PR 的改动及 review/CI；旧 #103/#112 不再作为开放 PR 队列。 + +## 已复现与本次修复 + +失败基线：head `4bfeece690a26970b66f86b06cd3817dbb0b38cd`，tree `9f2b8db3861e8e299c6522d21a18d03d3d4b5dcd`。 + +云端专项 `36471956362` / job `109096152390` 使用真实 PostgreSQL 和 Python 3.15.0rc2，55 个用例中 **15 failed / 40 passed / 0 skipped / 0 errors**；原有两文件 36 个用例全部通过。已下载 artifact `10992276490`，实际核对 ZIP SHA256 `01d114f54d75777640e49df670f7a0dbb0c7f9404c726768065daeee201fe9d4`，source.json 与以上 head/tree 一致。 + +本次仅修实现，不修改失败断言： + +1. `topic_ban.apply` 不再继承旧快照中的 expires_at；无到期字段的检查点恢复为 active/NULL，保留原 actor/reason，重复与并发重放仍只改变一次。 +2. 退役证明读取和导入共同重验当前 Root 自签证书结构、签名、有效期及 signing credential 有效期；到期边界不再误报 verified，失败读取不写业务状态，失败导入不写审计。 +3. 有效期用时间差而非向不可信日期加 400 天，避免 year 9999 上溢；未来证明仍受当前时间检查拒绝。 + +本次修复后的 CI 尚待回读，不将预期通过写成通过。专项 source.json 绑定实际 checkout SHA/tree、PR head、run/attempt；完整四分片、逐 node-ID gate、conformance、构建及恢复专项不变。下一步检查本机命令接入及安全文件边界，再回读最终组合 head 的完整 CI。 ## 全部开放项（尚无整项关闭证据） diff --git a/src/msg/admin/backup_retirement.py b/src/msg/admin/backup_retirement.py index adb6b164..ef0ca393 100644 --- a/src/msg/admin/backup_retirement.py +++ b/src/msg/admin/backup_retirement.py @@ -36,7 +36,7 @@ async def import_record(tx, record, *, now, operator): isinstance(body.get('old_identity_key_id'), str), 'backup_retirement_invalid') subject = body['subject_id'] details = completed_migration(tx, subject, body['old_identity_key_id']) - attestation = check(record, await root_verifier(tx), subject, details, now=now) + attestation = check(record, await root_verifier(tx, now=now), subject, details, now=now) key = setting_key(subject, attestation.old_identity_key_id) previous = tx.setting(key) tx.set_setting(key, record) diff --git a/src/msg/admin/recovery_replay.py b/src/msg/admin/recovery_replay.py index f0a61b1e..573f8121 100644 --- a/src/msg/admin/recovery_replay.py +++ b/src/msg/admin/recovery_replay.py @@ -165,15 +165,17 @@ async def _apply(tx, fact): if kind == 'topic_ban.apply': require(tx.one('SELECT id FROM resources WHERE id=?', (target,)) is not None, 'recovery_fact_missing') - row = tx.one('SELECT status FROM topic_bans WHERE topic=? AND subject=?', (target, subject)) + row = tx.one('SELECT status,expires_at FROM topic_bans WHERE topic=? AND subject=?', (target, subject)) if row is None: tx.execute('''INSERT INTO topic_bans (topic,subject,actor,created_at,expires_at,reason,status) VALUES (?,?,?,?,NULL,?,?)''', (target, subject, ROOT_SUBJECT, at, 'recovery-replay', 'active'), write=True) return True - if row[0] != 'active': - tx.execute("UPDATE topic_bans SET status='active' WHERE topic=? AND subject=?", + # This checkpoint fact has no expiry. Do not inherit one from an older + # snapshot and silently lift the restored deny, now or in the future. + if row[0] != 'active' or row[1] is not None: + tx.execute("UPDATE topic_bans SET status='active',expires_at=NULL WHERE topic=? AND subject=?", (target, subject), write=True) return True return False diff --git a/src/msg/security/backup_retirement.py b/src/msg/security/backup_retirement.py index f0001e80..84c54ea9 100644 --- a/src/msg/security/backup_retirement.py +++ b/src/msg/security/backup_retirement.py @@ -16,6 +16,7 @@ from msg.core.codec import canonical, decode, digest, parse_time, wire from msg.core.errors import Failure, require from msg.core.models import Signature +from msg.security.certificates import verify_certificate from msg.security.crypto import key_id, verify DOMAIN = 'msg-custodial-backup-retirement-v1' @@ -65,18 +66,24 @@ def sign_record(body, signer): return {'statement': body, 'signature': wire(signer.sign(canonical(body), purpose=PURPOSE))} -async def root_verifier(tx): - """The key of the active, unrevoked root certificate; rotation retires old records.""" +async def root_verifier(tx, *, now): + """Recheck the current self-signed Root and its credential, not just historical validity.""" try: certificate_id = tx.setting('active_root_certificate') certificate = await tx.certificate(certificate_id) credential = await tx.credential(certificate.key_id) - require(certificate.subject_id == ROOT_SUBJECT and + require(certificate.resource_id == certificate_id and + certificate.subject_id == certificate.issuer_id == ROOT_SUBJECT and + certificate.parent_certificate_id is None and certificate.kind == 'ca' and + certificate.issuance is not None and + certificate.not_before <= now < certificate.expires_at and not await tx.certificate_revoked(certificate_id) and credential.subject_id == ROOT_SUBJECT and credential.kind == 'signing_key' and - credential.revoked_at is None and + credential.revoked_at is None and credential.not_before <= now and + (credential.expires_at is None or now < credential.expires_at) and key_id(credential.verifier) == credential.id == certificate.key_id, 'backup_retirement_root_unavailable') + verify_certificate(certificate, credential.verifier) except (Failure, KeyError, TypeError, ValueError) as exc: raise Failure('backup_retirement_root_unavailable') from exc return credential.verifier @@ -100,8 +107,9 @@ def check(record, verifier, subject_id, details, *, now): attested, not_after = parse_time(body['attested_at']), parse_time(body['not_after']) signature = decode(Signature, record['signature']) verify(verifier, canonical(body), signature, purpose=PURPOSE) - require(attested < not_after <= attested + MAX_VALIDITY, 'backup_retirement_invalid') - except (Failure, KeyError, TypeError, ValueError) as exc: + # Subtract instead of adding to an untrusted date near datetime.max. + require(timedelta(0) < not_after - attested <= MAX_VALIDITY, 'backup_retirement_invalid') + except (Failure, KeyError, TypeError, ValueError, OverflowError) as exc: raise Failure('backup_retirement_invalid') from exc require(isinstance(details, dict) and body['subject_id'] == subject_id and body['old_identity_key_id'] == details.get('old_identity_key_id') and @@ -130,6 +138,6 @@ async def verified(tx, subject_id, details, *, now): if now is None: return 'clock_unavailable', None try: - return 'verified', check(record, await root_verifier(tx), subject_id, details, now=now) + return 'verified', check(record, await root_verifier(tx, now=now), subject_id, details, now=now) except Failure as exc: return ('expired' if exc.code == 'backup_retirement_expired' else 'invalid'), None From f317920c06d4c2ecd9df9d72f4099604b5d4eeed Mon Sep 17 00:00:00 2001 From: LIghtJUNction Date: Tue, 29 Sep 2026 04:31:40 +0800 Subject: [PATCH 006/136] fix: verify active root certificate with existing primitives --- src/msg/security/backup_retirement.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/msg/security/backup_retirement.py b/src/msg/security/backup_retirement.py index 84c54ea9..17d0dad9 100644 --- a/src/msg/security/backup_retirement.py +++ b/src/msg/security/backup_retirement.py @@ -16,7 +16,7 @@ from msg.core.codec import canonical, decode, digest, parse_time, wire from msg.core.errors import Failure, require from msg.core.models import Signature -from msg.security.certificates import verify_certificate +from msg.security.certificates import certificate_body from msg.security.crypto import key_id, verify DOMAIN = 'msg-custodial-backup-retirement-v1' @@ -83,7 +83,7 @@ async def root_verifier(tx, *, now): (credential.expires_at is None or now < credential.expires_at) and key_id(credential.verifier) == credential.id == certificate.key_id, 'backup_retirement_root_unavailable') - verify_certificate(certificate, credential.verifier) + verify(credential.verifier, canonical(certificate_body(certificate)),\n certificate.signature, purpose='certificate') except (Failure, KeyError, TypeError, ValueError) as exc: raise Failure('backup_retirement_root_unavailable') from exc return credential.verifier From 0e4dbe777053e354bcf4845b851a0c9b86ab8310 Mon Sep 17 00:00:00 2001 From: LIghtJUNction Date: Tue, 29 Sep 2026 04:33:19 +0800 Subject: [PATCH 007/136] fix: correct root certificate verification call syntax --- src/msg/security/backup_retirement.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/msg/security/backup_retirement.py b/src/msg/security/backup_retirement.py index 17d0dad9..fcd10c90 100644 --- a/src/msg/security/backup_retirement.py +++ b/src/msg/security/backup_retirement.py @@ -83,7 +83,8 @@ async def root_verifier(tx, *, now): (credential.expires_at is None or now < credential.expires_at) and key_id(credential.verifier) == credential.id == certificate.key_id, 'backup_retirement_root_unavailable') - verify(credential.verifier, canonical(certificate_body(certificate)),\n certificate.signature, purpose='certificate') + verify(credential.verifier, canonical(certificate_body(certificate)), + certificate.signature, purpose='certificate') except (Failure, KeyError, TypeError, ValueError) as exc: raise Failure('backup_retirement_root_unavailable') from exc return credential.verifier From 6ea463d6fa8b256cb07d0ede2d7795a5336eda4f Mon Sep 17 00:00:00 2001 From: LIghtJUNction Date: Tue, 29 Sep 2026 05:00:55 +0800 Subject: [PATCH 008/136] test(client): verify a dependency-clean wheel and four signed transports Refs #158 #82 #83 #85. Add strict import, helper compatibility and server preflight regressions before changing implementation; retain all full-server CI gates. --- .github/workflows/client-boundary.yml | 72 +++++++++++++++ docs/CLIENT_BOUNDARY_PROGRESS.md | 21 +++++ scripts/check_client_install.py | 124 ++++++++++++++++++++++++++ tests/test_client_boundary.py | 123 +++++++++++++++++++++++++ 4 files changed, 340 insertions(+) create mode 100644 .github/workflows/client-boundary.yml create mode 100644 docs/CLIENT_BOUNDARY_PROGRESS.md create mode 100644 scripts/check_client_install.py create mode 100644 tests/test_client_boundary.py diff --git a/.github/workflows/client-boundary.yml b/.github/workflows/client-boundary.yml new file mode 100644 index 00000000..383239b6 --- /dev/null +++ b/.github/workflows/client-boundary.yml @@ -0,0 +1,72 @@ +name: Client boundary +on: + push: + branches: ['fix/client-boundary-*'] + pull_request: + paths: + - 'src/msg/**' + - 'pyproject.toml' + - 'tests/test_client_boundary.py' + - 'scripts/check_client_install.py' + - '.github/workflows/client-boundary.yml' + workflow_dispatch: +permissions: + contents: read +concurrency: + group: client-boundary-${{ github.ref }} + cancel-in-progress: true +jobs: + installed-client: + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@v7 + with: + persist-credentials: false + - uses: actions/setup-python@v7 + with: + python-version: '3.15' + allow-prereleases: true + cache: pip + - name: Record tested source + run: | + mkdir -p artifacts + python - <<'PY' + import json, os, pathlib, subprocess + def git(*args): + return subprocess.check_output(['git', *args], text=True).strip() + pathlib.Path('artifacts/source.json').write_text(json.dumps({ + 'commit': git('rev-parse', 'HEAD'), 'tree': git('rev-parse', 'HEAD^{tree}'), + 'run_id': os.environ['GITHUB_RUN_ID'], 'event': os.environ['GITHUB_EVENT_NAME'], + }, indent=2) + '\n') + PY + - name: Install full development dependencies + run: python -m pip install -e '.[dev]' + - name: Verify helper ownership, exact proof context and fail-closed assembly + run: python -m pytest tests/test_client_boundary.py --junitxml=artifacts/client-boundary.xml + - name: Build the distributable wheel and source archive + if: always() + run: | + python -m build + python scripts/check_package_artifacts.py dist + - name: Install a clean client-only wheel outside the checkout + if: always() + run: | + python -m venv "$RUNNER_TEMP/msg-client-only" + client_python="$RUNNER_TEMP/msg-client-only/bin/python" + "$client_python" -m pip install dist/*.whl + "$client_python" -m pip check + mkdir -p "$RUNNER_TEMP/msg-client-outside" + cd "$RUNNER_TEMP/msg-client-outside" + "$RUNNER_TEMP/msg-client-only/bin/msg" --help > "$GITHUB_WORKSPACE/artifacts/client-help.txt" + "$RUNNER_TEMP/msg-client-only/bin/msgd" --help > "$GITHUB_WORKSPACE/artifacts/daemon-help.txt" + "$client_python" -I "$GITHUB_WORKSPACE/scripts/check_client_install.py" --minimal-install \ + > "$GITHUB_WORKSPACE/artifacts/client-install.json" + "$client_python" -m pip list --format=json > "$GITHUB_WORKSPACE/artifacts/client-packages.json" + - uses: actions/upload-artifact@v4 + if: always() + with: + name: client-boundary + path: | + artifacts/ + dist/ diff --git a/docs/CLIENT_BOUNDARY_PROGRESS.md b/docs/CLIENT_BOUNDARY_PROGRESS.md new file mode 100644 index 00000000..11966c65 --- /dev/null +++ b/docs/CLIENT_BOUNDARY_PROGRESS.md @@ -0,0 +1,21 @@ +# Client dependency boundary progress — 2026-09-29 + +Refs #158/#82/#83/#85. Branch `fix/client-boundary-20260929` starts at tool-boundary head `f14fd1533f70983b046df4b4c813993e8a536d12` (#165). It does not write the #155 recovery, #156 executor or #166 metadata-session branches. + +## Regression-first work + +This commit adds seven boundary/compatibility/installation checks and a standalone fresh-interpreter client probe. It contains no implementation fix yet. The probe imports client/CLI/TUI modules under a strict server-import guard, creates/reloads protected client key files, and sends the same genuinely signed request through HTTP, path GET, GraphQL and MCP using an in-process mock HTTP endpoint. It never sends to a real service. + +The independent workflow builds the real wheel and installs it with normal dependencies into a fresh virtualenv outside the source checkout. It runs pip check, both command help entrypoints and the probe with explicit rejection of server-only packages. No PostgreSQL/Valkey service is configured for this workflow; full repository CI remains unchanged and mandatory. + +## Intended ownership and compatibility + +- Compact result serialization belongs to protocol codec; executor keeps only a compatibility import. +- Atomic protected local-file replacement has one narrowly named owner, not the Git/CAS backend. Existing permissions/fsync/replacement semantics remain unchanged. +- Custodial proof input/derivation and signed migration statements share one state-free owner; server-held vault and retirement verification stay server-side. Existing exports remain aliases, not copied security code. +- MCP version negotiation constants must not require importing the server implementation from the network client. +- Base installation expresses client dependencies; the server extra is the complete server install. The existing dev extra retains server dependencies and msg/msgd command names are unchanged. Documentation must explicitly migrate full server installation to the server extra; no automatic package installation. + +## Evidence still required + +Read the RED run before implementation. Then verify helper compatibility, four real client transport calls, a dependency-clean wheel environment, all existing credential journal/recovery and server tests, full exact-node-ID/conformance/build gates and current integration tree. No size, speed, Windows/Android field result or production readiness is claimed by this refactor. Keep #158 open until those facts are proved. diff --git a/scripts/check_client_install.py b/scripts/check_client_install.py new file mode 100644 index 00000000..f12ed662 --- /dev/null +++ b/scripts/check_client_install.py @@ -0,0 +1,124 @@ +"""Exercise real client imports and signed envelopes without a server runtime. + +Run with --minimal-install in a fresh wheel-only virtualenv, outside the source +checkout. Network traffic uses httpx.MockTransport, not a remote service. +""" +from __future__ import annotations + +import argparse +import asyncio +import importlib +import importlib.abc +import importlib.util +import json +from pathlib import Path +import stat +import sys +import tempfile + +SERVER_DEPENDENCIES = ('starlette', 'uvicorn', 'psycopg', 'valkey', 'aiohttp', 'dns', 'graphql') +FORBIDDEN = ('msg.application', 'msg.core.executor', 'msg.storage', 'msg.admin', + 'msg.workers', 'msg.security.vault', 'msg.security.custodial_migration', + 'msg.transports.http', 'msg.transports.graphql', 'msg.transports.mcp') + + +class ClientBoundary(importlib.abc.MetaPathFinder): + def find_spec(self, fullname, path=None, target=None): + if any(fullname == name or fullname.startswith(name + '.') for name in FORBIDDEN): + raise RuntimeError('client imported a server implementation: ' + fullname) + return None + + +async def check(directory): + import httpx + from msg.client import ClientState + from msg.core.codec import canonical, loads, wire + from msg.core.requests import request_for, signing_bytes + from msg.security.crypto import Ed25519Signer, verify + from msg.transports.client import TRANSPORTS + from msg.transports.packet import decode_packet, path_packet + + for name in ('cli', 'tui', 'client_certificates', 'client_content', 'client_custodial', + 'client_market', 'client_recovery', 'client_secrets', 'client_tokens', 'client_upgrade'): + importlib.import_module('msg.' + name) + state = ClientState(directory, server='https://unit.invalid') + signer = Ed25519Signer.from_bytes(bytes(range(32))) + state.save_signer(signer) + state.ensure_encryption_key() + restored = ClientState(directory) + assert restored.signer.public_key == signer.public_key + assert restored.encryption_recipient == state.encryption_recipient + for path in (state.path, state.key_path, state.age_key_path): + assert stat.S_IMODE(path.stat().st_mode) == 0o600 + assert stat.S_IMODE(directory.stat().st_mode) == 0o700 + request = request_for('content.read', {'id': 'r_test'}, state.server, + subject='u_test', signer=signer, request_id='client-boundary') + calls = [] + + def respond(incoming): + path = incoming.url.path + if path == '/_transports': + return httpx.Response(200, json={'version': 1, 'target_service': state.server, + 'operations': {'content.read': 'read'}, 'limits': { + 'max_request_bytes': 1048576, 'max_response_bytes': 1048576, + 'max_path_bytes': 8192, 'encodings': ['j', 'gz']}}) + if path.startswith('/-/g/'): + _, _, _, operation, encoding, encoded = path.split('/') + packet = path_packet(encoded, encoding, 1048576) + assert packet.operation == operation and incoming.method == 'GET' + kind = 'path_get' + else: + data = loads(incoming.content) + if path == '/_read/graphql': + data = data['variables']['packet'] + kind = 'graphql' + elif path == '/-/mcp': + assert incoming.headers['MCP-Protocol-Version'] == '2025-11-25' + data = data['params']['arguments']['packet'] + kind = 'mcp_http' + else: + assert path == '/-/p/content.read' + kind = 'http' + assert incoming.method == 'POST' + packet = decode_packet(data) + assert canonical(packet) == canonical(request) + verify(signer.public_key, signing_bytes(packet), packet.proof.signature, purpose='request') + calls.append(kind) + result = {'request_id': packet.request_id, 'operation': packet.operation, 'status': 'ok', + 'actor': 'u_test', 'subject': 'u_test', 'resources': [{'id': 'r_test'}], + 'data': {'content': 'signed client response'}} + if kind == 'graphql': + result = {'data': {'call': result}} + elif kind == 'mcp_http': + result = {'jsonrpc': '2.0', 'id': packet.request_id, + 'result': {'structuredContent': result}} + return httpx.Response(200, json=result) + + async with httpx.AsyncClient(transport=httpx.MockTransport(respond)) as http: + for name, transport in TRANSPORTS.items(): + result = await transport(state.server, http=http).call(request) + assert result.status == 'ok' and result.resources[0].id == 'r_test', (name, wire(result)) + assert result.data['content'] == 'signed client response' + assert sorted(calls) == sorted(TRANSPORTS) + assert not any(any(name == prefix or name.startswith(prefix + '.') for prefix in FORBIDDEN) + for name in sys.modules) + return calls + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('--minimal-install', action='store_true') + args = parser.parse_args() + if args.minimal_install: + present = [name for name in SERVER_DEPENDENCIES if importlib.util.find_spec(name) is not None] + assert not present, 'server-only Python dependencies in client environment: ' + ', '.join(present) + sys.meta_path.insert(0, ClientBoundary()) + with tempfile.TemporaryDirectory(prefix='msg-client-install-') as folder: + calls = asyncio.run(check(Path(folder) / 'client')) + print(json.dumps({'status': 'ok', 'transports': sorted(calls), + 'server_implementation_imported': False, + 'minimal_install': args.minimal_install})) + + +if __name__ == '__main__': + main() diff --git a/tests/test_client_boundary.py b/tests/test_client_boundary.py new file mode 100644 index 00000000..02e46139 --- /dev/null +++ b/tests/test_client_boundary.py @@ -0,0 +1,123 @@ +"""Public helper ownership, unchanged bytes and supported client/server installs.""" +from __future__ import annotations + +import hashlib +import hmac +import importlib.util +from pathlib import Path +import subprocess +import sys +import tomllib + +import pytest + +ROOT = Path(__file__).resolve().parents[1] + + +def test_result_serialization_has_one_protocol_owner(): + from msg.core.codec import result_wire + from msg.core.executor import result_wire as compatibility_export + from msg.core.models import OperationResult + from msg.cli import result_wire as client_export + + assert result_wire is compatibility_export is client_export + assert result_wire.__module__ == 'msg.core.codec' + result = OperationResult(request_id='r', operation='content.read', status='ok', actor=None, + subject=None, resources=(), data={'text': 'plain'}) + value = result_wire(result) + assert value['data'] == {'text': 'plain'} + assert 'replayed' not in value and 'prefer_cli' not in value + + +def test_atomic_file_owner_keeps_compatibility_and_failed_replace_protection(tmp_path, monkeypatch): + from msg.atomic_file import durable_write + from msg.storage.git import durable_write as compatibility_export + import msg.atomic_file as atomic + + assert durable_write is compatibility_export + path = tmp_path / 'state' / 'identity' + durable_write(path, b'first') + assert path.read_bytes() == b'first' and path.stat().st_mode & 0o777 == 0o600 + def fail_replace(*args): + raise OSError('simulated replace refusal') + monkeypatch.setattr(atomic.os, 'replace', fail_replace) + with pytest.raises(OSError, match='simulated replace refusal'): + durable_write(path, b'not committed') + assert path.read_bytes() == b'first' + assert list(path.parent.iterdir()) == [path] + + +def test_custodial_proof_and_statements_are_shared_without_vault_state(): + from msg.security.custodial_protocol import ( + UPGRADE_CONTEXT_FIELDS, _proof, client_upgrade_proof, stage_statement, ack_statement) + from msg.security.vault import client_upgrade_proof as old_proof + from msg.security.custodial_migration import stage_statement as old_stage, ack_statement as old_ack + from msg.core.codec import b64, canonical + from cryptography.hazmat.primitives import hashes + from cryptography.hazmat.primitives.kdf.hkdf import HKDF + + assert client_upgrade_proof is old_proof + assert stage_statement is old_stage and ack_statement is old_ack + assert client_upgrade_proof.__module__ == 'msg.security.custodial_protocol' + expected_fields = ('subject_id', 'challenge_id', 'request_id', 'handle', 'public_key', + 'encryption_recipient', 'server_public', 'nonce', 'expires_at') + assert UPGRADE_CONTEXT_FIELDS == expected_fields + challenge = {name: name + '-fixture' for name in expected_fields} + challenge['nonce'] = b64(bytes(range(32))) + challenge['not_signed'] = 'not part of this published proof context' + context = {name: challenge[name] for name in expected_fields} + shared = bytes(range(32, 64)) + key = HKDF(algorithm=hashes.SHA256(), length=32, salt=bytes(range(32)), + info=b'msg-custodial-upgrade-pop-v1').derive(shared) + expected = b64(hmac.digest(key, canonical(context), 'sha256')) + assert _proof(shared, challenge) == expected + assert stage_statement('u', {'choice': 'rewrap', 'migration_ack': 'omit'}, 'r') == { + 'domain': 'msg-custodial-decision-v1', 'subject_id': 'u', 'request_id': 'r', + 'decision': {'choice': 'rewrap'}} + assert ack_statement('u', 'c', 'd', {'id': 'old'}, {'id': 'new'}, 'p', 'r') == { + 'domain': 'msg-custodial-history-ack-v1', 'subject_id': 'u', 'challenge_id': 'c', + 'inventory_digest': 'd', 'source': {'id': 'old'}, 'target': {'id': 'new'}, + 'method': 'rewrap', 'plaintext_digest': 'p', 'request_id': 'r'} + + +def test_mcp_client_uses_neutral_version_declaration(): + from msg.transports.mcp_protocol import PROTOCOL_VERSION, SUPPORTED_VERSIONS + from msg.transports import mcp + assert PROTOCOL_VERSION == mcp.PROTOCOL_VERSION == '2025-11-25' + assert SUPPORTED_VERSIONS is mcp.SUPPORTED_VERSIONS + assert SUPPORTED_VERSIONS == frozenset({'2025-11-25', '2025-06-18', '2025-03-26'}) + + +def test_fresh_client_import_guard_covers_actual_four_transport_calls(): + completed = subprocess.run([sys.executable, str(ROOT / 'scripts/check_client_install.py')], + capture_output=True, text=True, timeout=30) + assert completed.returncode == 0, completed.stdout + completed.stderr + import json + report = json.loads(completed.stdout) + assert report['transports'] == ['graphql', 'http', 'mcp_http', 'path_get'] + assert report['server_implementation_imported'] is False + + +def test_install_metadata_separates_roles_but_dev_keeps_full_server(): + project = tomllib.loads((ROOT / 'pyproject.toml').read_text())['project'] + def name(requirement): + return requirement.split('>=')[0].split('[')[0] + server_names = {'starlette', 'uvicorn', 'psycopg', 'valkey', 'aiohttp', 'dnspython', 'graphql-core'} + assert not server_names & {name(value) for value in project['dependencies']} + extras = project['optional-dependencies'] + assert server_names <= {name(value) for value in extras['server']} + assert 'msg-lmm-best[server]' in extras['dev'] + assert project['scripts'] == {'msg': 'msg.cli:main', 'msgd': 'msg.daemon:main'} + + +def test_missing_server_dependencies_are_reported_before_local_state(tmp_path, monkeypatch, capsys): + import msg.daemon as daemon + original = importlib.util.find_spec + monkeypatch.setattr(importlib.util, 'find_spec', + lambda name, *args, **kwargs: None if name == 'psycopg' else original(name, *args, **kwargs)) + directory = tmp_path / 'not-created' + assert daemon.main(['--config-dir', str(directory), 'serve']) == 2 + captured = capsys.readouterr() + assert 'server_dependencies_required' in captured.err + assert 'msg-lmm-best[server]' in captured.err + assert not directory.exists() From 8692ad6c0056e793bbe7b0efc9f5fd0b08ab89c8 Mon Sep 17 00:00:00 2001 From: LIghtJUNction Date: Tue, 29 Sep 2026 05:02:02 +0800 Subject: [PATCH 009/136] test: define market ownership and protected posting contracts --- .github/workflows/market-boundaries.yml | 66 ++++++++++ docs/MARKET_BOUNDARY_PROGRESS.md | 57 +++++++++ tests/test_market_boundaries.py | 159 ++++++++++++++++++++++++ 3 files changed, 282 insertions(+) create mode 100644 .github/workflows/market-boundaries.yml create mode 100644 docs/MARKET_BOUNDARY_PROGRESS.md create mode 100644 tests/test_market_boundaries.py diff --git a/.github/workflows/market-boundaries.yml b/.github/workflows/market-boundaries.yml new file mode 100644 index 00000000..572838b3 --- /dev/null +++ b/.github/workflows/market-boundaries.yml @@ -0,0 +1,66 @@ +name: Market boundary contracts +on: + push: + branches: [fix/market-boundaries-20260929] + workflow_dispatch: +permissions: + contents: read +concurrency: + group: market-boundaries-${{ github.ref }} + cancel-in-progress: false +jobs: + contracts: + runs-on: ubuntu-latest + timeout-minutes: 20 + services: + postgres: + image: postgres:16 + env: + POSTGRES_USER: msgtest + POSTGRES_PASSWORD: msgtest + POSTGRES_DB: postgres + ports: ['5432:5432'] + options: >- + --health-cmd "pg_isready -U msgtest" + --health-interval 10s --health-timeout 5s --health-retries 5 + env: + MSG_TEST_POSTGRES_URL_TEMPLATE: postgresql://msgtest:msgtest@127.0.0.1:5432/{database} + steps: + - uses: actions/checkout@v7 + with: + persist-credentials: false + - uses: actions/setup-python@v7 + with: + python-version: '3.15' + allow-prereleases: true + cache: pip + - name: Install isolated test dependencies + run: | + sudo apt-get update -qq + sudo apt-get install -y -qq age + python -m pip install -e '.[dev]' + - name: Record exact source + env: + SOURCE_SHA: ${{ github.sha }} + run: | + mkdir -p artifacts + test "$(git rev-parse HEAD)" = "$SOURCE_SHA" + python - <<'PY' + import json,os,subprocess + from pathlib import Path + git=lambda *args:subprocess.check_output(['git',*args],text=True).strip() + Path('artifacts/source.json').write_text(json.dumps({'commit':git('rev-parse','HEAD'), + 'tree':git('rev-parse','HEAD^{tree}'),'event_sha':os.environ['SOURCE_SHA'], + 'dirty':bool(git('status','--porcelain'))},indent=2)+'\n') + PY + - name: Run ownership and actual ledger contracts + run: python -m pytest tests/test_market_boundaries.py --junitxml=artifacts/market-boundaries.xml + - name: Preserve all published checkout and recovery semantics + run: | + python -m pytest tests/test_orders.py tests/test_managed_checkout.py tests/test_checkout_decision_integrity.py tests/test_market_contracts.py tests/test_market_integration.py tests/test_market_recovery.py tests/test_market_account_scope.py tests/test_market_71.py tests/test_market_72.py tests/test_money_core.py tests/test_market_redemption.py tests/test_bounty.py tests/test_store_foundation.py --junitxml=artifacts/market-regressions.xml + - uses: actions/upload-artifact@v4 + if: always() + with: + name: market-boundaries-${{ github.sha }} + path: artifacts/ + if-no-files-found: error diff --git a/docs/MARKET_BOUNDARY_PROGRESS.md b/docs/MARKET_BOUNDARY_PROGRESS.md new file mode 100644 index 00000000..c67cd19d --- /dev/null +++ b/docs/MARKET_BOUNDARY_PROGRESS.md @@ -0,0 +1,57 @@ +# Market boundary progress + +## Scope and coordination + +Issue #159; references #85/#83, not completion of #71/#72 or production acceptance. +Base: `476e64f04ed1d9c7dcab4a41c4f3cdb72faad6a7`, tree +`40ca207a3c49be347120e058e47ed328ec4e3098`. Branch: +`fix/market-boundaries-20260929`. Coordination is #83 comment5878359290. + +This work does not modify #165/#167 core/tool/executor ownership, #155 recovery, +or #166 metadata session extraction. Main and all active claims were refreshed +before starting. No production, real funds, Root/PIN or outward delivery action. + +## Regression first + +The first commit changes no production implementation. It adds ten tests: +seven ownership/import/unique-guard checks and three real PostgreSQL behavior +contracts. Current expected failures concern the new module boundaries; expected +passing characterizations cover buyer/seller privacy, forged escrow authority +with no ledger changes, receipt verification/byte preservation and replay. +No test result is claimed until the exact cloud run is inspected. + +## Observed dependency graph + +`market.orders/email/arbitration/delivery -> plugins.orders` for order records, +subject checks and existing projections. `market -> plugins.store` for listing +and package reads. `market.escrow/orders -> plugins.money` for protected posting. +`market.policy/targets/notifications/escrow/delivery -> plugins.delivery` for +managed delivery records and verification. `plugins.orders/delivery -> market.escrow` +closes the reverse dependency. Function-local imports are included in this audit. + +## Intended ownership and invariants + +- `market.order_records`: signed subject/viewer checks, order IDs, authorized + record lookup and explicitly legacy-compatible projections. +- `market.catalog`: existing authorized listing/body/package reads, not plugin + registration or version-specific catalog schema/defaults. +- `market.ledger`: existing policy constants, account requirements and one posting + implementation. The in-process escrow guard has one owner; moving it does not + make it a credential or a network operation. Local-only Root issuance is unchanged. +- `market.managed_delivery`: existing bounded managed-package verification, + local preparation and read helpers. Their published version semantics remain; + they do not replace v3 service/manual delivery or arbitration state machines. + +Old plugin imports re-export these same functions, never wrappers with new policy +or copied implementations. Version-specific schema, request signing and response +adaptation stay at their existing entrypoints. Existing transactions, account +locks, receipt purposes, ID generation and recovery/lease fences remain intact. +No new Manager, Factory, Repository, ORM or alternate money/escrow path is added. + +## Verification and integration + +Focused cloud Actions run the new contracts and existing checkout v1/v2/v3, +account-scope, deterministic clearing, Bounty, redemption, storage and recovery +suites. Full four-shard exact node-ID/conformance/build and applicable recovery +checks remain required on the final synchronized head. No skips, weakened +assertions, changed published versions or historical fact/schema migration. diff --git a/tests/test_market_boundaries.py b/tests/test_market_boundaries.py new file mode 100644 index 00000000..0955e8ed --- /dev/null +++ b/tests/test_market_boundaries.py @@ -0,0 +1,159 @@ +"""Cross-version market ownership and unchanged real-ledger behavior.""" +from __future__ import annotations + +import ast +import importlib +from pathlib import Path +import subprocess +import sys + +import pytest + +from msg.core.codec import canonical, decode, wire +from msg.core.errors import Failure +from msg.core.models import Signature +from msg.security.crypto import verify +from test_managed_checkout import setup_sale, buy, ok + + +ENTRYPOINTS = {'msg.plugins.orders','msg.plugins.store','msg.plugins.money','msg.plugins.delivery'} +OWNERS = ( + ('msg.plugins.orders','msg.market.order_records',{ + '_subject':'require_signed_subject','_viewer':'require_order_viewer', + '_order_id':'new_order_id','_row':'read_order','_view':'legacy_order_view'}), + ('msg.plugins.store','msg.market.catalog',{ + '_listing':'read_listing','_body':'read_listing_body','_package_row':'read_package_record'}), + ('msg.plugins.money','msg.market.ledger',{ + 'account_requirements':'account_requirements','_owner':'require_money_subject', + '_amount':'checked_amount','_balance':'balance','_supply':'total_supply', + 'clearing_decision':'clearing_decision','_post_entry':'append_entry','_post_transfer':'post_transfer'}), + ('msg.plugins.delivery','msg.market.managed_delivery',{ + '_delivery':'read_delivery','_buyer_order':'read_buyer_order', + '_verified_delivery':'verify_managed_delivery','_package':'read_managed_package', + '_payload':'read_managed_payloads','prepare_managed':'prepare_managed', + 'delivery_summary':'delivery_summary'}), +) + + +def test_market_has_no_business_entrypoint_imports_even_inside_functions(): + import msg.market + violations=[] + for path in Path(msg.market.__file__).parent.rglob('*.py'): + for node in ast.walk(ast.parse(path.read_text())): + names=([node.module] if isinstance(node,ast.ImportFrom) else + [item.name for item in node.names] if isinstance(node,ast.Import) else []) + for name in names: + if name in ENTRYPOINTS: + violations.append((path.name,node.lineno,name)) + assert not violations, violations + + +@pytest.mark.parametrize('entrypoint,owner,names',OWNERS) +def test_compatibility_imports_are_the_same_owned_functions(entrypoint,owner,names): + adapter=importlib.import_module(entrypoint) + shared=importlib.import_module(owner) + assert not hasattr(shared,'install') + for old,new in names.items(): + function=getattr(shared,new) + assert function is getattr(adapter,old), (old,new) + assert function.__module__==owner + records=importlib.import_module('msg.market.order_records') + store=importlib.import_module('msg.plugins.store') + assert store._subject is records.require_signed_subject + + +def test_shared_market_and_versioned_use_cases_import_without_loading_entrypoints(): + code=''' +import importlib, importlib.abc, sys +forbidden={'msg.plugins.orders','msg.plugins.store','msg.plugins.money','msg.plugins.delivery'} +class NoEntrypoints(importlib.abc.MetaPathFinder): + def find_spec(self,fullname,path=None,target=None): + if fullname in forbidden: + raise AssertionError('shared market loaded business entrypoint '+fullname) +sys.meta_path.insert(0,NoEntrypoints()) +for name in ('order_records','catalog','ledger','managed_delivery','orders','delivery', + 'escrow','arbitration','email','policy','rationale','targets','delivery_notifications'): + importlib.import_module('msg.market.'+name) +assert not forbidden.intersection(sys.modules) +''' + result=subprocess.run([sys.executable,'-c',code],capture_output=True,text=True) + assert result.returncode==0,result.stderr + + +def test_ledger_guard_has_one_owner_and_does_not_depend_on_escrow_or_handlers(): + ledger=importlib.import_module('msg.market.ledger') + escrow=importlib.import_module('msg.market.escrow') + assert ledger._ESCROW_WRITE is escrow._ESCROW_WRITE + imports=[n.module for n in ast.walk(ast.parse(Path(ledger.__file__).read_text())) + if isinstance(n,ast.ImportFrom)] + assert 'msg.market.escrow' not in imports + assert not ENTRYPOINTS.intersection(imports) + money=importlib.import_module('msg.plugins.money') + for name in ('CURRENCY_ID','CODE','SCALE','MAX_MINOR','POLICY_VERSION','POLICY_DIGEST'): + assert getattr(money,name)==getattr(ledger,name) + + +async def test_shared_order_views_preserve_buyer_privacy_and_zero_write_semantics(installed): + from msg.plugins.orders import _row,_view + app,seller,buyer,_,args=await setup_sale(installed,policy='escrow-v1') + args['auto_accept']=False + order=ok(await buy(app,buyer,args,rid='shared-order-views')).data['order'] + async with app.metadata.transaction(write=False) as tx: + counts=lambda: tuple(tx.one(f'SELECT COUNT(*) FROM {name}')[0] + for name in ('money_ledger','store_orders','store_deliveries','results','events','jobs')) + before=counts() + record=_row(tx,order['id'],buyer[1]) + buyer_view=_view(record,buyer[1]) + seller_view=_view(_row(tx,order['id'],seller[1]),seller[1]) + assert buyer_view['delivery_target']=={'subject_id':buyer[1],'channel':'site'} + assert {'delivery_target','payment_intent_digest','receipt_refs'}<=set(buyer_view) + assert not {'delivery_target','payment_intent_digest','receipt_refs'}.intersection(seller_view) + assert buyer_view['state']==seller_view['state']=='delivered' + with pytest.raises(Failure,match='order_not_found'): + _row(tx,order['id'],'unrelated-viewer') + with pytest.raises(Failure,match='order_not_found'): + _row(tx,'ord_does_not_exist',buyer[1]) + assert counts()==before + + +async def test_shared_posting_path_rejects_forged_escrow_authority_without_writes(installed): + from msg.plugins.money import _balance,_post_transfer + from msg.plugins.orders import _row + app,seller,buyer,_,args=await setup_sale(installed,policy='escrow-v1') + args['auto_accept']=False + order=ok(await buy(app,buyer,args,rid='shared-posting-guard')).data['order'] + async with app.metadata.transaction(write=True) as tx: + record=_row(tx,order['id'],buyer[1]) + before=tuple(tx.one(f'SELECT COUNT(*) FROM {name}')[0] + for name in ('money_ledger','money_accounts','order_escrow_decisions','order_settlements')) + for authority in (None,object()): + with pytest.raises(Failure,match='escrow_release_forbidden'): + _post_transfer(tx,sender=record['escrow_subject'],recipient=seller[1], + amount=record['total_price_minor'],actor=buyer[1], + request_id='forged-shared-leg',now=app.clock(),receipt_signer=app.receipt_signer, + escrow_authority=authority) + after=tuple(tx.one(f'SELECT COUNT(*) FROM {name}')[0] + for name in ('money_ledger','money_accounts','order_escrow_decisions','order_settlements')) + assert before==after + assert _balance(tx,record['escrow_subject'])==record['total_price_minor'] + assert _balance(tx,seller[1])==0 + + +async def test_alias_reads_and_replay_preserve_signed_receipt_bytes(installed): + from msg.plugins.delivery import _delivery,_verified_delivery + from msg.plugins.orders import _row + app,_,buyer,_,args=await setup_sale(installed) + first=ok(await buy(app,buyer,args,rid='shared-receipt-bytes')) + payment=first.data['payment'] + verify(app.receipt_signer.public_key,canonical(payment['body']), + decode(Signature,payment['signature']),purpose='money-receipt') + async with app.metadata.transaction(write=False) as tx: + before=tuple(tx.rows('SELECT seq,receipt FROM money_ledger ORDER BY seq')) + record=_row(tx,first.data['order']['id'],buyer[1]) + await _verified_delivery(app,tx,record,_delivery(tx,record['id'])) + replay=ok(await buy(app,buyer,args,rid='shared-receipt-bytes')) + assert replay.replayed and replay.data==first.data + assert canonical(replay.data['payment'])==canonical(payment) + async with app.metadata.transaction(write=False) as tx: + assert tuple(tx.rows('SELECT seq,receipt FROM money_ledger ORDER BY seq'))==before + assert canonical(payment).decode() in [row[1] for row in before] From d0d7f4f61f138b0ced266f2324f4a90c1689b6b5 Mon Sep 17 00:00:00 2001 From: LIghtJUNction Date: Tue, 29 Sep 2026 05:05:41 +0800 Subject: [PATCH 010/136] ci: assemble and audit the exact market ownership extraction --- .github/tools/assemble_market_boundaries.py | 313 ++++++++++++++++++ .../workflows/assemble-market-boundaries.yml | 60 ++++ 2 files changed, 373 insertions(+) create mode 100644 .github/tools/assemble_market_boundaries.py create mode 100644 .github/workflows/assemble-market-boundaries.yml diff --git a/.github/tools/assemble_market_boundaries.py b/.github/tools/assemble_market_boundaries.py new file mode 100644 index 00000000..79fac431 --- /dev/null +++ b/.github/tools/assemble_market_boundaries.py @@ -0,0 +1,313 @@ +"""Exact-base source extraction for PR #168; never imports project modules.""" +from pathlib import Path +import ast +import copy +import difflib +import hashlib +import io +import json +import re +import subprocess +import tokenize + +ROOT = Path.cwd() +BASE = '8692ad6c0056e793bbe7b0efc9f5fd0b08ab89c8' +EXPECTED = { + 'orders':'f9abe3da981c548a60d0e1b1630d82e02d0e8491', + 'store':'09a1e14328d512e91913bf00e987cf234816348e', + 'money':'424a03f1e6de882aba425f857ab602df818f90c1', + 'delivery':'4500a65ed55e5ea06426c9ac3011f4317ecdc70b', +} +NAMES = { + 'orders':('order_records',{ + '_subject':'require_signed_subject','_viewer':'require_order_viewer', + '_order_id':'new_order_id','_row':'read_order','_view':'legacy_order_view'}), + 'store':('catalog',{ + '_listing':'read_listing','_body':'read_listing_body','_package_row':'read_package_record'}), + 'money':('ledger',{ + 'account_requirements':'account_requirements','_owner':'require_money_subject', + '_amount':'checked_amount','_balance':'balance','_supply':'total_supply', + 'clearing_decision':'clearing_decision','_post_entry':'append_entry','_post_transfer':'post_transfer'}), + 'delivery':('managed_delivery',{ + '_delivery':'read_delivery','_buyer_order':'read_buyer_order', + '_verified_delivery':'verify_managed_delivery','_package':'read_managed_package', + '_payload':'read_managed_payloads','prepare_managed':'prepare_managed', + 'delivery_summary':'delivery_summary'}), +} +CONSTANTS = {'orders':('_COLUMNS',),'store':(), + 'money':('CURRENCY_ID','CODE','SCALE','MAX_MINOR','POLICY_VERSION','POLICY_DIGEST'), + 'delivery':('MAX_INLINE_BYTES','_COLUMNS')} +HEADERS = { + 'orders':'''"""Shared order identity, authorized records and legacy-compatible projections. + +These functions own record access, not operation registration or settlement. +""" +from __future__ import annotations +import base64 +import os +from msg.core.codec import loads +from msg.core.errors import require + +''', + 'store':'''"""Authorized catalog reads shared by published checkout versions. + +Version-specific schemas, creation and updates stay at catalog entrypoints. +""" +from __future__ import annotations +from msg.core.codec import loads +from msg.core.errors import Failure, require +from msg.core.models import ResourceRef +from msg.plugins.common import check_access, resolve +from msg.market.order_records import require_signed_subject + +''', + 'money':'''"""One append-only ledger posting path in the caller's transaction. + +This module grants no commit rights. Public transfers cannot supply the internal +escrow guard; Root issuance remains a separate local-console use case. +""" +from __future__ import annotations +from uuid import uuid4 +from msg.constants import ROOT_SUBJECT +from msg.core.codec import canonical, digest, wire +from msg.core.errors import require +from msg.core.models import AccessRequirement, SignatureProof + +''', + 'delivery':'''"""Shared bounded managed-package verification and local preparation. + +These preserve the managed checkout contracts; versioned manual/service delivery, +arbitration and operation registration retain their existing entrypoints. +""" +from __future__ import annotations +from msg.core.codec import b64, canonical, decode, digest, loads, wire +from msg.core.errors import require +from msg.core.models import BlobRef +from msg.plugins.common import new_id +from msg.market.ledger import CURRENCY_ID, balance +from msg.market.catalog import read_package_record +from msg.market.order_records import read_order +from msg.market.delivery_targets import validate_target + +''', +} +EXTRA_RENAMES = {'store':{'_subject':'require_signed_subject'}, + 'delivery':{'order_row':'read_order','_package_row':'read_package_record','_balance':'balance'}} +before = {} +after = {} +audit = [] + + +def read(path): + text = (ROOT/path).read_text() + before.setdefault(path,text) + return text + + +def nodes(text): + return ast.parse(text).body + + +def functions(text): + return {node.name:node for node in nodes(text) + if isinstance(node,(ast.FunctionDef,ast.AsyncFunctionDef))} + + +def assignments(text): + return {target.id:node for node in nodes(text) if isinstance(node,ast.Assign) + for target in node.targets if isinstance(target,ast.Name)} + + +def segment(text,node): + return ast.get_source_segment(text,node) + + +def rename(text,mapping): + tokens = tokenize.generate_tokens(io.StringIO(text).readline) + return tokenize.untokenize(token._replace(string=mapping.get(token.string,token.string)) + if token.type==tokenize.NAME else token for token in tokens) + + +def splice(text,edits): + starts=[0] + for line in text.splitlines(True): + starts.append(starts[-1]+len(line)) + indexed=[] + for node,value in edits: + indexed.append((starts[node.lineno-1]+node.col_offset, + starts[node.end_lineno-1]+node.end_col_offset,value)) + for start,end,value in sorted(indexed,reverse=True): + text=text[:start]+value+text[end:] + return text + + +def clean(text): + return re.sub(r'\n{4,}','\n\n\n',text).rstrip()+'\n' + + +sources={name:read('src/msg/plugins/'+name+'.py') for name in NAMES} +for name,expected in EXPECTED.items(): + actual=subprocess.check_output(['git','hash-object','src/msg/plugins/'+name+'.py'],text=True).strip() + assert actual==expected,(name,actual,expected) +assert segment(sources['orders'],functions(sources['orders'])['_subject']) == segment( + sources['store'],functions(sources['store'])['_subject']) + +# Move the unique in-process guard with the posting invariant, not with an API version. +escrow_path='src/msg/market/escrow.py' +escrow=read(escrow_path) +assert subprocess.check_output(['git','hash-object',escrow_path],text=True).strip() == '92e0bb595c8fec8665285f0d131630a52500d9fa' +guard_start=escrow.index('# Not a credential, account or public capability.') +guard_end=escrow.index('TRANSITIONS =') +guard=escrow[guard_start:guard_end] +assert '_ESCROW_WRITE = object()' in guard +assert guard.count('object()')==1 +after[escrow_path]=escrow[:guard_start]+'from msg.market.ledger import _ESCROW_WRITE\n\n'+escrow[guard_end:] + +providers={} +for name,(owner,mapping) in NAMES.items(): + exports={old:('msg.market.'+owner,new) for old,new in mapping.items()} + exports.update({constant:('msg.market.'+owner,constant) for constant in CONSTANTS[name]}) + if name=='store': + exports['_subject']=('msg.market.order_records','require_signed_subject') + providers['msg.plugins.'+name]=exports + +for name,(owner,mapping) in NAMES.items(): + text=sources[name] + fs,cs=functions(text),assignments(text) + renames={**mapping,**EXTRA_RENAMES.get(name,{})} + pieces=[HEADERS[name]] + pieces.extend(segment(text,cs[constant])+'\n' for constant in CONSTANTS[name]) + if name=='money': + pieces.append('\n'+guard) + for old,new in mapping.items(): + body=segment(text,fs[old]) + if name=='money' and old=='_post_transfer': + line=' from msg.market.escrow import _ESCROW_WRITE\n' + assert body.count(line)==1 + body=body.replace(line,'') + pieces.append('\n\n'+rename(body,renames)+'\n') + target='src/msg/market/'+owner+'.py' + assert not (ROOT/target).exists(),target + before[target]='' + after[target]=clean(''.join(pieces)) + for old,new in mapping.items(): + original=segment(text,fs[old]) + if name=='money' and old=='_post_transfer': + original=original.replace(' from msg.market.escrow import _ESCROW_WRITE\n','') + expected=ast.dump(ast.parse(rename(original,renames)),include_attributes=False) + actual=ast.dump(ast.parse(segment(after[target],functions(after[target])[new])),include_attributes=False) + assert actual==expected,(name,old) + audit.append(name+'.'+old+' -> '+owner+'.'+new) + edits=[(fs[old],'') for old in mapping]+[(cs[c],'') for c in CONSTANTS[name]] + if name=='store': + edits.append((fs['_subject'],'')) + adapter=splice(text,edits) + # Existing entrypoint code still names the compatibility imports. Each alias + # is the exact owned function, not a wrapper or a second implementation. + imports=['# Compatibility imports; shared implementation has one market owner.'] + for old,(module,new) in providers['msg.plugins.'+name].items(): + imports.append('from '+module+' import '+new+(' as '+old if new!=old else '')) + adapter=adapter.replace('def install(app):','\n'.join(imports)+'\n\n\ndef install(app):',1) + after['src/msg/plugins/'+name+'.py']=clean(adapter) + + +def rewire(text): + edits=[] + for node in ast.walk(ast.parse(text)): + if not isinstance(node,ast.ImportFrom) or node.module not in providers: + continue + groups={} + for item in node.names: + target,new=providers[node.module].get(item.name,(node.module,item.name)) + local=item.asname or item.name + groups.setdefault(target,[]).append(new+(' as '+local if new!=local else '')) + replacement=('\n'+' '*node.col_offset).join( + 'from '+module+' import '+', '.join(names) for module,names in groups.items()) + edits.append((node,replacement)) + return splice(text,edits) + + +# Only import wiring changes in existing market use cases and versioned handlers. +# Source outside these four entrypoints and market remains untouched. +paths={str(p.relative_to(ROOT)) for p in (ROOT/'src/msg/market').rglob('*.py')} +paths.update('src/msg/plugins/'+name+'.py' for name in NAMES) +paths.update(after) +for path in sorted(paths): + text=after[path] if path in after else read(path) + candidate=rewire(text) + if candidate!=before.get(path,''): + after[path]=clean(candidate) + + +class WithoutImports(ast.NodeTransformer): + def visit_ImportFrom(self,node): + return None + def visit_Import(self,node): + return None + + +def stable(node): + return ast.dump(WithoutImports().visit(copy.deepcopy(node)),include_attributes=False) + + +for name in NAMES: + old=functions(sources[name]) + new=functions(after['src/msg/plugins/'+name+'.py']) + removed=set(NAMES[name][1])|({'_subject'} if name=='store' else set()) + assert set(new)==set(old)-removed,(name,set(new),set(old)-removed) + for function in new: + assert stable(old[function])==stable(new[function]),(name,function) + audit.append(name+'.'+function+' body/schema unchanged') +for path,text in after.items(): + if not before.get(path) or not path.startswith('src/msg/market/'): + continue + left=ast.parse(before[path]);right=ast.parse(text) + if path==escrow_path: + left.body=[n for n in left.body if not (isinstance(n,ast.Assign) and + any(isinstance(t,ast.Name) and t.id=='_ESCROW_WRITE' for t in n.targets))] + assert stable(left)==stable(right),path + audit.append(path+' business AST unchanged') +for path,text in after.items(): + if path.startswith('src/msg/market/'): + assert not [n.module for n in ast.walk(ast.parse(text)) + if isinstance(n,ast.ImportFrom) and n.module in providers],path +# All old constants are moved verbatim; the changed authority import is singular. +assert after['src/msg/market/ledger.py'].count('_ESCROW_WRITE = object()')==1 +assert '_ESCROW_WRITE = object()' not in after[escrow_path] + +notes='docs/MARKET_BOUNDARY_PROGRESS.md' +after[notes]=read(notes)+''' + +## Implementation and source audit + +PR #168 extracts the existing shared implementation into `order_records`, +`catalog`, `ledger` and `managed_delivery`, with canonical public function names. +Old plugin names re-export those same function objects. All existing market use +cases, including function-local imports, and legacy plugin handlers use these +owners. The duplicate store subject check becomes the same order-subject check. + +The exact-base cloud assembly checks source blob identities, compares every +moved function after identifier-only renaming, and checks every remaining handler +and market state machine after ignoring import statements. SQL strings, schemas, +IDs, receipt purposes, positional order and existing algorithm bodies are not +rewritten. The only guard change is its owner: `ledger` defines `_ESCROW_WRITE` +once, escrow re-exports it, and protected posting reads that same object directly. +The guard is still supplied only by the existing checked escrow release path. + +No generated source is imported or executed during assembly. Cloud test jobs run +both new real-database contracts and existing published-version/recovery suites. +The assembly artifact contains the exact changed paths, audit and patch. Its +one-shot helper files are removed from the submitted tree before final validation. +No successful final-head result is claimed by this source-only audit. +''' + +out=Path('/tmp/market-boundary-assembly');out.mkdir(parents=True,exist_ok=True) +changed={path:text for path,text in after.items() if text!=before.get(path,'')} +patch=''.join(''.join(difflib.unified_diff(before.get(path,'').splitlines(True),text.splitlines(True), + fromfile='a/'+path if before.get(path) else '/dev/null',tofile='b/'+path)) for path,text in sorted(changed.items())) +for path,text in changed.items(): + (ROOT/path).write_text(text) +(out/'changes.patch').write_text(patch) +(out/'audit.json').write_text(json.dumps({'baseline_commit':BASE,'baseline_blobs':EXPECTED, + 'checked_invariants':audit,'changed_paths':sorted(changed), + 'patch_sha256':hashlib.sha256(patch.encode()).hexdigest()},indent=2)+'\n') diff --git a/.github/workflows/assemble-market-boundaries.yml b/.github/workflows/assemble-market-boundaries.yml new file mode 100644 index 00000000..732be59a --- /dev/null +++ b/.github/workflows/assemble-market-boundaries.yml @@ -0,0 +1,60 @@ +name: Assemble audited market boundary extraction +on: + push: + branches: [build/market-boundaries-168] + paths: + - .github/tools/assemble_market_boundaries.py + - .github/workflows/assemble-market-boundaries.yml +permissions: + contents: write +jobs: + assemble: + if: github.repository == 'TokenNotIncluded/msg.lmm.best' && github.ref == 'refs/heads/build/market-boundaries-168' + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - uses: actions/checkout@v7 + with: + fetch-depth: 0 + - name: Require exact regression source + env: + INPUT_SHA: ${{ github.sha }} + run: | + test "$(git rev-parse HEAD)" = "$INPUT_SHA" + git merge-base --is-ancestor 8692ad6c0056e793bbe7b0efc9f5fd0b08ab89c8 HEAD + git diff --exit-code 8692ad6c0056e793bbe7b0efc9f5fd0b08ab89c8 -- src tests docs + - name: Assemble and audit shared functions and unchanged state machines + run: python .github/tools/assemble_market_boundaries.py + - name: Commit only audited source and notes + run: | + git add src/msg/market src/msg/plugins/orders.py src/msg/plugins/money.py src/msg/plugins/delivery.py src/msg/plugins/store.py docs/MARKET_BOUNDARY_PROGRESS.md + git diff --cached --check + python - <<'PY' + import json,subprocess + from pathlib import Path + expected=set(json.loads(Path('/tmp/market-boundary-assembly/audit.json').read_text())['changed_paths']) + actual=set(subprocess.check_output(['git','diff','--cached','--name-only'],text=True).splitlines()) + assert actual==expected,(actual-expected,expected-actual) + PY + git -c user.name='github-actions[bot]' -c user.email='41898282+github-actions[bot]@users.noreply.github.com' commit -m 'refactor(market): own shared records posting and managed delivery independently of entrypoints' + - name: Publish only the dedicated build branch without force + env: + INPUT_SHA: ${{ github.sha }} + run: | + test "$(git ls-remote origin refs/heads/build/market-boundaries-168 | cut -f1)" = "$INPUT_SHA" + git push origin HEAD:refs/heads/build/market-boundaries-168 + python - <<'PY' + import json,os,subprocess + from pathlib import Path + git=lambda *args:subprocess.check_output(['git',*args],text=True).strip() + Path('/tmp/market-boundary-assembly/source.json').write_text(json.dumps({ + 'input':os.environ['INPUT_SHA'],'commit':git('rev-parse','HEAD'), + 'tree':git('rev-parse','HEAD^{tree}'),'dirty':bool(git('status','--porcelain'))},indent=2)+'\n') + PY + git archive --format=tar.gz -o /tmp/market-boundary-assembly/source.tar.gz HEAD + - uses: actions/upload-artifact@v4 + if: always() + with: + name: market-boundary-assembly + path: /tmp/market-boundary-assembly/ + if-no-files-found: error From f92390fe8f54fbf63fe21d770f37d6fdbb56190e Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 28 Sep 2026 21:06:54 +0000 Subject: [PATCH 011/136] refactor(market): own shared records posting and managed delivery independently of entrypoints --- docs/MARKET_BOUNDARY_PROGRESS.md | 23 ++++ src/msg/market/arbitration.py | 2 +- src/msg/market/catalog.py | 39 ++++++ src/msg/market/delivery.py | 6 +- src/msg/market/delivery_notifications.py | 4 +- src/msg/market/email.py | 2 +- src/msg/market/escrow.py | 13 +- src/msg/market/ledger.py | 145 +++++++++++++++++++++++ src/msg/market/managed_delivery.py | 137 +++++++++++++++++++++ src/msg/market/order_records.py | 69 +++++++++++ src/msg/market/orders.py | 6 +- src/msg/market/policy.py | 2 +- src/msg/market/rationale.py | 2 +- src/msg/market/targets.py | 4 +- src/msg/plugins/delivery.py | 142 +++------------------- src/msg/plugins/money.py | 145 +++-------------------- src/msg/plugins/orders.py | 72 ++--------- src/msg/plugins/store.py | 43 ++----- 18 files changed, 480 insertions(+), 376 deletions(-) create mode 100644 src/msg/market/catalog.py create mode 100644 src/msg/market/ledger.py create mode 100644 src/msg/market/managed_delivery.py create mode 100644 src/msg/market/order_records.py diff --git a/docs/MARKET_BOUNDARY_PROGRESS.md b/docs/MARKET_BOUNDARY_PROGRESS.md index c67cd19d..6f6e1f0a 100644 --- a/docs/MARKET_BOUNDARY_PROGRESS.md +++ b/docs/MARKET_BOUNDARY_PROGRESS.md @@ -55,3 +55,26 @@ account-scope, deterministic clearing, Bounty, redemption, storage and recovery suites. Full four-shard exact node-ID/conformance/build and applicable recovery checks remain required on the final synchronized head. No skips, weakened assertions, changed published versions or historical fact/schema migration. + + +## Implementation and source audit + +PR #168 extracts the existing shared implementation into `order_records`, +`catalog`, `ledger` and `managed_delivery`, with canonical public function names. +Old plugin names re-export those same function objects. All existing market use +cases, including function-local imports, and legacy plugin handlers use these +owners. The duplicate store subject check becomes the same order-subject check. + +The exact-base cloud assembly checks source blob identities, compares every +moved function after identifier-only renaming, and checks every remaining handler +and market state machine after ignoring import statements. SQL strings, schemas, +IDs, receipt purposes, positional order and existing algorithm bodies are not +rewritten. The only guard change is its owner: `ledger` defines `_ESCROW_WRITE` +once, escrow re-exports it, and protected posting reads that same object directly. +The guard is still supplied only by the existing checked escrow release path. + +No generated source is imported or executed during assembly. Cloud test jobs run +both new real-database contracts and existing published-version/recovery suites. +The assembly artifact contains the exact changed paths, audit and patch. Its +one-shot helper files are removed from the submitted tree before final validation. +No successful final-head result is claimed by this source-only audit. diff --git a/src/msg/market/arbitration.py b/src/msg/market/arbitration.py index 03e9d3a3..ba05d0c0 100644 --- a/src/msg/market/arbitration.py +++ b/src/msg/market/arbitration.py @@ -15,7 +15,7 @@ from msg.market.rationale import PINNED_REF from msg.market.rationale import verified as verify_rationale from msg.plugins.common import check_access, new_id, no_requirements -from msg.plugins.orders import _row, _subject +from msg.market.order_records import read_order as _row, require_signed_subject as _subject from msg.plugins.schemas import IDENTIFIER, REF, SIGNATURE, obj from msg.security.crypto import verify diff --git a/src/msg/market/catalog.py b/src/msg/market/catalog.py new file mode 100644 index 00000000..78fa2cb4 --- /dev/null +++ b/src/msg/market/catalog.py @@ -0,0 +1,39 @@ +"""Authorized catalog reads shared by published checkout versions. + +Version-specific schemas, creation and updates stay at catalog entrypoints. +""" +from __future__ import annotations +from msg.core.codec import loads +from msg.core.errors import Failure, require +from msg.core.models import ResourceRef +from msg.plugins.common import check_access, resolve +from msg.market.order_records import require_signed_subject + + +async def read_listing(app, ctx, request, tx, value, *, seller=False): + try: + rid = await resolve(tx, value) + resource = await tx.resource(rid) + require(resource.type == 'listing' and resource.parent == 't_store', 'listing_not_found') + await check_access(app, ctx, request, tx, rid, 'read') + if seller: + require(resource.owner == require_signed_subject(ctx), 'listing_not_found') + return resource + except Failure as exc: + if exc.code in {'not_found', 'permission_denied', 'credential_ceiling', + 'certificate_gate', 'ancestor_inactive'}: + raise Failure('listing_not_found') from None + raise + + +async def read_listing_body(app, tx, resource, revision=None): + rev = await tx.revision(ResourceRef(id=resource.id, revision=revision)) + body = loads(await app.contents.read_bytes(rev.content)) + # Listings deposited before the sale/bounty split were all sale listings. + return {'mode': 'sale', **body}, rev + + +async def read_package_record(tx, package_id): + return tx.one('''SELECT id,listing_id,listing_revision,seller,revision,kind,manifest, + payload_refs,digest,total_size,delivery_mode,deposited_at FROM store_packages WHERE id=?''', + (package_id,)) diff --git a/src/msg/market/delivery.py b/src/msg/market/delivery.py index 940468c9..2c916ad7 100644 --- a/src/msg/market/delivery.py +++ b/src/msg/market/delivery.py @@ -11,10 +11,10 @@ from msg.market.policy import contract from msg.market.targets import enqueue_notification, validate_target from msg.plugins.common import check_access, new_id -from msg.plugins.delivery import _buyer_order, _delivery -from msg.plugins.orders import _row, _subject, _viewer +from msg.market.managed_delivery import read_buyer_order as _buyer_order, read_delivery as _delivery +from msg.market.order_records import read_order as _row, require_signed_subject as _subject, require_order_viewer as _viewer from msg.plugins.schemas import IDENTIFIER, REF, obj -from msg.plugins.store import _package_row +from msg.market.catalog import read_package_record as _package_row INLINE = 65536 PART = 65536 diff --git a/src/msg/market/delivery_notifications.py b/src/msg/market/delivery_notifications.py index 9a2b4f0c..5b2d0726 100644 --- a/src/msg/market/delivery_notifications.py +++ b/src/msg/market/delivery_notifications.py @@ -85,8 +85,8 @@ async def queue_notification(app, ctx, request, tx, order, *, enabled): async def project_notification(app, tx, job, principal): """Reconstruct the allowlisted message only after every live owner check.""" - from msg.plugins.orders import _row - from msg.plugins.delivery import _delivery, _verified_delivery + from msg.market.order_records import read_order as _row + from msg.market.managed_delivery import read_delivery as _delivery, verify_managed_delivery as _verified_delivery require(mail_enabled(app), 'mail_disabled') require((job.operation, job.arguments.get('contract_version')) in { ('orders.buy', 2), ('delivery.notify', 1)}, 'invalid_delivery_notification') diff --git a/src/msg/market/email.py b/src/msg/market/email.py index b1000832..0902b251 100644 --- a/src/msg/market/email.py +++ b/src/msg/market/email.py @@ -15,7 +15,7 @@ from msg.core.models import EffectJob, HandlerOutput from msg.market.targets import enqueue_notification, mail_enabled, save_target, validate_target from msg.plugins.common import new_id -from msg.plugins.orders import _row, _subject +from msg.market.order_records import read_order as _row, require_signed_subject as _subject from msg.plugins.schemas import IDENTIFIER, obj diff --git a/src/msg/market/escrow.py b/src/msg/market/escrow.py index 92e0bb59..39c4237d 100644 --- a/src/msg/market/escrow.py +++ b/src/msg/market/escrow.py @@ -9,12 +9,11 @@ from msg.core.requests import signing_bytes from msg.market.policy import contract, delivery_snapshot from msg.plugins.common import new_id -from msg.plugins.money import CURRENCY_ID, _balance, _post_transfer +from msg.market.ledger import CURRENCY_ID, balance as _balance, post_transfer as _post_transfer from msg.security.crypto import verify -# Not a credential, account or public capability. Only trusted in-process market -# code can reach this token; ordinary signed money.transfer never receives it. -_ESCROW_WRITE = object() +from msg.market.ledger import _ESCROW_WRITE + TRANSITIONS = { 'created': {'funded', 'cancelled'}, 'funded': {'delivered', 'refunded', 'disputed'}, @@ -132,7 +131,7 @@ async def resolve_due(app, *, limit=100): services/manual ciphertext refund; subjective complaints and expired panels remain held. This function has no external side effect. """ - from msg.plugins.orders import _row + from msg.market.order_records import read_order as _row now = app.clock() changed = [] async with app.metadata.transaction(write=True) as tx: @@ -200,8 +199,8 @@ def __init__(self, app): async def settle(self, ctx, request, tx, *, reason, order_id=None): # Import handlers' read-only projections, never their write entry points. - from msg.plugins.delivery import _delivery, _verified_delivery - from msg.plugins.orders import _row, _subject + from msg.market.managed_delivery import read_delivery as _delivery, verify_managed_delivery as _verified_delivery + from msg.market.order_records import read_order as _row, require_signed_subject as _subject buyer=_subject(ctx) require(order_id is None or reason == 'checkout_accept', 'escrow_decision_mismatch') order=_row(tx,order_id or request.arguments['order_id'],buyer) diff --git a/src/msg/market/ledger.py b/src/msg/market/ledger.py new file mode 100644 index 00000000..93b0901c --- /dev/null +++ b/src/msg/market/ledger.py @@ -0,0 +1,145 @@ +"""One append-only ledger posting path in the caller's transaction. + +This module grants no commit rights. Public transfers cannot supply the internal +escrow guard; Root issuance remains a separate local-console use case. +""" +from __future__ import annotations +from uuid import uuid4 +from msg.constants import ROOT_SUBJECT +from msg.core.codec import canonical, digest, wire +from msg.core.errors import require +from msg.core.models import AccessRequirement, SignatureProof + +CURRENCY_ID = 'primary' +CODE = 'MSG' +SCALE = 6 +MAX_MINOR = 2**63 - 1 +POLICY_VERSION = 2 +POLICY_DIGEST = digest({'version': POLICY_VERSION, 'currency_id': CURRENCY_ID, + 'scale': SCALE, 'transfer_fee': 0, 'allow_overdraft': False, + 'posting_identity': 'actor/request_id/entry_key'}) + +# Not a credential, account or public capability. Only trusted in-process market +# code can reach this token; ordinary signed money.transfer never receives it. +_ESCROW_WRITE = object() + + +async def account_requirements(request, tx): + """Require the credential's scope to cover its account, even on replay. + + An operation name in a listing-scoped key is not permission to spend the + owner's balance or read unrelated private orders. Anonymous calls still + reach the handler's uniform not-found/signature response. + """ + if request.proof is None: + return () + key_id = (request.proof.signature.key_id if isinstance(request.proof, SignatureProof) + else request.proof.credential_id) + credential = await tx.credential(key_id) + return (AccessRequirement(resource_id=request.subject or credential.subject_id, + operation=f'{request.operation}@{request.contract_version}', check='read'),) + + +def require_money_subject(ctx): + require(ctx.principal.subject is not None and + ctx.principal.actor == ctx.principal.subject, 'money_subject_required') + return ctx.principal.subject + + +def checked_amount(value): + require(type(value) is int and 0 < value <= MAX_MINOR, 'invalid_money_amount') + return value + + +def balance(tx, subject): + debit = tx.one("SELECT COALESCE(SUM(amount_minor),0) FROM money_ledger WHERE debit_account=?", + (subject,))[0] + credit = tx.one("SELECT COALESCE(SUM(amount_minor),0) FROM money_ledger WHERE credit_account=?", + (subject,))[0] + return int(credit) - int(debit) + + +def total_supply(tx): + issued = tx.one("SELECT COALESCE(SUM(amount_minor),0) FROM money_ledger WHERE kind='mint'")[0] + burned = tx.one("SELECT COALESCE(SUM(amount_minor),0) FROM money_ledger WHERE kind='burn'")[0] + return int(issued) - int(burned) + + +def clearing_decision(*, kind, sender, recipient, amount, sender_balance, + recipient_balance, supply, local_issuer=False): + """Pure policy v2: no clock, UUID, subject privileges or discretionary input.""" + amount = checked_amount(amount) + require(all(type(value) is int and 0 <= value <= MAX_MINOR + for value in (sender_balance, recipient_balance, supply)), 'invalid_money_state') + require(kind in {'mint', 'burn', 'transfer', 'redeem', 'refund'}, 'invalid_money_kind') + if kind in {'mint', 'burn'}: + require(local_issuer, 'root_local_only') + require((kind == 'mint' and sender is None and recipient == ROOT_SUBJECT) or + (kind == 'burn' and sender == ROOT_SUBJECT and recipient is None), + 'invalid_money_accounts') + else: + require(sender is not None and recipient is not None, 'invalid_money_accounts') + require(sender != recipient, 'self_transfer_forbidden') + if sender in {ROOT_SUBJECT, '@root'}: + require(local_issuer, 'root_local_only') + if sender is not None: + require(sender_balance >= amount, 'insufficient_funds') + if recipient is not None: + require(recipient_balance + amount <= MAX_MINOR, 'money_overflow') + if kind == 'burn': + require(supply >= amount, 'invalid_money_state') + if kind == 'mint': + require(supply + amount <= MAX_MINOR, 'money_overflow') + return {'kind': kind, 'debit_account': sender, 'credit_account': recipient, + 'amount_minor': amount, 'currency_id': CURRENCY_ID, + 'policy_version': POLICY_VERSION, 'policy_digest': POLICY_DIGEST} + + +def append_entry(tx, *, sender, recipient, amount, actor, request_id, now, + receipt_signer, reference=None, kind='transfer', entry_key='primary', + _local_issuer=False): + """Append one named leg in the caller's serialized transaction, never commit. + + Named legs permit atomic fund+release without manufacturing request IDs. + Historical receipts remain byte-for-byte untouched by the schema upgrade. + Only the separate local RootAdmin use case supplies _local_issuer. + """ + require(not _local_issuer or actor == ROOT_SUBJECT, 'root_local_only') + clearing_decision(kind=kind, sender=sender, recipient=recipient, amount=amount, + sender_balance=balance(tx, sender) if sender else 0, + recipient_balance=balance(tx, recipient) if recipient else 0, + supply=total_supply(tx), local_issuer=_local_issuer) + for account in {sender, recipient} - {None}: + tx.execute('INSERT INTO money_accounts(subject_id,currency_id) VALUES (?,?) ' + 'ON CONFLICT(subject_id,currency_id) DO NOTHING', + (account, CURRENCY_ID), write=True) + transaction_id = 'lt_' + uuid4().hex + sequence = tx.one("SELECT nextval(pg_get_serial_sequence('money_ledger','seq'))")[0] + body = {'transaction_id': transaction_id, 'kind': kind, 'currency_id': CURRENCY_ID, + 'amount_minor': amount, 'from_subject': sender, 'to_subject': recipient, + 'actor': actor, 'request_id': request_id, 'entry_key': entry_key, + 'reference': reference, 'committed_at': wire(now), + 'policy_version': POLICY_VERSION, 'policy_digest': POLICY_DIGEST, + 'ledger_sequence': sequence} + receipt = {'body': body, + 'signature': wire(receipt_signer.sign(canonical(body), purpose='money-receipt'))} + tx.execute('''INSERT INTO money_ledger + (seq,id,kind,currency_id,amount_minor,debit_account,credit_account,actor,request_id, + reference,committed_at,policy_version,policy_digest,receipt,entry_key) + VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)''', + (sequence,transaction_id,kind,CURRENCY_ID,amount,sender,recipient,actor, + request_id,reference,wire(now),POLICY_VERSION,POLICY_DIGEST, + canonical(receipt).decode(),entry_key), write=True) + return receipt + + +def post_transfer(tx, *, sender, recipient, amount, actor, request_id, now, + receipt_signer, reference=None, kind='transfer', entry_key='primary', + escrow_authority=None): + require(kind in {'transfer', 'redeem', 'refund'}, 'invalid_money_kind') + account = tx.one('SELECT kind FROM ledger_accounts WHERE id=?', (sender,)) + if account and account[0] == 'order_escrow': + require(escrow_authority is _ESCROW_WRITE, 'escrow_release_forbidden') + return append_entry(tx, sender=sender, recipient=recipient, amount=amount, + actor=actor, request_id=request_id, now=now, receipt_signer=receipt_signer, + reference=reference, kind=kind, entry_key=entry_key) diff --git a/src/msg/market/managed_delivery.py b/src/msg/market/managed_delivery.py new file mode 100644 index 00000000..e04b8c9e --- /dev/null +++ b/src/msg/market/managed_delivery.py @@ -0,0 +1,137 @@ +"""Shared bounded managed-package verification and local preparation. + +These preserve the managed checkout contracts; versioned manual/service delivery, +arbitration and operation registration retain their existing entrypoints. +""" +from __future__ import annotations +from msg.core.codec import b64, canonical, decode, digest, loads, wire +from msg.core.errors import require +from msg.core.models import BlobRef +from msg.plugins.common import new_id +from msg.market.ledger import CURRENCY_ID, balance +from msg.market.catalog import read_package_record +from msg.market.order_records import read_order +from msg.market.delivery_targets import validate_target + +MAX_INLINE_BYTES = 1024 * 1024 +_COLUMNS = ('id', 'order_id', 'recipient_subject', 'kind', 'payload_refs', + 'manifest', 'package_digest', 'delivery_digest', 'channel', + 'state', 'prepared_at', 'claimed_at', 'receipt') + + +def read_delivery(tx, order_id): + row = tx.one('''SELECT id,order_id,recipient_subject,kind,payload_refs, + manifest,package_digest,delivery_digest,channel,state,prepared_at, + claimed_at,receipt FROM store_deliveries WHERE order_id=?''', (order_id,)) + if row is None: + return None + result = dict(zip(_COLUMNS, row)) + result['payload_refs'] = loads(result['payload_refs']) + result['manifest'] = loads(result['manifest']) + result['receipt'] = loads(result['receipt']) if result['receipt'] else None + return result + + +def read_buyer_order(tx, order_id, buyer): + row = read_order(tx, order_id, buyer) + require(row['buyer'] == buyer, 'order_not_found') + return row + + +async def verify_managed_delivery(app, tx, order, delivery): + validate_target(order, delivery) + require(delivery is not None and + delivery['recipient_subject'] == order['buyer'] and + delivery['package_digest'] == order['package_digest'] and + delivery['channel'] == 'site', 'delivery_mismatch') + kind, manifest, refs = await read_managed_package(app, tx, order) + body = {'delivery_id': delivery['id'], 'order_id': order['id'], + 'recipient_subject': order['buyer'], 'kind': kind, + 'manifest': manifest, 'payload_refs': refs, + 'package_digest': order['package_digest'], 'channel': 'site'} + require(delivery['kind'] == kind and delivery['manifest'] == manifest and + delivery['payload_refs'] == refs and + delivery['delivery_digest'] == digest(body), 'delivery_mismatch') + + +async def read_managed_package(app, tx, order): + require(order['package_id'] and order['package_revision'] and + order['package_digest'], 'delivery_package_mismatch') + row = await read_package_record(tx, order['package_id']) + require(row is not None and row[1] == order['listing_id'] and + row[3] == order['seller'] and row[4] == order['package_revision'] and + row[8] == order['package_digest'] and row[10] == 'managed_instant' and + row[5] in {'file', 'bundle'}, 'delivery_package_mismatch') + manifest, refs = loads(row[6]), loads(row[7]) + snapshot = {'seller': row[3], 'listing_id': row[1], + 'listing_revision': row[2], 'revision': row[4], 'kind': row[5], + 'manifest': manifest, 'payload_refs': refs, + 'total_size': row[9], 'delivery_mode': row[10]} + require(digest(snapshot) == order['package_digest'] and + row[9] <= MAX_INLINE_BYTES and row[9] >= 0, + 'delivery_package_mismatch') + require(refs and (row[5] == 'bundle' or len(refs) == 1), + 'delivery_package_mismatch') + total = 0 + for ref in refs: + require(isinstance(ref, dict) and isinstance(ref.get('blob'), dict), + 'delivery_package_mismatch') + blob = decode(BlobRef, ref['blob']) + require(blob.size >= 0 and blob.size <= MAX_INLINE_BYTES, + 'delivery_package_mismatch') + data = await app.contents.read_bytes(blob, limit=MAX_INLINE_BYTES) + require(len(data) == blob.size and digest(data) == blob.digest, + 'delivery_package_mismatch') + total += blob.size + require(total == row[9], 'delivery_package_mismatch') + return row[5], manifest, refs + + +async def read_managed_payloads(app, delivery): + files = [] + for ref in delivery['payload_refs']: + blob = decode(BlobRef, ref['blob']) + data = await app.contents.read_bytes(blob, limit=MAX_INLINE_BYTES) + require(len(data) == blob.size and digest(data) == blob.digest, + 'delivery_package_mismatch') + files.append({'id': ref['id'], 'revision': ref['revision'], + 'media_type': blob.media_type, 'digest': blob.digest, + 'size': blob.size, 'data': b64(data)}) + return files + + +async def prepare_managed(app, ctx, tx, order): + """Prepare inside the caller's transaction; never authenticate or commit here.""" + buyer = order['buyer'] + require(order['state'] == 'funded' and order['delivered_at'] is None, + 'order_not_deliverable') + validate_target(order) + require(order['currency_id'] == CURRENCY_ID and + balance(tx, order['escrow_subject']) == order['total_price_minor'], + 'escrow_balance_mismatch') + kind, manifest, refs = await read_managed_package(app, tx, order) + delivery_id = new_id('dlv') + body = {'delivery_id': delivery_id, 'order_id': order['id'], + 'recipient_subject': buyer, 'kind': kind, + 'manifest': manifest, 'payload_refs': refs, + 'package_digest': order['package_digest'], 'channel': 'site'} + delivery_digest = digest(body) + tx.execute('''INSERT INTO store_deliveries + (id,order_id,recipient_subject,kind,payload_refs,manifest, + package_digest,delivery_digest,channel,state,prepared_at,claimed_at,receipt) + VALUES (?,?,?,?,?,?,?,?,?,'prepared',?,NULL,NULL)''', + (delivery_id, order['id'], buyer, kind, canonical(refs).decode(), + canonical(manifest).decode(), order['package_digest'], + delivery_digest, 'site', wire(ctx.now)), write=True) + changed = tx.execute('''UPDATE store_orders SET state='delivered',delivered_at=? + WHERE id=? AND buyer=? AND state='funded' AND delivered_at IS NULL''', + (wire(ctx.now), order['id'], buyer), write=True) + require(changed.rowcount == 1, 'order_not_deliverable') + return read_delivery(tx, order['id']) + + +def delivery_summary(delivery): + return {'delivery_id': delivery['id'], 'order_id': delivery['order_id'], + 'state': delivery['state'], 'channel': delivery['channel'], + 'package_digest': delivery['package_digest'], + 'delivery_digest': delivery['delivery_digest']} diff --git a/src/msg/market/order_records.py b/src/msg/market/order_records.py new file mode 100644 index 00000000..3f3e3136 --- /dev/null +++ b/src/msg/market/order_records.py @@ -0,0 +1,69 @@ +"""Shared order identity, authorized records and legacy-compatible projections. + +These functions own record access, not operation registration or settlement. +""" +from __future__ import annotations +import base64 +import os +from msg.core.codec import loads +from msg.core.errors import require + +_COLUMNS = ('id', 'buyer', 'seller', 'listing_id', 'listing_revision', + 'package_id', 'package_revision', 'package_digest', 'quantity', + 'unit_price_minor', 'total_price_minor', 'currency_id', + 'escrow_subject', 'escrow_policy', 'dispute_policy', 'terms_digest', + 'delivery_target', 'payment_intent_digest', 'payment_transaction_id', + 'state', 'created_at', 'funded_at', 'delivered_at', 'settled_at', + 'receipt_refs') + + +def require_signed_subject(ctx): + subject = ctx.principal.subject + require(subject is not None and ctx.principal.actor == subject and + ctx.principal.method == 'signature', 'signature_required') + return subject + + +def require_order_viewer(ctx): + subject = ctx.principal.subject + require(subject is not None and ctx.principal.actor == subject, + 'order_not_found') + return subject + + +def new_order_id(): + # 160 random bits, unguessable even if an attacker sees other order IDs. + return 'ord_' + base64.b32encode(os.urandom(20)).decode('ascii').rstrip('=').lower() + + +def read_order(tx, order_id, viewer): + row = tx.one('''SELECT id,buyer,seller,listing_id,listing_revision, + package_id,package_revision,package_digest,quantity,unit_price_minor, + total_price_minor,currency_id,escrow_subject,escrow_policy, + dispute_policy,terms_digest,delivery_target,payment_intent_digest, + payment_transaction_id,state,created_at,funded_at,delivered_at, + settled_at,receipt_refs FROM store_orders WHERE id=?''', (order_id,)) + # A valid ID is not an access grant. Keep nonexistent and unauthorized alike. + require(row is not None and viewer in row[1:3], 'order_not_found') + result = dict(zip(_COLUMNS, row)) + result['delivery_target'] = loads(result['delivery_target']) + result['receipt_refs'] = loads(result['receipt_refs']) + return result + + +def legacy_order_view(row, viewer): + keys = ('id', 'buyer', 'seller', 'listing_id', 'listing_revision', + 'package_revision', 'package_digest', 'quantity', + 'unit_price_minor', 'total_price_minor', 'currency_id', + 'escrow_policy', 'dispute_policy', 'terms_digest', 'state', + 'created_at', 'funded_at', 'delivered_at', 'settled_at') + result = {key: row[key] for key in keys} + result['payment_status'] = ('refunded' if row['state'] == 'refunded' else + 'funded' if row['payment_transaction_id'] else + 'pending') + result['delivery_channel'] = row['delivery_target']['channel'] + if viewer == row['buyer']: + result['delivery_target'] = row['delivery_target'] + result['payment_intent_digest'] = row['payment_intent_digest'] + result['receipt_refs'] = row['receipt_refs'] + return result diff --git a/src/msg/market/orders.py b/src/msg/market/orders.py index 308c2d9f..d7b7c7c7 100644 --- a/src/msg/market/orders.py +++ b/src/msg/market/orders.py @@ -11,10 +11,10 @@ from msg.market.targets import ( enqueue_notification, notification_view, save_target, target_for, validate_target, ) -from msg.plugins.money import CURRENCY_ID, MAX_MINOR, _post_transfer -from msg.plugins.orders import _order_id, _row, _subject, _view, _viewer +from msg.market.ledger import CURRENCY_ID, MAX_MINOR, post_transfer as _post_transfer +from msg.market.order_records import new_order_id as _order_id, read_order as _row, require_signed_subject as _subject, legacy_order_view as _view, require_order_viewer as _viewer from msg.plugins.schemas import IDENTIFIER, obj -from msg.plugins.store import _body, _listing, _package_row +from msg.market.catalog import read_listing_body as _body, read_listing as _listing, read_package_record as _package_row HASH = {'type': 'string', 'pattern': '^sha256:[a-f0-9]{64}$'} ORDER = obj({'order_id': IDENTIFIER}, ('order_id',)) diff --git a/src/msg/market/policy.py b/src/msg/market/policy.py index 3b414cca..8d7c25e1 100644 --- a/src/msg/market/policy.py +++ b/src/msg/market/policy.py @@ -79,7 +79,7 @@ def contract(tx, order_id): def delivery_snapshot(tx, order_id): """Commit delivery identity and prepared bytes, excluding subsequent buyer ACK.""" - from msg.plugins.delivery import _delivery + from msg.market.managed_delivery import read_delivery as _delivery delivery = _delivery(tx, order_id) return digest(None if delivery is None else { key: value for key, value in delivery.items() diff --git a/src/msg/market/rationale.py b/src/msg/market/rationale.py index 291da1d2..ecd18b4c 100644 --- a/src/msg/market/rationale.py +++ b/src/msg/market/rationale.py @@ -10,7 +10,7 @@ from msg.core.models import BlobRef, HandlerOutput, ResourceRef from msg.market.delivery import verify_blob from msg.plugins.common import check_access -from msg.plugins.orders import _subject +from msg.market.order_records import require_signed_subject as _subject from msg.plugins.schemas import IDENTIFIER, obj PINNED_REF = obj({'id': IDENTIFIER, 'revision': IDENTIFIER}, ('id', 'revision')) diff --git a/src/msg/market/targets.py b/src/msg/market/targets.py index 1833d0aa..a4226be6 100644 --- a/src/msg/market/targets.py +++ b/src/msg/market/targets.py @@ -106,8 +106,8 @@ async def enqueue_notification(app, tx, ctx, request, order): async def render_notification(app, tx, job): - from msg.plugins.delivery import _delivery - from msg.plugins.orders import _row + from msg.market.managed_delivery import read_delivery as _delivery + from msg.market.order_records import read_order as _row order = _row(tx, job.arguments['order_id'], job.principal.subject) require(order['buyer'] == job.principal.subject and job.principal.actor == order['buyer'], 'delivery_recipient_mismatch') diff --git a/src/msg/plugins/delivery.py b/src/msg/plugins/delivery.py index 4500a65e..fd01f408 100644 --- a/src/msg/plugins/delivery.py +++ b/src/msg/plugins/delivery.py @@ -12,136 +12,24 @@ from msg.core.errors import require from msg.core.models import BlobRef, HandlerOutput from msg.plugins.common import new_id, registration -from msg.plugins.money import CURRENCY_ID, _balance, account_requirements +from msg.market.ledger import CURRENCY_ID, balance as _balance, account_requirements from msg.market.escrow import EscrowEngine from msg.market.delivery_targets import validate_target -from msg.plugins.orders import _row as order_row, _subject +from msg.market.order_records import read_order as order_row, require_signed_subject as _subject from msg.plugins.schemas import IDENTIFIER, obj -from msg.plugins.store import _package_row - - -MAX_INLINE_BYTES = 1024 * 1024 -_COLUMNS = ('id', 'order_id', 'recipient_subject', 'kind', 'payload_refs', - 'manifest', 'package_digest', 'delivery_digest', 'channel', - 'state', 'prepared_at', 'claimed_at', 'receipt') - - -def _delivery(tx, order_id): - row = tx.one('''SELECT id,order_id,recipient_subject,kind,payload_refs, - manifest,package_digest,delivery_digest,channel,state,prepared_at, - claimed_at,receipt FROM store_deliveries WHERE order_id=?''', (order_id,)) - if row is None: - return None - result = dict(zip(_COLUMNS, row)) - result['payload_refs'] = loads(result['payload_refs']) - result['manifest'] = loads(result['manifest']) - result['receipt'] = loads(result['receipt']) if result['receipt'] else None - return result - - -def _buyer_order(tx, order_id, buyer): - row = order_row(tx, order_id, buyer) - require(row['buyer'] == buyer, 'order_not_found') - return row - - -async def _verified_delivery(app, tx, order, delivery): - validate_target(order, delivery) - require(delivery is not None and - delivery['recipient_subject'] == order['buyer'] and - delivery['package_digest'] == order['package_digest'] and - delivery['channel'] == 'site', 'delivery_mismatch') - kind, manifest, refs = await _package(app, tx, order) - body = {'delivery_id': delivery['id'], 'order_id': order['id'], - 'recipient_subject': order['buyer'], 'kind': kind, - 'manifest': manifest, 'payload_refs': refs, - 'package_digest': order['package_digest'], 'channel': 'site'} - require(delivery['kind'] == kind and delivery['manifest'] == manifest and - delivery['payload_refs'] == refs and - delivery['delivery_digest'] == digest(body), 'delivery_mismatch') - - -async def _package(app, tx, order): - require(order['package_id'] and order['package_revision'] and - order['package_digest'], 'delivery_package_mismatch') - row = await _package_row(tx, order['package_id']) - require(row is not None and row[1] == order['listing_id'] and - row[3] == order['seller'] and row[4] == order['package_revision'] and - row[8] == order['package_digest'] and row[10] == 'managed_instant' and - row[5] in {'file', 'bundle'}, 'delivery_package_mismatch') - manifest, refs = loads(row[6]), loads(row[7]) - snapshot = {'seller': row[3], 'listing_id': row[1], - 'listing_revision': row[2], 'revision': row[4], 'kind': row[5], - 'manifest': manifest, 'payload_refs': refs, - 'total_size': row[9], 'delivery_mode': row[10]} - require(digest(snapshot) == order['package_digest'] and - row[9] <= MAX_INLINE_BYTES and row[9] >= 0, - 'delivery_package_mismatch') - require(refs and (row[5] == 'bundle' or len(refs) == 1), - 'delivery_package_mismatch') - total = 0 - for ref in refs: - require(isinstance(ref, dict) and isinstance(ref.get('blob'), dict), - 'delivery_package_mismatch') - blob = decode(BlobRef, ref['blob']) - require(blob.size >= 0 and blob.size <= MAX_INLINE_BYTES, - 'delivery_package_mismatch') - data = await app.contents.read_bytes(blob, limit=MAX_INLINE_BYTES) - require(len(data) == blob.size and digest(data) == blob.digest, - 'delivery_package_mismatch') - total += blob.size - require(total == row[9], 'delivery_package_mismatch') - return row[5], manifest, refs - - -async def _payload(app, delivery): - files = [] - for ref in delivery['payload_refs']: - blob = decode(BlobRef, ref['blob']) - data = await app.contents.read_bytes(blob, limit=MAX_INLINE_BYTES) - require(len(data) == blob.size and digest(data) == blob.digest, - 'delivery_package_mismatch') - files.append({'id': ref['id'], 'revision': ref['revision'], - 'media_type': blob.media_type, 'digest': blob.digest, - 'size': blob.size, 'data': b64(data)}) - return files - - -async def prepare_managed(app, ctx, tx, order): - """Prepare inside the caller's transaction; never authenticate or commit here.""" - buyer = order['buyer'] - require(order['state'] == 'funded' and order['delivered_at'] is None, - 'order_not_deliverable') - validate_target(order) - require(order['currency_id'] == CURRENCY_ID and - _balance(tx, order['escrow_subject']) == order['total_price_minor'], - 'escrow_balance_mismatch') - kind, manifest, refs = await _package(app, tx, order) - delivery_id = new_id('dlv') - body = {'delivery_id': delivery_id, 'order_id': order['id'], - 'recipient_subject': buyer, 'kind': kind, - 'manifest': manifest, 'payload_refs': refs, - 'package_digest': order['package_digest'], 'channel': 'site'} - delivery_digest = digest(body) - tx.execute('''INSERT INTO store_deliveries - (id,order_id,recipient_subject,kind,payload_refs,manifest, - package_digest,delivery_digest,channel,state,prepared_at,claimed_at,receipt) - VALUES (?,?,?,?,?,?,?,?,?,'prepared',?,NULL,NULL)''', - (delivery_id, order['id'], buyer, kind, canonical(refs).decode(), - canonical(manifest).decode(), order['package_digest'], - delivery_digest, 'site', wire(ctx.now)), write=True) - changed = tx.execute('''UPDATE store_orders SET state='delivered',delivered_at=? - WHERE id=? AND buyer=? AND state='funded' AND delivered_at IS NULL''', - (wire(ctx.now), order['id'], buyer), write=True) - require(changed.rowcount == 1, 'order_not_deliverable') - return _delivery(tx, order['id']) - - -def delivery_summary(delivery): - return {'delivery_id': delivery['id'], 'order_id': delivery['order_id'], - 'state': delivery['state'], 'channel': delivery['channel'], - 'package_digest': delivery['package_digest'], - 'delivery_digest': delivery['delivery_digest']} +from msg.market.catalog import read_package_record as _package_row + + +# Compatibility imports; shared implementation has one market owner. +from msg.market.managed_delivery import read_delivery as _delivery +from msg.market.managed_delivery import read_buyer_order as _buyer_order +from msg.market.managed_delivery import verify_managed_delivery as _verified_delivery +from msg.market.managed_delivery import read_managed_package as _package +from msg.market.managed_delivery import read_managed_payloads as _payload +from msg.market.managed_delivery import prepare_managed +from msg.market.managed_delivery import delivery_summary +from msg.market.managed_delivery import MAX_INLINE_BYTES +from msg.market.managed_delivery import _COLUMNS def install(app): diff --git a/src/msg/plugins/money.py b/src/msg/plugins/money.py index 424a03f1..89f25df7 100644 --- a/src/msg/plugins/money.py +++ b/src/msg/plugins/money.py @@ -16,136 +16,21 @@ from msg.plugins.schemas import IDENTIFIER, obj -CURRENCY_ID = 'primary' -CODE = 'MSG' -SCALE = 6 -MAX_MINOR = 2**63 - 1 -POLICY_VERSION = 2 -POLICY_DIGEST = digest({'version': POLICY_VERSION, 'currency_id': CURRENCY_ID, - 'scale': SCALE, 'transfer_fee': 0, 'allow_overdraft': False, - 'posting_identity': 'actor/request_id/entry_key'}) - - -async def account_requirements(request, tx): - """Require the credential's scope to cover its account, even on replay. - - An operation name in a listing-scoped key is not permission to spend the - owner's balance or read unrelated private orders. Anonymous calls still - reach the handler's uniform not-found/signature response. - """ - if request.proof is None: - return () - key_id = (request.proof.signature.key_id if isinstance(request.proof, SignatureProof) - else request.proof.credential_id) - credential = await tx.credential(key_id) - return (AccessRequirement(resource_id=request.subject or credential.subject_id, - operation=f'{request.operation}@{request.contract_version}', check='read'),) - - -def _owner(ctx): - require(ctx.principal.subject is not None and - ctx.principal.actor == ctx.principal.subject, 'money_subject_required') - return ctx.principal.subject - - -def _amount(value): - require(type(value) is int and 0 < value <= MAX_MINOR, 'invalid_money_amount') - return value - - -def _balance(tx, subject): - debit = tx.one("SELECT COALESCE(SUM(amount_minor),0) FROM money_ledger WHERE debit_account=?", - (subject,))[0] - credit = tx.one("SELECT COALESCE(SUM(amount_minor),0) FROM money_ledger WHERE credit_account=?", - (subject,))[0] - return int(credit) - int(debit) - - -def _supply(tx): - issued = tx.one("SELECT COALESCE(SUM(amount_minor),0) FROM money_ledger WHERE kind='mint'")[0] - burned = tx.one("SELECT COALESCE(SUM(amount_minor),0) FROM money_ledger WHERE kind='burn'")[0] - return int(issued) - int(burned) - - -def clearing_decision(*, kind, sender, recipient, amount, sender_balance, - recipient_balance, supply, local_issuer=False): - """Pure policy v2: no clock, UUID, subject privileges or discretionary input.""" - amount = _amount(amount) - require(all(type(value) is int and 0 <= value <= MAX_MINOR - for value in (sender_balance, recipient_balance, supply)), 'invalid_money_state') - require(kind in {'mint', 'burn', 'transfer', 'redeem', 'refund'}, 'invalid_money_kind') - if kind in {'mint', 'burn'}: - require(local_issuer, 'root_local_only') - require((kind == 'mint' and sender is None and recipient == ROOT_SUBJECT) or - (kind == 'burn' and sender == ROOT_SUBJECT and recipient is None), - 'invalid_money_accounts') - else: - require(sender is not None and recipient is not None, 'invalid_money_accounts') - require(sender != recipient, 'self_transfer_forbidden') - if sender in {ROOT_SUBJECT, '@root'}: - require(local_issuer, 'root_local_only') - if sender is not None: - require(sender_balance >= amount, 'insufficient_funds') - if recipient is not None: - require(recipient_balance + amount <= MAX_MINOR, 'money_overflow') - if kind == 'burn': - require(supply >= amount, 'invalid_money_state') - if kind == 'mint': - require(supply + amount <= MAX_MINOR, 'money_overflow') - return {'kind': kind, 'debit_account': sender, 'credit_account': recipient, - 'amount_minor': amount, 'currency_id': CURRENCY_ID, - 'policy_version': POLICY_VERSION, 'policy_digest': POLICY_DIGEST} - - -def _post_entry(tx, *, sender, recipient, amount, actor, request_id, now, - receipt_signer, reference=None, kind='transfer', entry_key='primary', - _local_issuer=False): - """Append one named leg in the caller's serialized transaction, never commit. - - Named legs permit atomic fund+release without manufacturing request IDs. - Historical receipts remain byte-for-byte untouched by the schema upgrade. - Only the separate local RootAdmin use case supplies _local_issuer. - """ - require(not _local_issuer or actor == ROOT_SUBJECT, 'root_local_only') - clearing_decision(kind=kind, sender=sender, recipient=recipient, amount=amount, - sender_balance=_balance(tx, sender) if sender else 0, - recipient_balance=_balance(tx, recipient) if recipient else 0, - supply=_supply(tx), local_issuer=_local_issuer) - for account in {sender, recipient} - {None}: - tx.execute('INSERT INTO money_accounts(subject_id,currency_id) VALUES (?,?) ' - 'ON CONFLICT(subject_id,currency_id) DO NOTHING', - (account, CURRENCY_ID), write=True) - transaction_id = 'lt_' + uuid4().hex - sequence = tx.one("SELECT nextval(pg_get_serial_sequence('money_ledger','seq'))")[0] - body = {'transaction_id': transaction_id, 'kind': kind, 'currency_id': CURRENCY_ID, - 'amount_minor': amount, 'from_subject': sender, 'to_subject': recipient, - 'actor': actor, 'request_id': request_id, 'entry_key': entry_key, - 'reference': reference, 'committed_at': wire(now), - 'policy_version': POLICY_VERSION, 'policy_digest': POLICY_DIGEST, - 'ledger_sequence': sequence} - receipt = {'body': body, - 'signature': wire(receipt_signer.sign(canonical(body), purpose='money-receipt'))} - tx.execute('''INSERT INTO money_ledger - (seq,id,kind,currency_id,amount_minor,debit_account,credit_account,actor,request_id, - reference,committed_at,policy_version,policy_digest,receipt,entry_key) - VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)''', - (sequence,transaction_id,kind,CURRENCY_ID,amount,sender,recipient,actor, - request_id,reference,wire(now),POLICY_VERSION,POLICY_DIGEST, - canonical(receipt).decode(),entry_key), write=True) - return receipt - - -def _post_transfer(tx, *, sender, recipient, amount, actor, request_id, now, - receipt_signer, reference=None, kind='transfer', entry_key='primary', - escrow_authority=None): - require(kind in {'transfer', 'redeem', 'refund'}, 'invalid_money_kind') - account = tx.one('SELECT kind FROM ledger_accounts WHERE id=?', (sender,)) - if account and account[0] == 'order_escrow': - from msg.market.escrow import _ESCROW_WRITE - require(escrow_authority is _ESCROW_WRITE, 'escrow_release_forbidden') - return _post_entry(tx, sender=sender, recipient=recipient, amount=amount, - actor=actor, request_id=request_id, now=now, receipt_signer=receipt_signer, - reference=reference, kind=kind, entry_key=entry_key) +# Compatibility imports; shared implementation has one market owner. +from msg.market.ledger import account_requirements +from msg.market.ledger import require_money_subject as _owner +from msg.market.ledger import checked_amount as _amount +from msg.market.ledger import balance as _balance +from msg.market.ledger import total_supply as _supply +from msg.market.ledger import clearing_decision +from msg.market.ledger import append_entry as _post_entry +from msg.market.ledger import post_transfer as _post_transfer +from msg.market.ledger import CURRENCY_ID +from msg.market.ledger import CODE +from msg.market.ledger import SCALE +from msg.market.ledger import MAX_MINOR +from msg.market.ledger import POLICY_VERSION +from msg.market.ledger import POLICY_DIGEST def install(app): diff --git a/src/msg/plugins/orders.py b/src/msg/plugins/orders.py index f9abe3da..4e70aa16 100644 --- a/src/msg/plugins/orders.py +++ b/src/msg/plugins/orders.py @@ -14,70 +14,18 @@ from msg.core.errors import require from msg.core.models import HandlerOutput from msg.plugins.common import registration -from msg.plugins.money import CURRENCY_ID, MAX_MINOR, _balance, _post_transfer, account_requirements +from msg.market.ledger import CURRENCY_ID, MAX_MINOR, balance as _balance, post_transfer as _post_transfer, account_requirements from msg.plugins.schemas import IDENTIFIER, obj -from msg.plugins.store import _body, _listing, _package_row +from msg.market.catalog import read_listing_body as _body, read_listing as _listing, read_package_record as _package_row -_COLUMNS = ('id', 'buyer', 'seller', 'listing_id', 'listing_revision', - 'package_id', 'package_revision', 'package_digest', 'quantity', - 'unit_price_minor', 'total_price_minor', 'currency_id', - 'escrow_subject', 'escrow_policy', 'dispute_policy', 'terms_digest', - 'delivery_target', 'payment_intent_digest', 'payment_transaction_id', - 'state', 'created_at', 'funded_at', 'delivered_at', 'settled_at', - 'receipt_refs') - - -def _subject(ctx): - subject = ctx.principal.subject - require(subject is not None and ctx.principal.actor == subject and - ctx.principal.method == 'signature', 'signature_required') - return subject - - -def _viewer(ctx): - subject = ctx.principal.subject - require(subject is not None and ctx.principal.actor == subject, - 'order_not_found') - return subject - - -def _order_id(): - # 160 random bits, unguessable even if an attacker sees other order IDs. - return 'ord_' + base64.b32encode(os.urandom(20)).decode('ascii').rstrip('=').lower() - - -def _row(tx, order_id, viewer): - row = tx.one('''SELECT id,buyer,seller,listing_id,listing_revision, - package_id,package_revision,package_digest,quantity,unit_price_minor, - total_price_minor,currency_id,escrow_subject,escrow_policy, - dispute_policy,terms_digest,delivery_target,payment_intent_digest, - payment_transaction_id,state,created_at,funded_at,delivered_at, - settled_at,receipt_refs FROM store_orders WHERE id=?''', (order_id,)) - # A valid ID is not an access grant. Keep nonexistent and unauthorized alike. - require(row is not None and viewer in row[1:3], 'order_not_found') - result = dict(zip(_COLUMNS, row)) - result['delivery_target'] = loads(result['delivery_target']) - result['receipt_refs'] = loads(result['receipt_refs']) - return result - - -def _view(row, viewer): - keys = ('id', 'buyer', 'seller', 'listing_id', 'listing_revision', - 'package_revision', 'package_digest', 'quantity', - 'unit_price_minor', 'total_price_minor', 'currency_id', - 'escrow_policy', 'dispute_policy', 'terms_digest', 'state', - 'created_at', 'funded_at', 'delivered_at', 'settled_at') - result = {key: row[key] for key in keys} - result['payment_status'] = ('refunded' if row['state'] == 'refunded' else - 'funded' if row['payment_transaction_id'] else - 'pending') - result['delivery_channel'] = row['delivery_target']['channel'] - if viewer == row['buyer']: - result['delivery_target'] = row['delivery_target'] - result['payment_intent_digest'] = row['payment_intent_digest'] - result['receipt_refs'] = row['receipt_refs'] - return result +# Compatibility imports; shared implementation has one market owner. +from msg.market.order_records import require_signed_subject as _subject +from msg.market.order_records import require_order_viewer as _viewer +from msg.market.order_records import new_order_id as _order_id +from msg.market.order_records import read_order as _row +from msg.market.order_records import legacy_order_view as _view +from msg.market.order_records import _COLUMNS def install(app): @@ -188,7 +136,7 @@ async def buy(ctx, request, tx): canonical([receipt_id]).decode()), write=True) data = {'payment': receipt} if instant: - from msg.plugins.delivery import prepare_managed, delivery_summary + from msg.market.managed_delivery import prepare_managed, delivery_summary from msg.market.delivery_notifications import initialize_notification order = _row(tx, order_id, buyer) delivery = await prepare_managed(app, ctx, tx, order) diff --git a/src/msg/plugins/store.py b/src/msg/plugins/store.py index 09a1e143..e5da5477 100644 --- a/src/msg/plugins/store.py +++ b/src/msg/plugins/store.py @@ -17,36 +17,6 @@ MAX_MANIFEST_BYTES = 65536 -def _subject(ctx): - subject = ctx.principal.subject - require(subject is not None and ctx.principal.actor == subject and - ctx.principal.method == 'signature', 'signature_required') - return subject - - -async def _listing(app, ctx, request, tx, value, *, seller=False): - try: - rid = await resolve(tx, value) - resource = await tx.resource(rid) - require(resource.type == 'listing' and resource.parent == 't_store', 'listing_not_found') - await check_access(app, ctx, request, tx, rid, 'read') - if seller: - require(resource.owner == _subject(ctx), 'listing_not_found') - return resource - except Failure as exc: - if exc.code in {'not_found', 'permission_denied', 'credential_ceiling', - 'certificate_gate', 'ancestor_inactive'}: - raise Failure('listing_not_found') from None - raise - - -async def _body(app, tx, resource, revision=None): - rev = await tx.revision(ResourceRef(id=resource.id, revision=revision)) - body = loads(await app.contents.read_bytes(rev.content)) - # Listings deposited before the sale/bounty split were all sale listings. - return {'mode': 'sale', **body}, rev - - def _public(resource, body): # A package identifier is not a capability; package payloads are seller-only. return {'listing_id': resource.id, 'listing_revision': resource.revision, @@ -74,12 +44,6 @@ def _validate(body, now): require(body['package_ref'] is not None, 'package_required') -async def _package_row(tx, package_id): - return tx.one('''SELECT id,listing_id,listing_revision,seller,revision,kind,manifest, - payload_refs,digest,total_size,delivery_mode,deposited_at FROM store_packages WHERE id=?''', - (package_id,)) - - def _package_public(row): return dict(zip(('id', 'listing_id', 'listing_revision', 'seller', 'revision', 'kind', 'manifest', 'payload_refs', 'digest', 'total_size', @@ -87,6 +51,13 @@ def _package_public(row): (*row[:6], loads(row[6]), loads(row[7]), *row[8:]))) +# Compatibility imports; shared implementation has one market owner. +from msg.market.catalog import read_listing as _listing +from msg.market.catalog import read_listing_body as _body +from msg.market.catalog import read_package_record as _package_row +from msg.market.order_records import require_signed_subject as _subject + + def install(app): op, finish = registration(app, 'store', ('content',)) listing_fields = { From 65b7330d248f4af96ac9ecd1f1fad3fdc6f59782 Mon Sep 17 00:00:00 2001 From: LIghtJUNction Date: Tue, 29 Sep 2026 05:09:22 +0800 Subject: [PATCH 012/136] ci: remove market assembly tooling and preserve verified red evidence --- .github/tools/assemble_market_boundaries.py | 313 ------------------ .../workflows/assemble-market-boundaries.yml | 60 ---- docs/MARKET_BOUNDARY_PROGRESS.md | 132 ++++---- 3 files changed, 64 insertions(+), 441 deletions(-) delete mode 100644 .github/tools/assemble_market_boundaries.py delete mode 100644 .github/workflows/assemble-market-boundaries.yml diff --git a/.github/tools/assemble_market_boundaries.py b/.github/tools/assemble_market_boundaries.py deleted file mode 100644 index 79fac431..00000000 --- a/.github/tools/assemble_market_boundaries.py +++ /dev/null @@ -1,313 +0,0 @@ -"""Exact-base source extraction for PR #168; never imports project modules.""" -from pathlib import Path -import ast -import copy -import difflib -import hashlib -import io -import json -import re -import subprocess -import tokenize - -ROOT = Path.cwd() -BASE = '8692ad6c0056e793bbe7b0efc9f5fd0b08ab89c8' -EXPECTED = { - 'orders':'f9abe3da981c548a60d0e1b1630d82e02d0e8491', - 'store':'09a1e14328d512e91913bf00e987cf234816348e', - 'money':'424a03f1e6de882aba425f857ab602df818f90c1', - 'delivery':'4500a65ed55e5ea06426c9ac3011f4317ecdc70b', -} -NAMES = { - 'orders':('order_records',{ - '_subject':'require_signed_subject','_viewer':'require_order_viewer', - '_order_id':'new_order_id','_row':'read_order','_view':'legacy_order_view'}), - 'store':('catalog',{ - '_listing':'read_listing','_body':'read_listing_body','_package_row':'read_package_record'}), - 'money':('ledger',{ - 'account_requirements':'account_requirements','_owner':'require_money_subject', - '_amount':'checked_amount','_balance':'balance','_supply':'total_supply', - 'clearing_decision':'clearing_decision','_post_entry':'append_entry','_post_transfer':'post_transfer'}), - 'delivery':('managed_delivery',{ - '_delivery':'read_delivery','_buyer_order':'read_buyer_order', - '_verified_delivery':'verify_managed_delivery','_package':'read_managed_package', - '_payload':'read_managed_payloads','prepare_managed':'prepare_managed', - 'delivery_summary':'delivery_summary'}), -} -CONSTANTS = {'orders':('_COLUMNS',),'store':(), - 'money':('CURRENCY_ID','CODE','SCALE','MAX_MINOR','POLICY_VERSION','POLICY_DIGEST'), - 'delivery':('MAX_INLINE_BYTES','_COLUMNS')} -HEADERS = { - 'orders':'''"""Shared order identity, authorized records and legacy-compatible projections. - -These functions own record access, not operation registration or settlement. -""" -from __future__ import annotations -import base64 -import os -from msg.core.codec import loads -from msg.core.errors import require - -''', - 'store':'''"""Authorized catalog reads shared by published checkout versions. - -Version-specific schemas, creation and updates stay at catalog entrypoints. -""" -from __future__ import annotations -from msg.core.codec import loads -from msg.core.errors import Failure, require -from msg.core.models import ResourceRef -from msg.plugins.common import check_access, resolve -from msg.market.order_records import require_signed_subject - -''', - 'money':'''"""One append-only ledger posting path in the caller's transaction. - -This module grants no commit rights. Public transfers cannot supply the internal -escrow guard; Root issuance remains a separate local-console use case. -""" -from __future__ import annotations -from uuid import uuid4 -from msg.constants import ROOT_SUBJECT -from msg.core.codec import canonical, digest, wire -from msg.core.errors import require -from msg.core.models import AccessRequirement, SignatureProof - -''', - 'delivery':'''"""Shared bounded managed-package verification and local preparation. - -These preserve the managed checkout contracts; versioned manual/service delivery, -arbitration and operation registration retain their existing entrypoints. -""" -from __future__ import annotations -from msg.core.codec import b64, canonical, decode, digest, loads, wire -from msg.core.errors import require -from msg.core.models import BlobRef -from msg.plugins.common import new_id -from msg.market.ledger import CURRENCY_ID, balance -from msg.market.catalog import read_package_record -from msg.market.order_records import read_order -from msg.market.delivery_targets import validate_target - -''', -} -EXTRA_RENAMES = {'store':{'_subject':'require_signed_subject'}, - 'delivery':{'order_row':'read_order','_package_row':'read_package_record','_balance':'balance'}} -before = {} -after = {} -audit = [] - - -def read(path): - text = (ROOT/path).read_text() - before.setdefault(path,text) - return text - - -def nodes(text): - return ast.parse(text).body - - -def functions(text): - return {node.name:node for node in nodes(text) - if isinstance(node,(ast.FunctionDef,ast.AsyncFunctionDef))} - - -def assignments(text): - return {target.id:node for node in nodes(text) if isinstance(node,ast.Assign) - for target in node.targets if isinstance(target,ast.Name)} - - -def segment(text,node): - return ast.get_source_segment(text,node) - - -def rename(text,mapping): - tokens = tokenize.generate_tokens(io.StringIO(text).readline) - return tokenize.untokenize(token._replace(string=mapping.get(token.string,token.string)) - if token.type==tokenize.NAME else token for token in tokens) - - -def splice(text,edits): - starts=[0] - for line in text.splitlines(True): - starts.append(starts[-1]+len(line)) - indexed=[] - for node,value in edits: - indexed.append((starts[node.lineno-1]+node.col_offset, - starts[node.end_lineno-1]+node.end_col_offset,value)) - for start,end,value in sorted(indexed,reverse=True): - text=text[:start]+value+text[end:] - return text - - -def clean(text): - return re.sub(r'\n{4,}','\n\n\n',text).rstrip()+'\n' - - -sources={name:read('src/msg/plugins/'+name+'.py') for name in NAMES} -for name,expected in EXPECTED.items(): - actual=subprocess.check_output(['git','hash-object','src/msg/plugins/'+name+'.py'],text=True).strip() - assert actual==expected,(name,actual,expected) -assert segment(sources['orders'],functions(sources['orders'])['_subject']) == segment( - sources['store'],functions(sources['store'])['_subject']) - -# Move the unique in-process guard with the posting invariant, not with an API version. -escrow_path='src/msg/market/escrow.py' -escrow=read(escrow_path) -assert subprocess.check_output(['git','hash-object',escrow_path],text=True).strip() == '92e0bb595c8fec8665285f0d131630a52500d9fa' -guard_start=escrow.index('# Not a credential, account or public capability.') -guard_end=escrow.index('TRANSITIONS =') -guard=escrow[guard_start:guard_end] -assert '_ESCROW_WRITE = object()' in guard -assert guard.count('object()')==1 -after[escrow_path]=escrow[:guard_start]+'from msg.market.ledger import _ESCROW_WRITE\n\n'+escrow[guard_end:] - -providers={} -for name,(owner,mapping) in NAMES.items(): - exports={old:('msg.market.'+owner,new) for old,new in mapping.items()} - exports.update({constant:('msg.market.'+owner,constant) for constant in CONSTANTS[name]}) - if name=='store': - exports['_subject']=('msg.market.order_records','require_signed_subject') - providers['msg.plugins.'+name]=exports - -for name,(owner,mapping) in NAMES.items(): - text=sources[name] - fs,cs=functions(text),assignments(text) - renames={**mapping,**EXTRA_RENAMES.get(name,{})} - pieces=[HEADERS[name]] - pieces.extend(segment(text,cs[constant])+'\n' for constant in CONSTANTS[name]) - if name=='money': - pieces.append('\n'+guard) - for old,new in mapping.items(): - body=segment(text,fs[old]) - if name=='money' and old=='_post_transfer': - line=' from msg.market.escrow import _ESCROW_WRITE\n' - assert body.count(line)==1 - body=body.replace(line,'') - pieces.append('\n\n'+rename(body,renames)+'\n') - target='src/msg/market/'+owner+'.py' - assert not (ROOT/target).exists(),target - before[target]='' - after[target]=clean(''.join(pieces)) - for old,new in mapping.items(): - original=segment(text,fs[old]) - if name=='money' and old=='_post_transfer': - original=original.replace(' from msg.market.escrow import _ESCROW_WRITE\n','') - expected=ast.dump(ast.parse(rename(original,renames)),include_attributes=False) - actual=ast.dump(ast.parse(segment(after[target],functions(after[target])[new])),include_attributes=False) - assert actual==expected,(name,old) - audit.append(name+'.'+old+' -> '+owner+'.'+new) - edits=[(fs[old],'') for old in mapping]+[(cs[c],'') for c in CONSTANTS[name]] - if name=='store': - edits.append((fs['_subject'],'')) - adapter=splice(text,edits) - # Existing entrypoint code still names the compatibility imports. Each alias - # is the exact owned function, not a wrapper or a second implementation. - imports=['# Compatibility imports; shared implementation has one market owner.'] - for old,(module,new) in providers['msg.plugins.'+name].items(): - imports.append('from '+module+' import '+new+(' as '+old if new!=old else '')) - adapter=adapter.replace('def install(app):','\n'.join(imports)+'\n\n\ndef install(app):',1) - after['src/msg/plugins/'+name+'.py']=clean(adapter) - - -def rewire(text): - edits=[] - for node in ast.walk(ast.parse(text)): - if not isinstance(node,ast.ImportFrom) or node.module not in providers: - continue - groups={} - for item in node.names: - target,new=providers[node.module].get(item.name,(node.module,item.name)) - local=item.asname or item.name - groups.setdefault(target,[]).append(new+(' as '+local if new!=local else '')) - replacement=('\n'+' '*node.col_offset).join( - 'from '+module+' import '+', '.join(names) for module,names in groups.items()) - edits.append((node,replacement)) - return splice(text,edits) - - -# Only import wiring changes in existing market use cases and versioned handlers. -# Source outside these four entrypoints and market remains untouched. -paths={str(p.relative_to(ROOT)) for p in (ROOT/'src/msg/market').rglob('*.py')} -paths.update('src/msg/plugins/'+name+'.py' for name in NAMES) -paths.update(after) -for path in sorted(paths): - text=after[path] if path in after else read(path) - candidate=rewire(text) - if candidate!=before.get(path,''): - after[path]=clean(candidate) - - -class WithoutImports(ast.NodeTransformer): - def visit_ImportFrom(self,node): - return None - def visit_Import(self,node): - return None - - -def stable(node): - return ast.dump(WithoutImports().visit(copy.deepcopy(node)),include_attributes=False) - - -for name in NAMES: - old=functions(sources[name]) - new=functions(after['src/msg/plugins/'+name+'.py']) - removed=set(NAMES[name][1])|({'_subject'} if name=='store' else set()) - assert set(new)==set(old)-removed,(name,set(new),set(old)-removed) - for function in new: - assert stable(old[function])==stable(new[function]),(name,function) - audit.append(name+'.'+function+' body/schema unchanged') -for path,text in after.items(): - if not before.get(path) or not path.startswith('src/msg/market/'): - continue - left=ast.parse(before[path]);right=ast.parse(text) - if path==escrow_path: - left.body=[n for n in left.body if not (isinstance(n,ast.Assign) and - any(isinstance(t,ast.Name) and t.id=='_ESCROW_WRITE' for t in n.targets))] - assert stable(left)==stable(right),path - audit.append(path+' business AST unchanged') -for path,text in after.items(): - if path.startswith('src/msg/market/'): - assert not [n.module for n in ast.walk(ast.parse(text)) - if isinstance(n,ast.ImportFrom) and n.module in providers],path -# All old constants are moved verbatim; the changed authority import is singular. -assert after['src/msg/market/ledger.py'].count('_ESCROW_WRITE = object()')==1 -assert '_ESCROW_WRITE = object()' not in after[escrow_path] - -notes='docs/MARKET_BOUNDARY_PROGRESS.md' -after[notes]=read(notes)+''' - -## Implementation and source audit - -PR #168 extracts the existing shared implementation into `order_records`, -`catalog`, `ledger` and `managed_delivery`, with canonical public function names. -Old plugin names re-export those same function objects. All existing market use -cases, including function-local imports, and legacy plugin handlers use these -owners. The duplicate store subject check becomes the same order-subject check. - -The exact-base cloud assembly checks source blob identities, compares every -moved function after identifier-only renaming, and checks every remaining handler -and market state machine after ignoring import statements. SQL strings, schemas, -IDs, receipt purposes, positional order and existing algorithm bodies are not -rewritten. The only guard change is its owner: `ledger` defines `_ESCROW_WRITE` -once, escrow re-exports it, and protected posting reads that same object directly. -The guard is still supplied only by the existing checked escrow release path. - -No generated source is imported or executed during assembly. Cloud test jobs run -both new real-database contracts and existing published-version/recovery suites. -The assembly artifact contains the exact changed paths, audit and patch. Its -one-shot helper files are removed from the submitted tree before final validation. -No successful final-head result is claimed by this source-only audit. -''' - -out=Path('/tmp/market-boundary-assembly');out.mkdir(parents=True,exist_ok=True) -changed={path:text for path,text in after.items() if text!=before.get(path,'')} -patch=''.join(''.join(difflib.unified_diff(before.get(path,'').splitlines(True),text.splitlines(True), - fromfile='a/'+path if before.get(path) else '/dev/null',tofile='b/'+path)) for path,text in sorted(changed.items())) -for path,text in changed.items(): - (ROOT/path).write_text(text) -(out/'changes.patch').write_text(patch) -(out/'audit.json').write_text(json.dumps({'baseline_commit':BASE,'baseline_blobs':EXPECTED, - 'checked_invariants':audit,'changed_paths':sorted(changed), - 'patch_sha256':hashlib.sha256(patch.encode()).hexdigest()},indent=2)+'\n') diff --git a/.github/workflows/assemble-market-boundaries.yml b/.github/workflows/assemble-market-boundaries.yml deleted file mode 100644 index 732be59a..00000000 --- a/.github/workflows/assemble-market-boundaries.yml +++ /dev/null @@ -1,60 +0,0 @@ -name: Assemble audited market boundary extraction -on: - push: - branches: [build/market-boundaries-168] - paths: - - .github/tools/assemble_market_boundaries.py - - .github/workflows/assemble-market-boundaries.yml -permissions: - contents: write -jobs: - assemble: - if: github.repository == 'TokenNotIncluded/msg.lmm.best' && github.ref == 'refs/heads/build/market-boundaries-168' - runs-on: ubuntu-latest - timeout-minutes: 5 - steps: - - uses: actions/checkout@v7 - with: - fetch-depth: 0 - - name: Require exact regression source - env: - INPUT_SHA: ${{ github.sha }} - run: | - test "$(git rev-parse HEAD)" = "$INPUT_SHA" - git merge-base --is-ancestor 8692ad6c0056e793bbe7b0efc9f5fd0b08ab89c8 HEAD - git diff --exit-code 8692ad6c0056e793bbe7b0efc9f5fd0b08ab89c8 -- src tests docs - - name: Assemble and audit shared functions and unchanged state machines - run: python .github/tools/assemble_market_boundaries.py - - name: Commit only audited source and notes - run: | - git add src/msg/market src/msg/plugins/orders.py src/msg/plugins/money.py src/msg/plugins/delivery.py src/msg/plugins/store.py docs/MARKET_BOUNDARY_PROGRESS.md - git diff --cached --check - python - <<'PY' - import json,subprocess - from pathlib import Path - expected=set(json.loads(Path('/tmp/market-boundary-assembly/audit.json').read_text())['changed_paths']) - actual=set(subprocess.check_output(['git','diff','--cached','--name-only'],text=True).splitlines()) - assert actual==expected,(actual-expected,expected-actual) - PY - git -c user.name='github-actions[bot]' -c user.email='41898282+github-actions[bot]@users.noreply.github.com' commit -m 'refactor(market): own shared records posting and managed delivery independently of entrypoints' - - name: Publish only the dedicated build branch without force - env: - INPUT_SHA: ${{ github.sha }} - run: | - test "$(git ls-remote origin refs/heads/build/market-boundaries-168 | cut -f1)" = "$INPUT_SHA" - git push origin HEAD:refs/heads/build/market-boundaries-168 - python - <<'PY' - import json,os,subprocess - from pathlib import Path - git=lambda *args:subprocess.check_output(['git',*args],text=True).strip() - Path('/tmp/market-boundary-assembly/source.json').write_text(json.dumps({ - 'input':os.environ['INPUT_SHA'],'commit':git('rev-parse','HEAD'), - 'tree':git('rev-parse','HEAD^{tree}'),'dirty':bool(git('status','--porcelain'))},indent=2)+'\n') - PY - git archive --format=tar.gz -o /tmp/market-boundary-assembly/source.tar.gz HEAD - - uses: actions/upload-artifact@v4 - if: always() - with: - name: market-boundary-assembly - path: /tmp/market-boundary-assembly/ - if-no-files-found: error diff --git a/docs/MARKET_BOUNDARY_PROGRESS.md b/docs/MARKET_BOUNDARY_PROGRESS.md index 6f6e1f0a..cb88a4b9 100644 --- a/docs/MARKET_BOUNDARY_PROGRESS.md +++ b/docs/MARKET_BOUNDARY_PROGRESS.md @@ -2,79 +2,75 @@ ## Scope and coordination -Issue #159; references #85/#83, not completion of #71/#72 or production acceptance. -Base: `476e64f04ed1d9c7dcab4a41c4f3cdb72faad6a7`, tree -`40ca207a3c49be347120e058e47ed328ec4e3098`. Branch: -`fix/market-boundaries-20260929`. Coordination is #83 comment5878359290. +Issue #159, PR #168; references #85/#83, not completion of #71/#72 or production acceptance. +Base main: `476e64f04ed1d9c7dcab4a41c4f3cdb72faad6a7`, tree +`40ca207a3c49be347120e058e47ed328ec4e3098`. Branch: `fix/market-boundaries-20260929`. +Coordination: #83 comments5878359290 and5878591036; #159 comment5878570249. -This work does not modify #165/#167 core/tool/executor ownership, #155 recovery, -or #166 metadata session extraction. Main and all active claims were refreshed -before starting. No production, real funds, Root/PIN or outward delivery action. +This change does not modify #165/#167 core/tool/executor ownership, #155 recovery, +or #166 metadata sessions. No production, real funds, Root/PIN or outward delivery. -## Regression first +## Verified regression-first evidence -The first commit changes no production implementation. It adds ten tests: -seven ownership/import/unique-guard checks and three real PostgreSQL behavior -contracts. Current expected failures concern the new module boundaries; expected -passing characterizations cover buyer/seller privacy, forged escrow authority -with no ledger changes, receipt verification/byte preservation and replay. -No test result is claimed until the exact cloud run is inspected. +Red run **36483360971**, job109134178838, tested exact commit +`8692ad6c0056e793bbe7b0efc9f5fd0b08ab89c8` / tree +`907f51806164625749e57bf754713ae9f365f9b6`: **7 failed, 3 passed, +0 errors, 0 skipped**. All failures were new ownership/import assertions. +The real PostgreSQL buyer/seller privacy, forged-escrow zero-write guard, +and signed-receipt byte preservation/idempotent replay tests passed. -## Observed dependency graph +Artifact10998081065 was independently downloaded and hashed; source.json and JUnit +were read. SHA-256: `07f566e15500142c37223782d5cc10086bb9e92a967e5a204c52c374abae3600`. +No project tests ran locally. -`market.orders/email/arbitration/delivery -> plugins.orders` for order records, -subject checks and existing projections. `market -> plugins.store` for listing -and package reads. `market.escrow/orders -> plugins.money` for protected posting. -`market.policy/targets/notifications/escrow/delivery -> plugins.delivery` for -managed delivery records and verification. `plugins.orders/delivery -> market.escrow` -closes the reverse dependency. Function-local imports are included in this audit. +## Implemented ownership -## Intended ownership and invariants +Before: market use cases imported records/catalog/posting/managed-delivery from +`plugins.orders/store/money/delivery`, including function-local imports, while +those entrypoints depended on market escrow and policies. -- `market.order_records`: signed subject/viewer checks, order IDs, authorized +After: + +- `market.order_records` owns signed subject/viewer checks, order IDs, authorized record lookup and explicitly legacy-compatible projections. -- `market.catalog`: existing authorized listing/body/package reads, not plugin - registration or version-specific catalog schema/defaults. -- `market.ledger`: existing policy constants, account requirements and one posting - implementation. The in-process escrow guard has one owner; moving it does not - make it a credential or a network operation. Local-only Root issuance is unchanged. -- `market.managed_delivery`: existing bounded managed-package verification, - local preparation and read helpers. Their published version semantics remain; - they do not replace v3 service/manual delivery or arbitration state machines. - -Old plugin imports re-export these same functions, never wrappers with new policy -or copied implementations. Version-specific schema, request signing and response -adaptation stay at their existing entrypoints. Existing transactions, account -locks, receipt purposes, ID generation and recovery/lease fences remain intact. -No new Manager, Factory, Repository, ORM or alternate money/escrow path is added. - -## Verification and integration - -Focused cloud Actions run the new contracts and existing checkout v1/v2/v3, -account-scope, deterministic clearing, Bounty, redemption, storage and recovery -suites. Full four-shard exact node-ID/conformance/build and applicable recovery -checks remain required on the final synchronized head. No skips, weakened -assertions, changed published versions or historical fact/schema migration. - - -## Implementation and source audit - -PR #168 extracts the existing shared implementation into `order_records`, -`catalog`, `ledger` and `managed_delivery`, with canonical public function names. -Old plugin names re-export those same function objects. All existing market use -cases, including function-local imports, and legacy plugin handlers use these -owners. The duplicate store subject check becomes the same order-subject check. - -The exact-base cloud assembly checks source blob identities, compares every -moved function after identifier-only renaming, and checks every remaining handler -and market state machine after ignoring import statements. SQL strings, schemas, -IDs, receipt purposes, positional order and existing algorithm bodies are not -rewritten. The only guard change is its owner: `ledger` defines `_ESCROW_WRITE` -once, escrow re-exports it, and protected posting reads that same object directly. -The guard is still supplied only by the existing checked escrow release path. - -No generated source is imported or executed during assembly. Cloud test jobs run -both new real-database contracts and existing published-version/recovery suites. -The assembly artifact contains the exact changed paths, audit and patch. Its -one-shot helper files are removed from the submitted tree before final validation. -No successful final-head result is claimed by this source-only audit. +- `market.catalog` owns existing authorized listing/body/package reads, not + version-specific schemas, defaults or registration. +- `market.ledger` owns policy constants, account requirements and the single + posting implementation. It alone defines the existing `_ESCROW_WRITE` object; + escrow re-exports that same object. It is not a public/network credential. +- `market.managed_delivery` owns existing bounded managed-package verification, + preparation and read helpers; it does not replace manual/service/arbitration flows. + +Old plugin paths re-export the same function objects, never wrappers or copies. +Existing market use cases and versioned handlers import the neutral owners. +The duplicate store subject check is the same function as the order subject check. +No new Manager, Factory, Repository, ORM, ledger or escrow state machine exists. + +## Verified source assembly and review + +Cloud assembly **36483781552** succeeded from input +`d0d7f4f61f138b0ced266f2324f4a90c1689b6b5`. Output commit +`f92390fe8f54fbf63fe21d770f37d6fdbb56190e`, tree +`f3f8a2c20c031e0fdf0e1129b2123e7350e7cfaa`. + +Its audit checks exact input blobs and all **23** moved functions after identifier-only +renaming, every remaining handler/schema, and the complete existing market state +machines after excluding import statements. The only guard change is its owner; +its identity check and the checked escrow-release call path remain unchanged. +SQL strings, account locks, signatures, IDs, receipt purposes and transactions were +not rewritten. Shared posting still cannot commit or grant remote Root issuance. + +Artifact10997307359 was downloaded, its audit/patch/source read, and its SHA-256 +verified as `2dbcbf167086480216596757ffd32b42fa8ccc61cdea3d767e2862d1aba3ae74`. +The contained patch independently hashes to +`0f3ee2de9d5cb04c50ebf4b573c44652fa7a50a225f3e006f50944432e9173c9`. +The two one-shot assembly files are removed from the submitted tree. + +## Final-head gates + +Source review/assembly is not green test evidence. The new boundary contracts, +existing v1/v2/v3 checkout, clearing, Bounty, redemption and recovery regressions, +and complete four-shard exact-node-ID/conformance/build gate must run on the final +synchronized head. No skip, weakened assertion or published-version migration. +Actual final results and merge SHA are recorded in PR #168 and coordination #83; +this source commit does not predict those outcomes or close umbrella issues. From 93d864b3b003310c6a139a918e5e14e65482e404 Mon Sep 17 00:00:00 2001 From: LIghtJUNction Date: Tue, 29 Sep 2026 05:14:48 +0800 Subject: [PATCH 013/136] ci(client): apply the reviewed dependency-boundary patch on its fixed cloud branch Refs #158 #82 #83 #85. Patch SHA-256 244778f572724b991a8b14ff3334b810a3af14329de999bf148c5445950c8a02. Restrict publishing to 19 enumerated files and this feature branch; remove the one-shot publisher in the generated source commit. Record RED and explicit full-server installation compatibility. --- .github/workflows/prepare-client-boundary.yml | 82 ++++ docs/CLIENT_BOUNDARY_PROGRESS.md | 24 +- docs/CLIENT_INSTALLATION.md | 31 ++ scripts/patches/client-boundary.patch | 456 ++++++++++++++++++ 4 files changed, 581 insertions(+), 12 deletions(-) create mode 100644 .github/workflows/prepare-client-boundary.yml create mode 100644 docs/CLIENT_INSTALLATION.md create mode 100644 scripts/patches/client-boundary.patch diff --git a/.github/workflows/prepare-client-boundary.yml b/.github/workflows/prepare-client-boundary.yml new file mode 100644 index 00000000..34249493 --- /dev/null +++ b/.github/workflows/prepare-client-boundary.yml @@ -0,0 +1,82 @@ +name: Apply reviewed client boundary patch +on: + push: + branches: ['fix/client-boundary-20260929'] + paths: ['scripts/patches/client-boundary.patch'] +permissions: + contents: read +jobs: + apply-reviewed-patch: + if: github.repository == 'TokenNotIncluded/msg.lmm.best' + runs-on: ubuntu-latest + timeout-minutes: 5 + permissions: + contents: write + steps: + - uses: actions/checkout@v7 + with: + persist-credentials: false + fetch-depth: 2 + - name: Apply only the reviewed patch to its recorded parent + run: | + test "$(git rev-parse HEAD^)" = 6ea463d6fa8b256cb07d0ede2d7795a5336eda4f + git apply --index --check scripts/patches/client-boundary.patch + git apply --index scripts/patches/client-boundary.patch + git diff --cached --check + - name: Publish only this feature branch without force + env: + GH_TOKEN: ${{ github.token }} + run: | + python - <<'PY' + import hashlib, json, os, pathlib, subprocess, urllib.request + repo = 'TokenNotIncluded/msg.lmm.best' + branch = 'fix/client-boundary-20260929' + assert os.environ['GITHUB_REPOSITORY'] == repo + assert os.environ['GITHUB_REF'] == 'refs/heads/' + branch + paths = [ + 'README.md', 'docs/DEPLOYMENT.md', 'pyproject.toml', 'src/msg/atomic_file.py', + 'src/msg/cli.py', 'src/msg/client.py', 'src/msg/client_certificates.py', + 'src/msg/client_custodial.py', 'src/msg/client_upgrade.py', 'src/msg/core/codec.py', + 'src/msg/core/executor.py', 'src/msg/daemon.py', 'src/msg/security/custodial_migration.py', + 'src/msg/security/custodial_protocol.py', 'src/msg/security/vault.py', + 'src/msg/storage/git.py', 'src/msg/transports/client.py', 'src/msg/transports/mcp.py', + 'src/msg/transports/mcp_protocol.py', + ] + changed = subprocess.check_output(['git', 'diff', '--cached', '--name-only'], text=True).splitlines() + assert sorted(changed) == sorted(paths), changed + def api(method, endpoint, payload=None): + data = None if payload is None else json.dumps(payload).encode() + request = urllib.request.Request('https://api.github.com/repos/' + repo + endpoint, + data=data, method=method, headers={ + 'Authorization': 'Bearer ' + os.environ['GH_TOKEN'], + 'Accept': 'application/vnd.github+json', 'Content-Type': 'application/json', + 'X-GitHub-Api-Version': '2022-11-28', + }) + with urllib.request.urlopen(request, timeout=60) as response: + return json.load(response) + parent = os.environ['GITHUB_SHA'] + assert api('GET', '/git/ref/heads/' + branch)['object']['sha'] == parent + old = api('GET', '/git/commits/' + parent) + assert [p['sha'] for p in old['parents']] == ['6ea463d6fa8b256cb07d0ede2d7795a5336eda4f'] + entries = [{'path': p, 'mode': '100644', 'type': 'blob', + 'content': pathlib.Path(p).read_text()} for p in paths] + for p in ['scripts/patches/client-boundary.patch', '.github/workflows/prepare-client-boundary.yml']: + entries.append({'path': p, 'mode': '100644', 'type': 'blob', 'sha': None}) + tree = api('POST', '/git/trees', {'base_tree': old['tree']['sha'], 'tree': entries}) + commit = api('POST', '/git/commits', { + 'message': 'refactor(client): isolate shared protocol helpers and client-only dependencies\n\nRefs #158 #82 #83 #85. Preserve signed bytes, credential persistence and explicit full-server installation.', + 'tree': tree['sha'], 'parents': [parent], + }) + api('PATCH', '/git/refs/heads/' + branch, {'sha': commit['sha'], 'force': False}) + pathlib.Path('artifacts').mkdir(exist_ok=True) + pathlib.Path('artifacts/patch-result.json').write_text(json.dumps({ + 'source': parent, 'commit': commit['sha'], 'tree': tree['sha'], + 'patch_sha256': hashlib.sha256(pathlib.Path('scripts/patches/client-boundary.patch').read_bytes()).hexdigest(), + 'paths': paths, + }, indent=2) + '\n') + print('Published reviewed source:', commit['sha'], tree['sha']) + PY + - uses: actions/upload-artifact@v4 + with: + name: client-patch-result + path: artifacts/ diff --git a/docs/CLIENT_BOUNDARY_PROGRESS.md b/docs/CLIENT_BOUNDARY_PROGRESS.md index 11966c65..c1f09106 100644 --- a/docs/CLIENT_BOUNDARY_PROGRESS.md +++ b/docs/CLIENT_BOUNDARY_PROGRESS.md @@ -1,21 +1,21 @@ # Client dependency boundary progress — 2026-09-29 -Refs #158/#82/#83/#85. Branch `fix/client-boundary-20260929` starts at tool-boundary head `f14fd1533f70983b046df4b4c813993e8a536d12` (#165). It does not write the #155 recovery, #156 executor or #166 metadata-session branches. +Refs #158/#82/#83/#85. Branch `fix/client-boundary-20260929` starts at tool head f14fd1533f70983b046df4b4c813993e8a536d12. #165 was subsequently merged as 08c58928ee3f6c6cb350d9e9fba75d072bd78f08 with the same exact tested tree; #160 is closed. Do not write #155 recovery, #169 executor or #166 metadata-session branches from this slice. -## Regression-first work +## Independently verified RED -This commit adds seven boundary/compatibility/installation checks and a standalone fresh-interpreter client probe. It contains no implementation fix yet. The probe imports client/CLI/TUI modules under a strict server-import guard, creates/reloads protected client key files, and sends the same genuinely signed request through HTTP, path GET, GraphQL and MCP using an in-process mock HTTP endpoint. It never sends to a real service. +- Head **6ea463d6fa8b256cb07d0ede2d7795a5336eda4f**, tree **2a89a0a36f2d87646cccd4fdbd5ed8420f31c9b9**. +- Cloud run **36483229167**, artifact **10998106395**, SHA-256 `449fc080ba7232cceea4c6f6444d1468fda03b6278112d1c9cbec96dcfdce112` was downloaded and independently verified. Source JSON matches this commit/tree; JUnit reports **7 failed / 0 errors / 0 skipped**, including the actual client import guard refusing server vault imports. This is expected regression-first failure, not a successful implementation. +- The real baseline wheel was built; minimal installation remains a failure because its base dependencies include server packages. No project code/test/build ran locally. -The independent workflow builds the real wheel and installs it with normal dependencies into a fresh virtualenv outside the source checkout. It runs pip check, both command help entrypoints and the probe with explicit rejection of server-only packages. No PostgreSQL/Valkey service is configured for this workflow; full repository CI remains unchanged and mandatory. +## Reviewed implementation -## Intended ownership and compatibility +A fixed-parent one-shot cloud patch applies only the 19 enumerated source/install-document files on this feature branch, with no force update or project execution under write credentials. The temporary publisher and patch are removed from the resulting source tree. Exact generated head/tree and artifact hash must be read before final PR verification. -- Compact result serialization belongs to protocol codec; executor keeps only a compatibility import. -- Atomic protected local-file replacement has one narrowly named owner, not the Git/CAS backend. Existing permissions/fsync/replacement semantics remain unchanged. -- Custodial proof input/derivation and signed migration statements share one state-free owner; server-held vault and retirement verification stay server-side. Existing exports remain aliases, not copied security code. -- MCP version negotiation constants must not require importing the server implementation from the network client. -- Base installation expresses client dependencies; the server extra is the complete server install. The existing dev extra retains server dependencies and msg/msgd command names are unchanged. Documentation must explicitly migrate full server installation to the server extra; no automatic package installation. +The change moves compact result encoding, atomic local-file replacement, state-free custodial proof/statement bytes and MCP version constants into their actual shared owners, retaining identity-preserving compatibility exports. Clients no longer import executor, Git storage or server vault for those helpers. Published bytes, credential file protections and server-held authority/state stay unchanged. Base metadata becomes client-only; server extra holds the original server constraints, dev still includes server, and daemon commands fail clearly before state creation when those dependencies are absent. See CLIENT_INSTALLATION.md for migration instructions. -## Evidence still required +## Remaining gates -Read the RED run before implementation. Then verify helper compatibility, four real client transport calls, a dependency-clean wheel environment, all existing credential journal/recovery and server tests, full exact-node-ID/conformance/build gates and current integration tree. No size, speed, Windows/Android field result or production readiness is claimed by this refactor. Keep #158 open until those facts are proved. +The independent workflow must prove all seven checks, real wheel outside the checkout, no server-only Python dependencies, both help entrypoints and four actual signed transport calls using an in-process mock endpoint. Full original server, journal/recovery, protocol and four-shard exact-node-ID/build gates remain mandatory. Review and verify the final combination with current main/#169 rather than treating this isolated branch as the combined result. No GREEN, merge or #158 closure is claimed at this checkpoint. + +No production deployment, Root/PIN, funds, real external notification, key destruction or recovery promotion was performed. Production and platform-field issues remain blocked on their own evidence. diff --git a/docs/CLIENT_INSTALLATION.md b/docs/CLIENT_INSTALLATION.md new file mode 100644 index 00000000..8b013e83 --- /dev/null +++ b/docs/CLIENT_INSTALLATION.md @@ -0,0 +1,31 @@ +# 客户端与服务端安装边界 + +本页对应 #158 的依赖边界调整，不表示新的协议版本或生产部署已经完成。Python 版本要求仍以 pyproject.toml 为准。 + +## 安装角色 + +仅连接远程 msg 服务的客户端，源码安装使用 `python -m pip install .` 或 `uv sync`。安装已经构建的发行文件时，使用 `python -m pip install /path/to/msg_lmm_best--py3-none-any.whl`。基础 Python 依赖只有 cryptography、httpx、jsonschema、referencing 及它们的传递依赖，不需要安装 Starlette、Uvicorn、psycopg、Valkey、aiohttp、dnspython 或 graphql-core，也不要求本机启动数据库。 + +完整服务端使用 `python -m pip install '.[server]'` 或 `uv sync --extra server`。发行包安装使用 `python -m pip install 'msg-lmm-best[server]'`，应选用已发布且与服务端部署匹配的版本。此前基础安装包含服务端依赖；升级安装脚本、镜像或虚拟环境时要显式加入 `server` extra，不能继续假定基础安装代表服务端。现有 `.[dev]` 仍拉入完整服务端依赖，原全量 CI 安装路径不变。 + +`msg`、`msgd` 两个命令入口都保留。`--help` 不要求服务端依赖。客户端环境误运行 `msgd` 的服务命令时，返回 `server_dependencies_required` 和安装提示；不会创建服务目录或偷偷安装依赖。安装 extra 只安装 Python 包，不启动数据库、不部署服务、不授予证书权限。`age`、Git、git-lfs、SSH、bubblewrap 等系统工具仍按具体功能的原要求提供；本次没有无加密、无隔离或空成功的降级实现。 + +## 共享能力归属 + +| 能力 | 唯一实现所有者 | 服务端兼容导出 | +| --- | --- | --- | +| OperationResult 的紧凑 wire 表示 | `msg.core.codec.result_wire` | `msg.core.executor.result_wire` | +| 受保护本地文件的原子替换和同步 | `msg.atomic_file.durable_write` | `msg.storage.git.durable_write` | +| 托管升级 PoP 上下文、派生及客户端证明 | `msg.security.custodial_protocol` | `msg.security.vault` 对应名称 | +| 托管迁移决策和 ACK 的签署语句 | `msg.security.custodial_protocol` | `msg.security.custodial_migration` 对应名称 | +| 已发布 MCP 协商版本集合 | `msg.transports.mcp_protocol` | `msg.transports.mcp` 对应名称 | + +兼容导出是同一函数或同一常量对象，不复制安全敏感实现。共享协议模块不持有服务器密钥、不接受 Application 或事务，也不验证销毁/退役事实。服务端 vault、权限复核、历史清单、恢复和事务发布继续留在原服务端模块。客户端签名仍绑定原 purpose、主体、请求和上下文；请求版本、签署字节、秘密一次释放、journal 与凭据权限语义不变。 + +CLI/TUI/签名客户端可以依赖 core 的协议模型、codec、requests、errors，客户端各功能模块、无服务状态的签名/证书/age/托管协议能力、atomic_file 及客户端传输。不能通过执行器、Git/CAS 存储、托管 vault、admin 或 workers 取得公共辅助函数。MCP HTTP 客户端读取版本常量也不再导入服务端适配器；stdio 的原行为与全量协议回归保留。 + +## 验证边界 + +`tests/test_client_boundary.py` 检查唯一实现、兼容导出、精确证明上下文、原子替换失败保护和缺失依赖前置错误。`scripts/check_client_install.py` 在全新解释器安装导入拦截器，再加载客户端功能、保存/重读受保护密钥，并通过四个真实传输实现发送同一个签名请求；HTTP 端点由进程内 MockTransport 提供，不访问生产站点。 + +`.github/workflows/client-boundary.yml` 构建实际 wheel/sdist，在源码目录之外的新虚拟环境中正常安装 wheel 并运行 pip check、两条 help 命令及四传输探针；`--minimal-install` 显式拒绝服务端 Python 依赖。全量 PostgreSQL/Valkey、CLI/MCP、凭据 journal/恢复、协议一致性与构建门禁仍必须通过。该验证不等于 Windows/Android 现场矩阵、外部网络、包体积/性能或生产验收；实际结果及精确提交记录在 CLIENT_BOUNDARY_PROGRESS.md 和对应 PR 中。 diff --git a/scripts/patches/client-boundary.patch b/scripts/patches/client-boundary.patch new file mode 100644 index 00000000..dbeb5aeb --- /dev/null +++ b/scripts/patches/client-boundary.patch @@ -0,0 +1,456 @@ +diff --git a/README.md b/README.md +index 74d7a1a..978ca32 100644 +--- a/README.md ++++ b/README.md +@@ -29,6 +29,15 @@ msg.lmm.best 是一个开放的交流空间。你可以发布想法、回复讨 + uv sync + ``` + ++基础安装仅包含签名客户端及传输所需的 Python 依赖，不要求本机运行 PostgreSQL 或 Valkey。 ++安装发行包时使用 `python -m pip install msg-lmm-best`。`age` 加密/备份等操作仍需相应系统工具， ++并不因为依赖分组而获得空实现或降级执行。 ++ ++部署完整服务端请使用 `uv sync --extra server`，或在源码目录安装 `python -m pip install '.[server]'`。 ++从此前默认包含服务端依赖的版本升级时，也应显式加入 `server` extra；`dev` extra 仍包含完整服务端依赖。 ++`msg`、`msgd` 的命令名和协议版本不变。仅客户端环境运行服务命令会提示缺少服务端依赖，而不会创建服务状态。 ++完整依赖与边界见[客户端安装说明](docs/CLIENT_INSTALLATION.md)。 ++ + 连接到**已启用这个版本**的服务。首次使用时将示例地址换成实际服务地址；客户端会保存服务地址和身份私钥，请妥善保管。 + + ```bash +diff --git a/docs/DEPLOYMENT.md b/docs/DEPLOYMENT.md +index 32ac5ef..45db645 100644 +--- a/docs/DEPLOYMENT.md ++++ b/docs/DEPLOYMENT.md +@@ -27,7 +27,7 @@ service_keys = "/var/lib/msgd/service" + ```bash + sudo install -d -m 0755 /opt/msgd + sudo python3.15 -m venv /opt/msgd/venv +-sudo /opt/msgd/venv/bin/python -m pip install . ++sudo /opt/msgd/venv/bin/python -m pip install '.[server]' + ``` + + 在真实本机 VT 或串行控制台运行： +diff --git a/pyproject.toml b/pyproject.toml +index cfc5664..cb7cf9f 100644 +--- a/pyproject.toml ++++ b/pyproject.toml +@@ -11,16 +11,9 @@ requires-python = ">=3.15" + license = "MIT" + dependencies = [ + "cryptography>=46,<52", +- "starlette>=0.50,<2", +- "uvicorn>=0.38,<1", + "httpx>=0.28,<1", + "jsonschema>=4.25,<5", + "referencing>=0.36,<1", +- "aiohttp>=3.13,<4", +- "dnspython>=2.8,<3", +- "graphql-core>=3.2.7,<3.3", +- "psycopg[binary]>=3.3,<4", +- "valkey>=6.1,<7", + ] + + [[project.authors]] +@@ -31,7 +24,17 @@ msg = "msg.cli:main" + msgd = "msg.daemon:main" + + [project.optional-dependencies] ++server = [ ++ "starlette>=0.50,<2", ++ "uvicorn>=0.38,<1", ++ "aiohttp>=3.13,<4", ++ "dnspython>=2.8,<3", ++ "graphql-core>=3.2.7,<3.3", ++ "psycopg[binary]>=3.3,<4", ++ "valkey>=6.1,<7", ++] + dev = [ ++ "msg-lmm-best[server]", + "pytest>=8.4", + "pytest-asyncio>=1.1", + "pytest-cov>=6", +diff --git a/src/msg/atomic_file.py b/src/msg/atomic_file.py +new file mode 100644 +index 0000000..beaf703 +--- /dev/null ++++ b/src/msg/atomic_file.py +@@ -0,0 +1,26 @@ ++"""Atomic local-file replacement with caller-selected mode and durable syncs.""" ++from __future__ import annotations ++ ++import os ++from pathlib import Path ++import tempfile ++ ++ ++def durable_write(path: Path, data: bytes, mode: int = 0o600): ++ path.parent.mkdir(parents=True,exist_ok=True) ++ fd,temporary=tempfile.mkstemp(prefix='.pending-',dir=path.parent) ++ try: ++ os.fchmod(fd,mode) ++ with os.fdopen(fd,'wb') as stream: ++ stream.write(data) ++ stream.flush() ++ os.fsync(stream.fileno()) ++ os.replace(temporary,path) ++ directory=os.open(path.parent,os.O_RDONLY|os.O_DIRECTORY) ++ try: ++ os.fsync(directory) ++ finally: ++ os.close(directory) ++ finally: ++ if os.path.exists(temporary): ++ os.unlink(temporary) +diff --git a/src/msg/cli.py b/src/msg/cli.py +index 9913a59..2f3d74f 100644 +--- a/src/msg/cli.py ++++ b/src/msg/cli.py +@@ -10,7 +10,7 @@ from pathlib import Path + from msg.client import ClientState,MsgClient + from msg.core.codec import canonical,loads,wire + from msg.core.errors import Failure,require +-from msg.core.executor import result_wire ++from msg.core.codec import result_wire + from msg.core.models import ResourceRef,OperationResult + from msg.transports.client import TRANSPORTS + +diff --git a/src/msg/client.py b/src/msg/client.py +index 144c0ff..326ad17 100644 +--- a/src/msg/client.py ++++ b/src/msg/client.py +@@ -20,8 +20,8 @@ from msg.core.requests import request_for + from msg.security.crypto import Ed25519Signer, subject_id, key_id + from msg.security.age_keys import (generate_age_key,recipient_from_identity, + public_from_recipient,encryption_key_id) +-from msg.security.vault import client_upgrade_proof +-from msg.storage.git import durable_write ++from msg.security.custodial_protocol import client_upgrade_proof ++from msg.atomic_file import durable_write + from msg.client_tokens import read_journal, remove_journal, token_operation + from msg.transports.client import HTTPTransport, GraphQLTransport, MCPHTTPTransport + +diff --git a/src/msg/client_certificates.py b/src/msg/client_certificates.py +index d8b9c30..75fa957 100644 +--- a/src/msg/client_certificates.py ++++ b/src/msg/client_certificates.py +@@ -12,7 +12,7 @@ from msg.core.errors import require + from msg.core.models import CertificateRequest,Certificate,Signature,CapabilityGrant,IssuancePolicy + from msg.security.certificates import csr_body,verify_csr,sign_certificate + from msg.security.crypto import Ed25519Signer,key_id +-from msg.storage.git import durable_write ++from msg.atomic_file import durable_write + + + def local_key(path, *, create=False): +diff --git a/src/msg/client_custodial.py b/src/msg/client_custodial.py +index f79430e..5c162dd 100644 +--- a/src/msg/client_custodial.py ++++ b/src/msg/client_custodial.py +@@ -10,9 +10,9 @@ from msg.client_secrets import write_private + from msg.core.codec import canonical, digest, loads, wire + from msg.core.errors import Failure, require + from msg.security.age_keys import encryption_key_id, public_from_recipient, recipient_from_identity +-from msg.security.custodial_migration import ack_statement, stage_statement +-from msg.security.vault import client_upgrade_proof +-from msg.storage.git import durable_write ++from msg.security.custodial_protocol import ack_statement, stage_statement ++from msg.security.custodial_protocol import client_upgrade_proof ++from msg.atomic_file import durable_write + + + def journal_for(state): +diff --git a/src/msg/client_upgrade.py b/src/msg/client_upgrade.py +index 03c5c92..b68e887 100644 +--- a/src/msg/client_upgrade.py ++++ b/src/msg/client_upgrade.py +@@ -8,7 +8,7 @@ from uuid import uuid4 + from msg.core.codec import b64, canonical, loads, wire + from msg.core.errors import Failure, require + from msg.security.crypto import Ed25519Signer +-from msg.storage.git import durable_write ++from msg.atomic_file import durable_write + + + @contextmanager +diff --git a/src/msg/core/codec.py b/src/msg/core/codec.py +index 133d30e..c3cdd9a 100644 +--- a/src/msg/core/codec.py ++++ b/src/msg/core/codec.py +@@ -251,3 +251,11 @@ def record(**kwargs): + cls.__post_init__ = validate_record + return dataclasses.dataclass(**kwargs)(cls) + return decorate ++ ++ ++def result_wire(result): ++ value=wire(result,compact=True) ++ for name in ('replayed','prefer_cli'): ++ if not value.get(name): ++ value.pop(name,None) ++ return value +diff --git a/src/msg/core/executor.py b/src/msg/core/executor.py +index d00ac08..5eb6ba4 100644 +--- a/src/msg/core/executor.py ++++ b/src/msg/core/executor.py +@@ -4,7 +4,7 @@ import logging + import time + from dataclasses import replace + from uuid import uuid4 +-from msg.core.codec import wire,digest ++from msg.core.codec import wire,digest,result_wire + from msg.core.errors import Failure,require + from msg.core.models import ExecutionContext,HandlerOutput,OperationResult,OperationError,Event,AccessRequirement + from msg.core.requests import SECRET_DELIVERY_MIN_VERSION,receipt_bytes +@@ -186,11 +186,3 @@ class OperationExecutor: + request_id=request.request_id,actor=principal.actor,subject=principal.subject,resources=result.resources,data={'operation':spec.name})) + await tx.save_result(principal.subject,request.payload_digest,result) + return replace(result,prefer_cli=request.source!='msg',cli_url='/AGENTS.md' if request.source!='msg' else None) +- +- +-def result_wire(result): +- value=wire(result,compact=True) +- for name in ('replayed','prefer_cli'): +- if not value.get(name): +- value.pop(name,None) +- return value +diff --git a/src/msg/daemon.py b/src/msg/daemon.py +index 637715d..fc6b26b 100644 +--- a/src/msg/daemon.py ++++ b/src/msg/daemon.py +@@ -1,6 +1,7 @@ + """msgd: local installation, serving, workers, diagnostics and restricted SSH.""" + from __future__ import annotations + import argparse ++import importlib.util + import asyncio + from datetime import UTC,datetime + import json +@@ -139,6 +140,11 @@ def parser(): + + def main(argv=None): + args=parser().parse_args(argv) ++ server_modules=('starlette','uvicorn','psycopg','valkey','aiohttp','dns','graphql') ++ if any(importlib.util.find_spec(name) is None for name in server_modules): ++ print(canonical({'status':'error','error':{'code':'server_dependencies_required', ++ 'hint':"Install msg-lmm-best[server] before running server commands."}}).decode(),file=sys.stderr) ++ return 2 + try: + if args.command in {'doctor','selftest'}: + from msg.admin.diagnostics import doctor,selftest +diff --git a/src/msg/security/custodial_migration.py b/src/msg/security/custodial_migration.py +index bcff825..022d59a 100644 +--- a/src/msg/security/custodial_migration.py ++++ b/src/msg/security/custodial_migration.py +@@ -13,6 +13,7 @@ from msg.core.codec import canonical, decode, digest, loads, unb64, wire + from msg.core.errors import Failure, require + from msg.core.models import ResourceRef, Signature + from msg.security.crypto import key_id, verify ++from msg.security.custodial_protocol import stage_statement, ack_statement + from msg.security.age_keys import encryption_key_id, public_from_recipient + from msg.security.backup_retirement import BackupRetirement, verified as verified_backup + +@@ -43,22 +44,6 @@ def inventory_commitment(subject, details, challenge): + 'source_keys': details.get('source_keys', {})}) + + +-def stage_statement(subject, arguments, request_id): +- """Sign every decision field, not a caller-selected subset of the manifest.""" +- return {'domain': 'msg-custodial-decision-v1', 'subject_id': subject, +- 'request_id': request_id, +- 'decision': {key: value for key, value in arguments.items() +- if key != 'migration_ack'}} +- +- +-def ack_statement(subject, challenge_id, inventory_digest, source, target, +- plaintext_digest, request_id, *, method='rewrap'): +- return {'domain': 'msg-custodial-history-ack-v1', 'subject_id': subject, +- 'challenge_id': challenge_id, 'inventory_digest': inventory_digest, +- 'source': source, 'target': target, 'method': method, +- 'plaintext_digest': plaintext_digest, 'request_id': request_id} +- +- + def keyed_source(item, details): + return dict(item, key_id=details.get('source_keys', {}).get( + item['revision'], details['old_encryption_key_id'])) +diff --git a/src/msg/security/custodial_protocol.py b/src/msg/security/custodial_protocol.py +new file mode 100644 +index 0000000..80821ca +--- /dev/null ++++ b/src/msg/security/custodial_protocol.py +@@ -0,0 +1,49 @@ ++"""Published custodial proof and statement bytes shared by both endpoints. ++ ++This module has no server-held key, transaction, current-authority or retirement ++state. Producing a proof does not verify authority or attest key destruction. ++""" ++from __future__ import annotations ++ ++import hmac ++from cryptography.hazmat.primitives.asymmetric.x25519 import X25519PublicKey ++from cryptography.hazmat.primitives.kdf.hkdf import HKDF ++from cryptography.hazmat.primitives import hashes ++from msg.core.codec import b64, canonical, unb64 ++from msg.security.age_keys import private_from_identity ++ ++ ++UPGRADE_CONTEXT_FIELDS=('subject_id','challenge_id','request_id','handle','public_key', ++ 'encryption_recipient','server_public','nonce','expires_at') ++ ++ ++def _proof(shared,challenge): ++ context={name:challenge[name] for name in UPGRADE_CONTEXT_FIELDS} ++ key=HKDF(algorithm=hashes.SHA256(),length=32, ++ salt=unb64(context['nonce'],limit=64), ++ info=b'msg-custodial-upgrade-pop-v1').derive(shared) ++ return b64(hmac.digest(key,canonical(context),'sha256')) ++ ++ ++def client_upgrade_proof(age_identity,challenge): ++ private=private_from_identity(age_identity) ++ shared=private.exchange(X25519PublicKey.from_public_bytes( ++ unb64(challenge['server_public'],limit=32))) ++ return _proof(shared,challenge) ++ ++ ++ ++def stage_statement(subject, arguments, request_id): ++ """Sign every decision field, not a caller-selected subset of the manifest.""" ++ return {'domain': 'msg-custodial-decision-v1', 'subject_id': subject, ++ 'request_id': request_id, ++ 'decision': {key: value for key, value in arguments.items() ++ if key != 'migration_ack'}} ++ ++ ++def ack_statement(subject, challenge_id, inventory_digest, source, target, ++ plaintext_digest, request_id, *, method='rewrap'): ++ return {'domain': 'msg-custodial-history-ack-v1', 'subject_id': subject, ++ 'challenge_id': challenge_id, 'inventory_digest': inventory_digest, ++ 'source': source, 'target': target, 'method': method, ++ 'plaintext_digest': plaintext_digest, 'request_id': request_id} +diff --git a/src/msg/security/vault.py b/src/msg/security/vault.py +index baff9a4..c61f454 100644 +--- a/src/msg/security/vault.py ++++ b/src/msg/security/vault.py +@@ -2,7 +2,6 @@ + from __future__ import annotations + + import os +-import hmac + import shutil + import subprocess + import tempfile +@@ -11,12 +10,11 @@ from pathlib import Path + from cryptography.exceptions import InvalidTag + from cryptography.hazmat.primitives.ciphers.aead import AESGCM + from cryptography.hazmat.primitives.asymmetric.x25519 import X25519PrivateKey,X25519PublicKey +-from cryptography.hazmat.primitives.kdf.hkdf import HKDF +-from cryptography.hazmat.primitives import hashes + + from msg.core.codec import b64,canonical,unb64,wire + from msg.core.errors import Failure,require +-from msg.security.age_keys import recipient_from_identity,private_from_identity ++from msg.security.age_keys import recipient_from_identity ++from msg.security.custodial_protocol import UPGRADE_CONTEXT_FIELDS, _proof, client_upgrade_proof + from msg.security.crypto import Ed25519Signer + + +@@ -128,25 +126,6 @@ def open_upgrade_private(app,subject,challenge_id,nonce,ciphertext): + raise Failure('custodial_upgrade_challenge_corrupt') from exc + + +-UPGRADE_CONTEXT_FIELDS=('subject_id','challenge_id','request_id','handle','public_key', +- 'encryption_recipient','server_public','nonce','expires_at') +- +- +-def _proof(shared,challenge): +- context={name:challenge[name] for name in UPGRADE_CONTEXT_FIELDS} +- key=HKDF(algorithm=hashes.SHA256(),length=32, +- salt=unb64(context['nonce'],limit=64), +- info=b'msg-custodial-upgrade-pop-v1').derive(shared) +- return b64(hmac.digest(key,canonical(context),'sha256')) +- +- +-def client_upgrade_proof(age_identity,challenge): +- private=private_from_identity(age_identity) +- shared=private.exchange(X25519PublicKey.from_public_bytes( +- unb64(challenge['server_public'],limit=32))) +- return _proof(shared,challenge) +- +- + def server_upgrade_proof(app,subject,challenge,nonce,ciphertext): + from msg.security.age_keys import public_from_recipient + private=open_upgrade_private(app,subject,challenge['challenge_id'],nonce,ciphertext) +diff --git a/src/msg/storage/git.py b/src/msg/storage/git.py +index 9846a21..43d51ca 100644 +--- a/src/msg/storage/git.py ++++ b/src/msg/storage/git.py +@@ -17,26 +17,7 @@ from pathlib import Path + from msg.core.codec import canonical, digest, loads + from msg.core.errors import Failure, require + from msg.core.models import BlobRef +- +- +-def durable_write(path: Path, data: bytes, mode: int = 0o600): +- path.parent.mkdir(parents=True,exist_ok=True) +- fd,temporary=tempfile.mkstemp(prefix='.pending-',dir=path.parent) +- try: +- os.fchmod(fd,mode) +- with os.fdopen(fd,'wb') as stream: +- stream.write(data) +- stream.flush() +- os.fsync(stream.fileno()) +- os.replace(temporary,path) +- directory=os.open(path.parent,os.O_RDONLY|os.O_DIRECTORY) +- try: +- os.fsync(directory) +- finally: +- os.close(directory) +- finally: +- if os.path.exists(temporary): +- os.unlink(temporary) ++from msg.atomic_file import durable_write + + + class LFSObjectStore: +diff --git a/src/msg/transports/client.py b/src/msg/transports/client.py +index dee3f63..546fbcf 100644 +--- a/src/msg/transports/client.py ++++ b/src/msg/transports/client.py +@@ -163,7 +163,7 @@ class MCPHTTPTransport(HTTPTransport): + async def call(self, request): + self._secure_delivery(request) + await self._effect(request.operation) +- from msg.transports.mcp import PROTOCOL_VERSION ++ from msg.transports.mcp_protocol import PROTOCOL_VERSION + tool_name = request.operation if request.contract_version==1 else ( + f'{request.operation}@{request.contract_version}') + value = await self._json('POST','/-/mcp',body={'jsonrpc':'2.0','id':request.request_id, +diff --git a/src/msg/transports/mcp.py b/src/msg/transports/mcp.py +index 8474f57..aa81f77 100644 +--- a/src/msg/transports/mcp.py ++++ b/src/msg/transports/mcp.py +@@ -9,11 +9,10 @@ from dataclasses import replace + from msg import __version__ + from msg.core.codec import canonical,digest,loads + from msg.core.errors import Failure,require +-from msg.core.executor import result_wire ++from msg.core.codec import result_wire + from msg.transports.packet import decode_packet,REQUEST_SCHEMA,RESULT_SCHEMA + +-PROTOCOL_VERSION='2025-11-25' +-SUPPORTED_VERSIONS=frozenset({PROTOCOL_VERSION,'2025-06-18','2025-03-26'}) ++from msg.transports.mcp_protocol import PROTOCOL_VERSION, SUPPORTED_VERSIONS + + + class MCPServer: +diff --git a/src/msg/transports/mcp_protocol.py b/src/msg/transports/mcp_protocol.py +new file mode 100644 +index 0000000..7eca1c5 +--- /dev/null ++++ b/src/msg/transports/mcp_protocol.py +@@ -0,0 +1,4 @@ ++"""Published MCP negotiation versions; no server or client runtime imports.""" ++ ++PROTOCOL_VERSION='2025-11-25' ++SUPPORTED_VERSIONS=frozenset({PROTOCOL_VERSION,'2025-06-18','2025-03-26'}) From 2dd5e9b859c3d1c4804c0240350b6e8e087fb29b Mon Sep 17 00:00:00 2001 From: LIghtJUNction Date: Tue, 29 Sep 2026 05:22:28 +0800 Subject: [PATCH 014/136] test(git): lock protocol ownership and shared publication contracts --- .../workflows/git-transport-boundaries.yml | 75 ++++++++++ docs/GIT_BOUNDARY_PROGRESS.md | 56 ++++++++ tests/test_git_transport_boundaries.py | 135 ++++++++++++++++++ 3 files changed, 266 insertions(+) create mode 100644 .github/workflows/git-transport-boundaries.yml create mode 100644 docs/GIT_BOUNDARY_PROGRESS.md create mode 100644 tests/test_git_transport_boundaries.py diff --git a/.github/workflows/git-transport-boundaries.yml b/.github/workflows/git-transport-boundaries.yml new file mode 100644 index 00000000..2f66e02b --- /dev/null +++ b/.github/workflows/git-transport-boundaries.yml @@ -0,0 +1,75 @@ +name: Git transport boundary contracts +on: + push: + branches: [fix/git-transport-boundaries-20260929] + workflow_dispatch: +permissions: + contents: read +concurrency: + group: git-transport-boundaries-${{ github.ref }} + cancel-in-progress: false +jobs: + contracts: + runs-on: ubuntu-latest + timeout-minutes: 20 + services: + postgres: + image: postgres:16 + env: + POSTGRES_USER: msgtest + POSTGRES_PASSWORD: msgtest + POSTGRES_DB: postgres + ports: ['5432:5432'] + options: >- + --health-cmd "pg_isready -U msgtest" + --health-interval 5s --health-timeout 5s --health-retries 10 + valkey: + image: valkey/valkey:9.0.6 + ports: ['6379:6379'] + options: >- + --health-cmd "valkey-cli ping" + --health-interval 5s --health-timeout 5s --health-retries 10 + env: + MSG_TEST_POSTGRES_URL_TEMPLATE: postgresql://msgtest:msgtest@127.0.0.1:5432/{database} + MSG_TEST_VALKEY_URL: redis://127.0.0.1:6379/0 + steps: + - uses: actions/checkout@v7 + with: + persist-credentials: false + - uses: actions/setup-python@v7 + with: + python-version: '3.15' + allow-prereleases: true + cache: pip + - name: Install actual Git, LFS and cryptographic test tools + run: | + sudo apt-get update -qq + sudo apt-get install -y -qq age git-lfs + git --version + git lfs version + python -m pip install -e '.[dev]' + - name: Record exact source + env: + SOURCE_SHA: ${{ github.sha }} + run: | + mkdir -p artifacts + test "$(git rev-parse HEAD)" = "$SOURCE_SHA" + python - <<'PY' + import json,os,subprocess + from pathlib import Path + git=lambda *args:subprocess.check_output(['git',*args],text=True).strip() + Path('artifacts/source.json').write_text(json.dumps({'commit':git('rev-parse','HEAD'), + 'tree':git('rev-parse','HEAD^{tree}'),'event_sha':os.environ['SOURCE_SHA'], + 'dirty':bool(git('status','--porcelain'))},indent=2)+'\n') + PY + - name: Verify ownership and unchanged real Git reads + run: python -m pytest tests/test_git_transport_boundaries.py --junitxml=artifacts/git-boundaries.xml + - name: Preserve real HTTP, SSH, bundle, LFS and publication fences + run: | + python -m pytest tests/test_git_http_push.py tests/test_native_git_read_only.py tests/test_git_lfs.py tests/test_lfs_shared_gc_quota.py tests/test_ssh_rss.py tests/test_extensions.py tests/test_storage_layout.py tests/test_effect_completion_fencing.py tests/issue_78_80/test_lfs_integrity.py tests/issue_78_80/test_lfs_migration_contract.py --junitxml=artifacts/git-regressions.xml + - uses: actions/upload-artifact@v4 + if: always() + with: + name: git-transport-boundaries-${{ github.sha }} + path: artifacts/ + if-no-files-found: error diff --git a/docs/GIT_BOUNDARY_PROGRESS.md b/docs/GIT_BOUNDARY_PROGRESS.md new file mode 100644 index 00000000..21aa088f --- /dev/null +++ b/docs/GIT_BOUNDARY_PROGRESS.md @@ -0,0 +1,56 @@ +# Git transport boundary progress + +## Scope and coordination + +Issue #163. Branch `fix/git-transport-boundaries-20260929`. Base main +`08c58928ee3f6c6cb350d9e9fba75d072bd78f08`, tree +`8a62150d8273a2c75775872694749cb529835966` includes merged #165. +Coordination: #163 comment5878830664 and #83 comment5878871933. +No changes to parallel #166/#167/#169 executor/storage or #155 recovery branches. + +Main was compared with the inspected source baseline476e64f: only #165's eight +files changed, none of the repository/SSH/HTTP modules under review. The source +blobs remain repositories940a1a1bd987a2d97273734a8c0646097abf3c83, +ssh_gitd6465ce37fd7e4743773942144e91e10a7c847af, and +http_routes04a7de0431f1211893111e887ef87661ef877c0f. + +## Regression-first stage + +Nine new tests cover shared import isolation, actual store/adapter ownership, +one shared publication implementation, shared HTTP-helper ownership, explicit +composition and two existing-behavior characterizations (real Git public reads +with unchanged database/refs, and bounded reference batch parsing). +Production source is unchanged in the first commit. Obtain actual red cloud +JUnit/source evidence before implementation; do not claim a result from intent. + +## Observed and intended boundaries + +NativeGitStore currently owns physical repositories and HTTP/LFS Request/Response +handling; HTTP receive imports guarded_command from ssh_git, which imports the +store back. The planned separation is: + +- NativeGitStore keeps repository/LFS paths, bounded store operations and physical + publication primitives, taking a snapshot of required settings rather than + retaining an Application service locator. Existing operation registration and + bundle worker behavior remain in their current module. +- transports.git_http.GitHTTPAdapter composes a NativeGitStore and owns all HTTP + envelopes, streaming/spooling and signed/Basic protocol adaptation. +- extensions.git_publication owns the existing ReferenceGuard and guarded Git + subprocess lifecycle. SSH and HTTP call this same implementation. Historical + ssh_git imports remain identical compatibility aliases, not duplicate wrappers. +- transports.http_common owns the three existing body/JSON/error helpers and base + headers; HTTP routes and Git HTTP import it, avoiding a new router/adapter cycle. + +These are source ownership changes, not a new Git workflow/authorization engine. +Original capacity limits, request IDs, signed envelopes, current-authority checks, +reference-transaction phases, replay cache, uncertainty reporting, LFS digest and +shared-volume constraints, and bundle job fences remain. No protocol version, +database schema, receipt, real repository migration or deployment change. + +## Cloud acceptance + +The dedicated workflow runs real PostgreSQL/Valkey/Git/git-lfs and the existing +HTTP/SSH/bundle/LFS/read-only and completion-fence tests. The full four-shard exact +node-ID/conformance/build and applicable Recovery safety gates remain required on +the final combined tree. No local project execution, skipped test, weakened +assertion, production credentials, real funds/outbound delivery or recovery promotion. diff --git a/tests/test_git_transport_boundaries.py b/tests/test_git_transport_boundaries.py new file mode 100644 index 00000000..19f9c907 --- /dev/null +++ b/tests/test_git_transport_boundaries.py @@ -0,0 +1,135 @@ +"""Git owns files/publication; protocol adapters own requests and responses.""" +from __future__ import annotations + +import ast +import importlib +from pathlib import Path +import subprocess +import sys + +import httpx +import pytest + +from msg.core.errors import Failure +from msg.core.codec import wire +from msg.extensions.repositories import NativeGitStore +from msg.extensions.ssh_git import ReferenceGuard +from msg.transports.http import create_app +from test_service import call, register +from test_native_git_read_only import business_snapshot, refs_snapshot, seed_ref + + +def dependencies(module): + tree=ast.parse(Path(importlib.import_module(module).__file__).read_text()) + return [node.module or '' for node in ast.walk(tree) if isinstance(node,ast.ImportFrom)] + [ + alias.name for node in ast.walk(tree) if isinstance(node,ast.Import) for alias in node.names] + + +def test_shared_repository_module_does_not_import_http_or_ssh_adapters(): + forbidden=('starlette','msg.transports.http','msg.transports.git_http','msg.extensions.ssh_git') + assert not [name for name in dependencies('msg.extensions.repositories') + if any(name==prefix or name.startswith(prefix+'.') for prefix in forbidden)] + + +def test_repository_store_has_no_http_handlers_or_request_conversion_methods(): + assert not {'http','http_push','http_lfs','_http_push_receive','_lfs_packet', + '_lfs_authorize','_lfs_preflight_publish','_repo_read_path','_job'}.intersection(vars(NativeGitStore)) + assert {'path','lfs','lfs_capacity','create','refs','import_bundle','update_refs'}<=set(vars(NativeGitStore)) + + +def test_shared_publication_has_no_protocol_adapter_dependency(): + module=importlib.import_module('msg.extensions.git_publication') + assert not hasattr(module,'receive_pack') + forbidden=('starlette','msg.transports.http','msg.transports.git_http','msg.extensions.ssh_git') + assert not [name for name in dependencies(module.__name__) + if any(name==prefix or name.startswith(prefix+'.') for prefix in forbidden)] + + +def test_both_adapters_use_the_same_publication_implementation(): + owner=importlib.import_module('msg.extensions.git_publication') + ssh=importlib.import_module('msg.extensions.ssh_git') + http=importlib.import_module('msg.transports.git_http') + for name in ('ReferenceGuard','hook_program','relay_bounded_stdin','guarded_command'): + assert getattr(ssh,name) is getattr(owner,name) + assert getattr(owner,name).__module__==owner.__name__ + assert http.guarded_command is owner.guarded_command + assert 'msg.extensions.ssh_git' not in dependencies(http.__name__) + + +def test_http_helpers_have_one_owner_without_a_router_import_cycle(): + common=importlib.import_module('msg.transports.http_common') + routes=importlib.import_module('msg.transports.http_routes') + public=importlib.import_module('msg.transports.http') + git=importlib.import_module('msg.transports.git_http') + for name in ('body_bytes','json_response','error_status'): + assert getattr(common,name) is getattr(routes,name) is getattr(public,name) is getattr(git,name) + assert getattr(common,name).__module__==common.__name__ + assert routes.BASE_HEADERS is common.BASE_HEADERS + assert not {'msg.transports.http','msg.transports.http_routes'}.intersection(dependencies(git.__name__)) + assert not {'msg.transports.http','msg.transports.http_routes','msg.transports.git_http'}.intersection(dependencies(common.__name__)) + + +def test_git_storage_and_publication_import_without_loading_protocol_adapters(): + code=''' +import importlib.abc,sys +class NoAdapters(importlib.abc.MetaPathFinder): + def find_spec(self,fullname,path=None,target=None): + if any(fullname==name or fullname.startswith(name+'.') for name in + ('starlette','msg.transports.http','msg.transports.http_routes', + 'msg.transports.git_http','msg.extensions.ssh_git')): + raise AssertionError('shared Git imported adapter '+fullname) +sys.meta_path.insert(0,NoAdapters()) +from msg.extensions.repositories import NativeGitStore +from msg.extensions.git_publication import ReferenceGuard,guarded_command +''' + result=subprocess.run([sys.executable,'-c',code],capture_output=True,text=True) + assert result.returncode==0,result.stderr + + +async def test_http_uses_explicit_store_composition_not_storage_inheritance(installed): + app,_=installed + adapter=importlib.import_module('msg.transports.git_http').GitHTTPAdapter + store=NativeGitStore(app) + assert not hasattr(store,'app') + assert store.root==app.settings.server.repositories_dir + assert store.blob_root==app.settings.server.blob_dir + http=adapter(app,store=store) + assert http.store is store and http.app is app + assert not isinstance(http,NativeGitStore) + assert '__getattr__' not in vars(adapter) + + +async def test_public_git_read_and_rejected_push_preserve_all_business_state(installed): + app,_=installed + key,uid,_=await register(app,'git-boundary-read') + created=await call(app,'git.create',{'parent':'/@git-boundary-read','name':'code.git'}, + key=key,subject=uid) + assert created.status=='ok',wire(created) + rid=created.resources[0].id + repo=NativeGitStore(app).path(rid) + seed_ref(repo) + before=await business_snapshot(app) + refs=refs_snapshot(repo) + async with httpx.AsyncClient(transport=httpx.ASGITransport(app=create_app(app)), + base_url=app.settings.service_url) as client: + path='/@git-boundary-read/code.git' + head=await client.get(path+'/HEAD') + assert head.status_code==200 and b'refs/heads/main' in head.content + advertised=await client.get(path+'/info/refs?service=git-upload-pack') + assert advertised.status_code==200 and b'git-upload-pack' in advertised.content + rejected=await client.post(path+'/git-receive-pack',content=b'0000') + assert rejected.status_code in {401,403,405} + unauthenticated=await client.get('/-/git/'+rid+'/info/refs?service=git-receive-pack') + assert unauthenticated.status_code==401 + assert refs_snapshot(repo)==refs + assert await business_snapshot(app)==before + + +def test_reference_batch_validation_preserves_exact_git_change_contract(): + old,new='0'*40,'1'*40 + rows=ReferenceGuard.validate_changes(f'{old} {new} refs/heads/main\n{old} {new} refs/tags/v1\n') + assert rows==[{'ref':'refs/heads/main','old':old,'new':new}, + {'ref':'refs/tags/v1','old':old,'new':new}] + for value in ('',f'{old} {new} refs/heads/main\n{old} {new} refs/heads/main', + f'bad {new} refs/heads/main',f'{old} {new} refs/remotes/private'): + with pytest.raises(Failure):ReferenceGuard.validate_changes(value) From 649a85da210bdc5d13f84a4339719f9628b23b7f Mon Sep 17 00:00:00 2001 From: LIghtJUNction Date: Tue, 29 Sep 2026 05:23:27 +0800 Subject: [PATCH 015/136] fix(ci): commit client dependency metadata in the repository's original format The source archive normalizes pyproject formatting; cloud run36484923358 refused that hunk before any publish. Commit the same reviewed dependency split against the live repository file and apply the unchanged verified source patch excluding only that already-committed metadata. Refs #158 #83. --- .github/workflows/prepare-client-boundary.yml | 28 +++++++++++++------ pyproject.toml | 5 ++-- 2 files changed, 22 insertions(+), 11 deletions(-) diff --git a/.github/workflows/prepare-client-boundary.yml b/.github/workflows/prepare-client-boundary.yml index 34249493..5b10f23e 100644 --- a/.github/workflows/prepare-client-boundary.yml +++ b/.github/workflows/prepare-client-boundary.yml @@ -2,7 +2,9 @@ name: Apply reviewed client boundary patch on: push: branches: ['fix/client-boundary-20260929'] - paths: ['scripts/patches/client-boundary.patch'] + paths: + - 'scripts/patches/client-boundary.patch' + - '.github/workflows/prepare-client-boundary.yml' permissions: contents: read jobs: @@ -17,24 +19,25 @@ jobs: with: persist-credentials: false fetch-depth: 2 - - name: Apply only the reviewed patch to its recorded parent + - name: Apply reviewed source patch after explicit repository-format metadata commit run: | - test "$(git rev-parse HEAD^)" = 6ea463d6fa8b256cb07d0ede2d7795a5336eda4f - git apply --index --check scripts/patches/client-boundary.patch - git apply --index scripts/patches/client-boundary.patch + test "$(git rev-parse HEAD^)" = 93d864b3b003310c6a139a918e5e14e65482e404 + echo '244778f572724b991a8b14ff3334b810a3af14329de999bf148c5445950c8a02 scripts/patches/client-boundary.patch' | sha256sum --check + git apply --index --check --exclude=pyproject.toml scripts/patches/client-boundary.patch + git apply --index --exclude=pyproject.toml scripts/patches/client-boundary.patch git diff --cached --check - name: Publish only this feature branch without force env: GH_TOKEN: ${{ github.token }} run: | python - <<'PY' - import hashlib, json, os, pathlib, subprocess, urllib.request + import hashlib, json, os, pathlib, subprocess, tomllib, urllib.request repo = 'TokenNotIncluded/msg.lmm.best' branch = 'fix/client-boundary-20260929' assert os.environ['GITHUB_REPOSITORY'] == repo assert os.environ['GITHUB_REF'] == 'refs/heads/' + branch paths = [ - 'README.md', 'docs/DEPLOYMENT.md', 'pyproject.toml', 'src/msg/atomic_file.py', + 'README.md', 'docs/DEPLOYMENT.md', 'src/msg/atomic_file.py', 'src/msg/cli.py', 'src/msg/client.py', 'src/msg/client_certificates.py', 'src/msg/client_custodial.py', 'src/msg/client_upgrade.py', 'src/msg/core/codec.py', 'src/msg/core/executor.py', 'src/msg/daemon.py', 'src/msg/security/custodial_migration.py', @@ -44,6 +47,13 @@ jobs: ] changed = subprocess.check_output(['git', 'diff', '--cached', '--name-only'], text=True).splitlines() assert sorted(changed) == sorted(paths), changed + metadata = tomllib.loads(pathlib.Path('pyproject.toml').read_text())['project'] + assert metadata['dependencies'] == ['cryptography>=46,<52', 'httpx>=0.28,<1', + 'jsonschema>=4.25,<5', 'referencing>=0.36,<1'] + assert metadata['optional-dependencies']['server'] == [ + 'starlette>=0.50,<2', 'uvicorn>=0.38,<1', 'aiohttp>=3.13,<4', 'dnspython>=2.8,<3', + 'graphql-core>=3.2.7,<3.3', 'psycopg[binary]>=3.3,<4', 'valkey>=6.1,<7'] + assert metadata['optional-dependencies']['dev'][0] == 'msg-lmm-best[server]' def api(method, endpoint, payload=None): data = None if payload is None else json.dumps(payload).encode() request = urllib.request.Request('https://api.github.com/repos/' + repo + endpoint, @@ -57,7 +67,7 @@ jobs: parent = os.environ['GITHUB_SHA'] assert api('GET', '/git/ref/heads/' + branch)['object']['sha'] == parent old = api('GET', '/git/commits/' + parent) - assert [p['sha'] for p in old['parents']] == ['6ea463d6fa8b256cb07d0ede2d7795a5336eda4f'] + assert [p['sha'] for p in old['parents']] == ['93d864b3b003310c6a139a918e5e14e65482e404'] entries = [{'path': p, 'mode': '100644', 'type': 'blob', 'content': pathlib.Path(p).read_text()} for p in paths] for p in ['scripts/patches/client-boundary.patch', '.github/workflows/prepare-client-boundary.yml']: @@ -72,7 +82,7 @@ jobs: pathlib.Path('artifacts/patch-result.json').write_text(json.dumps({ 'source': parent, 'commit': commit['sha'], 'tree': tree['sha'], 'patch_sha256': hashlib.sha256(pathlib.Path('scripts/patches/client-boundary.patch').read_bytes()).hexdigest(), - 'paths': paths, + 'paths': paths, 'metadata_committed_before_patch': True, }, indent=2) + '\n') print('Published reviewed source:', commit['sha'], tree['sha']) PY diff --git a/pyproject.toml b/pyproject.toml index dac7c944..43d0c21a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,14 +10,15 @@ readme = "README.md" requires-python = ">=3.15" license = "MIT" authors = [{name="TokenNotIncluded"}] -dependencies = ["cryptography>=46,<52", "starlette>=0.50,<2", "uvicorn>=0.38,<1", "httpx>=0.28,<1", "jsonschema>=4.25,<5", "referencing>=0.36,<1", "aiohttp>=3.13,<4", "dnspython>=2.8,<3", "graphql-core>=3.2.7,<3.3", "psycopg[binary]>=3.3,<4", "valkey>=6.1,<7"] +dependencies = ["cryptography>=46,<52", "httpx>=0.28,<1", "jsonschema>=4.25,<5", "referencing>=0.36,<1"] [project.scripts] msg = "msg.cli:main" msgd = "msg.daemon:main" [project.optional-dependencies] -dev = ["pytest>=8.4", "pytest-asyncio>=1.1", "pytest-cov>=6", "ruff>=0.14", "build>=1.2", "tiktoken>=0.12"] +server = ["starlette>=0.50,<2", "uvicorn>=0.38,<1", "aiohttp>=3.13,<4", "dnspython>=2.8,<3", "graphql-core>=3.2.7,<3.3", "psycopg[binary]>=3.3,<4", "valkey>=6.1,<7"] +dev = ["msg-lmm-best[server]", "pytest>=8.4", "pytest-asyncio>=1.1", "pytest-cov>=6", "ruff>=0.14", "build>=1.2", "tiktoken>=0.12"] [tool.uv.build-backend] module-name = "msg" From 5f56bbe4ea1cddc46bed12d29026589d65485d62 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 28 Sep 2026 21:24:52 +0000 Subject: [PATCH 016/136] refactor(client): isolate shared protocol helpers and client-only dependencies Refs #158 #82 #83 #85. Preserve signed bytes, credential persistence and explicit full-server installation. --- .github/workflows/prepare-client-boundary.yml | 92 ---- README.md | 9 + docs/DEPLOYMENT.md | 2 +- scripts/patches/client-boundary.patch | 456 ------------------ src/msg/atomic_file.py | 26 + src/msg/cli.py | 2 +- src/msg/client.py | 4 +- src/msg/client_certificates.py | 2 +- src/msg/client_custodial.py | 6 +- src/msg/client_upgrade.py | 2 +- src/msg/core/codec.py | 8 + src/msg/core/executor.py | 10 +- src/msg/daemon.py | 6 + src/msg/security/custodial_migration.py | 17 +- src/msg/security/custodial_protocol.py | 49 ++ src/msg/security/vault.py | 25 +- src/msg/storage/git.py | 21 +- src/msg/transports/client.py | 2 +- src/msg/transports/mcp.py | 5 +- src/msg/transports/mcp_protocol.py | 4 + 20 files changed, 119 insertions(+), 629 deletions(-) delete mode 100644 .github/workflows/prepare-client-boundary.yml delete mode 100644 scripts/patches/client-boundary.patch create mode 100644 src/msg/atomic_file.py create mode 100644 src/msg/security/custodial_protocol.py create mode 100644 src/msg/transports/mcp_protocol.py diff --git a/.github/workflows/prepare-client-boundary.yml b/.github/workflows/prepare-client-boundary.yml deleted file mode 100644 index 5b10f23e..00000000 --- a/.github/workflows/prepare-client-boundary.yml +++ /dev/null @@ -1,92 +0,0 @@ -name: Apply reviewed client boundary patch -on: - push: - branches: ['fix/client-boundary-20260929'] - paths: - - 'scripts/patches/client-boundary.patch' - - '.github/workflows/prepare-client-boundary.yml' -permissions: - contents: read -jobs: - apply-reviewed-patch: - if: github.repository == 'TokenNotIncluded/msg.lmm.best' - runs-on: ubuntu-latest - timeout-minutes: 5 - permissions: - contents: write - steps: - - uses: actions/checkout@v7 - with: - persist-credentials: false - fetch-depth: 2 - - name: Apply reviewed source patch after explicit repository-format metadata commit - run: | - test "$(git rev-parse HEAD^)" = 93d864b3b003310c6a139a918e5e14e65482e404 - echo '244778f572724b991a8b14ff3334b810a3af14329de999bf148c5445950c8a02 scripts/patches/client-boundary.patch' | sha256sum --check - git apply --index --check --exclude=pyproject.toml scripts/patches/client-boundary.patch - git apply --index --exclude=pyproject.toml scripts/patches/client-boundary.patch - git diff --cached --check - - name: Publish only this feature branch without force - env: - GH_TOKEN: ${{ github.token }} - run: | - python - <<'PY' - import hashlib, json, os, pathlib, subprocess, tomllib, urllib.request - repo = 'TokenNotIncluded/msg.lmm.best' - branch = 'fix/client-boundary-20260929' - assert os.environ['GITHUB_REPOSITORY'] == repo - assert os.environ['GITHUB_REF'] == 'refs/heads/' + branch - paths = [ - 'README.md', 'docs/DEPLOYMENT.md', 'src/msg/atomic_file.py', - 'src/msg/cli.py', 'src/msg/client.py', 'src/msg/client_certificates.py', - 'src/msg/client_custodial.py', 'src/msg/client_upgrade.py', 'src/msg/core/codec.py', - 'src/msg/core/executor.py', 'src/msg/daemon.py', 'src/msg/security/custodial_migration.py', - 'src/msg/security/custodial_protocol.py', 'src/msg/security/vault.py', - 'src/msg/storage/git.py', 'src/msg/transports/client.py', 'src/msg/transports/mcp.py', - 'src/msg/transports/mcp_protocol.py', - ] - changed = subprocess.check_output(['git', 'diff', '--cached', '--name-only'], text=True).splitlines() - assert sorted(changed) == sorted(paths), changed - metadata = tomllib.loads(pathlib.Path('pyproject.toml').read_text())['project'] - assert metadata['dependencies'] == ['cryptography>=46,<52', 'httpx>=0.28,<1', - 'jsonschema>=4.25,<5', 'referencing>=0.36,<1'] - assert metadata['optional-dependencies']['server'] == [ - 'starlette>=0.50,<2', 'uvicorn>=0.38,<1', 'aiohttp>=3.13,<4', 'dnspython>=2.8,<3', - 'graphql-core>=3.2.7,<3.3', 'psycopg[binary]>=3.3,<4', 'valkey>=6.1,<7'] - assert metadata['optional-dependencies']['dev'][0] == 'msg-lmm-best[server]' - def api(method, endpoint, payload=None): - data = None if payload is None else json.dumps(payload).encode() - request = urllib.request.Request('https://api.github.com/repos/' + repo + endpoint, - data=data, method=method, headers={ - 'Authorization': 'Bearer ' + os.environ['GH_TOKEN'], - 'Accept': 'application/vnd.github+json', 'Content-Type': 'application/json', - 'X-GitHub-Api-Version': '2022-11-28', - }) - with urllib.request.urlopen(request, timeout=60) as response: - return json.load(response) - parent = os.environ['GITHUB_SHA'] - assert api('GET', '/git/ref/heads/' + branch)['object']['sha'] == parent - old = api('GET', '/git/commits/' + parent) - assert [p['sha'] for p in old['parents']] == ['93d864b3b003310c6a139a918e5e14e65482e404'] - entries = [{'path': p, 'mode': '100644', 'type': 'blob', - 'content': pathlib.Path(p).read_text()} for p in paths] - for p in ['scripts/patches/client-boundary.patch', '.github/workflows/prepare-client-boundary.yml']: - entries.append({'path': p, 'mode': '100644', 'type': 'blob', 'sha': None}) - tree = api('POST', '/git/trees', {'base_tree': old['tree']['sha'], 'tree': entries}) - commit = api('POST', '/git/commits', { - 'message': 'refactor(client): isolate shared protocol helpers and client-only dependencies\n\nRefs #158 #82 #83 #85. Preserve signed bytes, credential persistence and explicit full-server installation.', - 'tree': tree['sha'], 'parents': [parent], - }) - api('PATCH', '/git/refs/heads/' + branch, {'sha': commit['sha'], 'force': False}) - pathlib.Path('artifacts').mkdir(exist_ok=True) - pathlib.Path('artifacts/patch-result.json').write_text(json.dumps({ - 'source': parent, 'commit': commit['sha'], 'tree': tree['sha'], - 'patch_sha256': hashlib.sha256(pathlib.Path('scripts/patches/client-boundary.patch').read_bytes()).hexdigest(), - 'paths': paths, 'metadata_committed_before_patch': True, - }, indent=2) + '\n') - print('Published reviewed source:', commit['sha'], tree['sha']) - PY - - uses: actions/upload-artifact@v4 - with: - name: client-patch-result - path: artifacts/ diff --git a/README.md b/README.md index 74d7a1a8..978ca324 100644 --- a/README.md +++ b/README.md @@ -29,6 +29,15 @@ msg.lmm.best 是一个开放的交流空间。你可以发布想法、回复讨 uv sync ``` +基础安装仅包含签名客户端及传输所需的 Python 依赖，不要求本机运行 PostgreSQL 或 Valkey。 +安装发行包时使用 `python -m pip install msg-lmm-best`。`age` 加密/备份等操作仍需相应系统工具， +并不因为依赖分组而获得空实现或降级执行。 + +部署完整服务端请使用 `uv sync --extra server`，或在源码目录安装 `python -m pip install '.[server]'`。 +从此前默认包含服务端依赖的版本升级时，也应显式加入 `server` extra；`dev` extra 仍包含完整服务端依赖。 +`msg`、`msgd` 的命令名和协议版本不变。仅客户端环境运行服务命令会提示缺少服务端依赖，而不会创建服务状态。 +完整依赖与边界见[客户端安装说明](docs/CLIENT_INSTALLATION.md)。 + 连接到**已启用这个版本**的服务。首次使用时将示例地址换成实际服务地址；客户端会保存服务地址和身份私钥，请妥善保管。 ```bash diff --git a/docs/DEPLOYMENT.md b/docs/DEPLOYMENT.md index 32ac5ef1..45db645c 100644 --- a/docs/DEPLOYMENT.md +++ b/docs/DEPLOYMENT.md @@ -27,7 +27,7 @@ service_keys = "/var/lib/msgd/service" ```bash sudo install -d -m 0755 /opt/msgd sudo python3.15 -m venv /opt/msgd/venv -sudo /opt/msgd/venv/bin/python -m pip install . +sudo /opt/msgd/venv/bin/python -m pip install '.[server]' ``` 在真实本机 VT 或串行控制台运行： diff --git a/scripts/patches/client-boundary.patch b/scripts/patches/client-boundary.patch deleted file mode 100644 index dbeb5aeb..00000000 --- a/scripts/patches/client-boundary.patch +++ /dev/null @@ -1,456 +0,0 @@ -diff --git a/README.md b/README.md -index 74d7a1a..978ca32 100644 ---- a/README.md -+++ b/README.md -@@ -29,6 +29,15 @@ msg.lmm.best 是一个开放的交流空间。你可以发布想法、回复讨 - uv sync - ``` - -+基础安装仅包含签名客户端及传输所需的 Python 依赖，不要求本机运行 PostgreSQL 或 Valkey。 -+安装发行包时使用 `python -m pip install msg-lmm-best`。`age` 加密/备份等操作仍需相应系统工具， -+并不因为依赖分组而获得空实现或降级执行。 -+ -+部署完整服务端请使用 `uv sync --extra server`，或在源码目录安装 `python -m pip install '.[server]'`。 -+从此前默认包含服务端依赖的版本升级时，也应显式加入 `server` extra；`dev` extra 仍包含完整服务端依赖。 -+`msg`、`msgd` 的命令名和协议版本不变。仅客户端环境运行服务命令会提示缺少服务端依赖，而不会创建服务状态。 -+完整依赖与边界见[客户端安装说明](docs/CLIENT_INSTALLATION.md)。 -+ - 连接到**已启用这个版本**的服务。首次使用时将示例地址换成实际服务地址；客户端会保存服务地址和身份私钥，请妥善保管。 - - ```bash -diff --git a/docs/DEPLOYMENT.md b/docs/DEPLOYMENT.md -index 32ac5ef..45db645 100644 ---- a/docs/DEPLOYMENT.md -+++ b/docs/DEPLOYMENT.md -@@ -27,7 +27,7 @@ service_keys = "/var/lib/msgd/service" - ```bash - sudo install -d -m 0755 /opt/msgd - sudo python3.15 -m venv /opt/msgd/venv --sudo /opt/msgd/venv/bin/python -m pip install . -+sudo /opt/msgd/venv/bin/python -m pip install '.[server]' - ``` - - 在真实本机 VT 或串行控制台运行： -diff --git a/pyproject.toml b/pyproject.toml -index cfc5664..cb7cf9f 100644 ---- a/pyproject.toml -+++ b/pyproject.toml -@@ -11,16 +11,9 @@ requires-python = ">=3.15" - license = "MIT" - dependencies = [ - "cryptography>=46,<52", -- "starlette>=0.50,<2", -- "uvicorn>=0.38,<1", - "httpx>=0.28,<1", - "jsonschema>=4.25,<5", - "referencing>=0.36,<1", -- "aiohttp>=3.13,<4", -- "dnspython>=2.8,<3", -- "graphql-core>=3.2.7,<3.3", -- "psycopg[binary]>=3.3,<4", -- "valkey>=6.1,<7", - ] - - [[project.authors]] -@@ -31,7 +24,17 @@ msg = "msg.cli:main" - msgd = "msg.daemon:main" - - [project.optional-dependencies] -+server = [ -+ "starlette>=0.50,<2", -+ "uvicorn>=0.38,<1", -+ "aiohttp>=3.13,<4", -+ "dnspython>=2.8,<3", -+ "graphql-core>=3.2.7,<3.3", -+ "psycopg[binary]>=3.3,<4", -+ "valkey>=6.1,<7", -+] - dev = [ -+ "msg-lmm-best[server]", - "pytest>=8.4", - "pytest-asyncio>=1.1", - "pytest-cov>=6", -diff --git a/src/msg/atomic_file.py b/src/msg/atomic_file.py -new file mode 100644 -index 0000000..beaf703 ---- /dev/null -+++ b/src/msg/atomic_file.py -@@ -0,0 +1,26 @@ -+"""Atomic local-file replacement with caller-selected mode and durable syncs.""" -+from __future__ import annotations -+ -+import os -+from pathlib import Path -+import tempfile -+ -+ -+def durable_write(path: Path, data: bytes, mode: int = 0o600): -+ path.parent.mkdir(parents=True,exist_ok=True) -+ fd,temporary=tempfile.mkstemp(prefix='.pending-',dir=path.parent) -+ try: -+ os.fchmod(fd,mode) -+ with os.fdopen(fd,'wb') as stream: -+ stream.write(data) -+ stream.flush() -+ os.fsync(stream.fileno()) -+ os.replace(temporary,path) -+ directory=os.open(path.parent,os.O_RDONLY|os.O_DIRECTORY) -+ try: -+ os.fsync(directory) -+ finally: -+ os.close(directory) -+ finally: -+ if os.path.exists(temporary): -+ os.unlink(temporary) -diff --git a/src/msg/cli.py b/src/msg/cli.py -index 9913a59..2f3d74f 100644 ---- a/src/msg/cli.py -+++ b/src/msg/cli.py -@@ -10,7 +10,7 @@ from pathlib import Path - from msg.client import ClientState,MsgClient - from msg.core.codec import canonical,loads,wire - from msg.core.errors import Failure,require --from msg.core.executor import result_wire -+from msg.core.codec import result_wire - from msg.core.models import ResourceRef,OperationResult - from msg.transports.client import TRANSPORTS - -diff --git a/src/msg/client.py b/src/msg/client.py -index 144c0ff..326ad17 100644 ---- a/src/msg/client.py -+++ b/src/msg/client.py -@@ -20,8 +20,8 @@ from msg.core.requests import request_for - from msg.security.crypto import Ed25519Signer, subject_id, key_id - from msg.security.age_keys import (generate_age_key,recipient_from_identity, - public_from_recipient,encryption_key_id) --from msg.security.vault import client_upgrade_proof --from msg.storage.git import durable_write -+from msg.security.custodial_protocol import client_upgrade_proof -+from msg.atomic_file import durable_write - from msg.client_tokens import read_journal, remove_journal, token_operation - from msg.transports.client import HTTPTransport, GraphQLTransport, MCPHTTPTransport - -diff --git a/src/msg/client_certificates.py b/src/msg/client_certificates.py -index d8b9c30..75fa957 100644 ---- a/src/msg/client_certificates.py -+++ b/src/msg/client_certificates.py -@@ -12,7 +12,7 @@ from msg.core.errors import require - from msg.core.models import CertificateRequest,Certificate,Signature,CapabilityGrant,IssuancePolicy - from msg.security.certificates import csr_body,verify_csr,sign_certificate - from msg.security.crypto import Ed25519Signer,key_id --from msg.storage.git import durable_write -+from msg.atomic_file import durable_write - - - def local_key(path, *, create=False): -diff --git a/src/msg/client_custodial.py b/src/msg/client_custodial.py -index f79430e..5c162dd 100644 ---- a/src/msg/client_custodial.py -+++ b/src/msg/client_custodial.py -@@ -10,9 +10,9 @@ from msg.client_secrets import write_private - from msg.core.codec import canonical, digest, loads, wire - from msg.core.errors import Failure, require - from msg.security.age_keys import encryption_key_id, public_from_recipient, recipient_from_identity --from msg.security.custodial_migration import ack_statement, stage_statement --from msg.security.vault import client_upgrade_proof --from msg.storage.git import durable_write -+from msg.security.custodial_protocol import ack_statement, stage_statement -+from msg.security.custodial_protocol import client_upgrade_proof -+from msg.atomic_file import durable_write - - - def journal_for(state): -diff --git a/src/msg/client_upgrade.py b/src/msg/client_upgrade.py -index 03c5c92..b68e887 100644 ---- a/src/msg/client_upgrade.py -+++ b/src/msg/client_upgrade.py -@@ -8,7 +8,7 @@ from uuid import uuid4 - from msg.core.codec import b64, canonical, loads, wire - from msg.core.errors import Failure, require - from msg.security.crypto import Ed25519Signer --from msg.storage.git import durable_write -+from msg.atomic_file import durable_write - - - @contextmanager -diff --git a/src/msg/core/codec.py b/src/msg/core/codec.py -index 133d30e..c3cdd9a 100644 ---- a/src/msg/core/codec.py -+++ b/src/msg/core/codec.py -@@ -251,3 +251,11 @@ def record(**kwargs): - cls.__post_init__ = validate_record - return dataclasses.dataclass(**kwargs)(cls) - return decorate -+ -+ -+def result_wire(result): -+ value=wire(result,compact=True) -+ for name in ('replayed','prefer_cli'): -+ if not value.get(name): -+ value.pop(name,None) -+ return value -diff --git a/src/msg/core/executor.py b/src/msg/core/executor.py -index d00ac08..5eb6ba4 100644 ---- a/src/msg/core/executor.py -+++ b/src/msg/core/executor.py -@@ -4,7 +4,7 @@ import logging - import time - from dataclasses import replace - from uuid import uuid4 --from msg.core.codec import wire,digest -+from msg.core.codec import wire,digest,result_wire - from msg.core.errors import Failure,require - from msg.core.models import ExecutionContext,HandlerOutput,OperationResult,OperationError,Event,AccessRequirement - from msg.core.requests import SECRET_DELIVERY_MIN_VERSION,receipt_bytes -@@ -186,11 +186,3 @@ class OperationExecutor: - request_id=request.request_id,actor=principal.actor,subject=principal.subject,resources=result.resources,data={'operation':spec.name})) - await tx.save_result(principal.subject,request.payload_digest,result) - return replace(result,prefer_cli=request.source!='msg',cli_url='/AGENTS.md' if request.source!='msg' else None) -- -- --def result_wire(result): -- value=wire(result,compact=True) -- for name in ('replayed','prefer_cli'): -- if not value.get(name): -- value.pop(name,None) -- return value -diff --git a/src/msg/daemon.py b/src/msg/daemon.py -index 637715d..fc6b26b 100644 ---- a/src/msg/daemon.py -+++ b/src/msg/daemon.py -@@ -1,6 +1,7 @@ - """msgd: local installation, serving, workers, diagnostics and restricted SSH.""" - from __future__ import annotations - import argparse -+import importlib.util - import asyncio - from datetime import UTC,datetime - import json -@@ -139,6 +140,11 @@ def parser(): - - def main(argv=None): - args=parser().parse_args(argv) -+ server_modules=('starlette','uvicorn','psycopg','valkey','aiohttp','dns','graphql') -+ if any(importlib.util.find_spec(name) is None for name in server_modules): -+ print(canonical({'status':'error','error':{'code':'server_dependencies_required', -+ 'hint':"Install msg-lmm-best[server] before running server commands."}}).decode(),file=sys.stderr) -+ return 2 - try: - if args.command in {'doctor','selftest'}: - from msg.admin.diagnostics import doctor,selftest -diff --git a/src/msg/security/custodial_migration.py b/src/msg/security/custodial_migration.py -index bcff825..022d59a 100644 ---- a/src/msg/security/custodial_migration.py -+++ b/src/msg/security/custodial_migration.py -@@ -13,6 +13,7 @@ from msg.core.codec import canonical, decode, digest, loads, unb64, wire - from msg.core.errors import Failure, require - from msg.core.models import ResourceRef, Signature - from msg.security.crypto import key_id, verify -+from msg.security.custodial_protocol import stage_statement, ack_statement - from msg.security.age_keys import encryption_key_id, public_from_recipient - from msg.security.backup_retirement import BackupRetirement, verified as verified_backup - -@@ -43,22 +44,6 @@ def inventory_commitment(subject, details, challenge): - 'source_keys': details.get('source_keys', {})}) - - --def stage_statement(subject, arguments, request_id): -- """Sign every decision field, not a caller-selected subset of the manifest.""" -- return {'domain': 'msg-custodial-decision-v1', 'subject_id': subject, -- 'request_id': request_id, -- 'decision': {key: value for key, value in arguments.items() -- if key != 'migration_ack'}} -- -- --def ack_statement(subject, challenge_id, inventory_digest, source, target, -- plaintext_digest, request_id, *, method='rewrap'): -- return {'domain': 'msg-custodial-history-ack-v1', 'subject_id': subject, -- 'challenge_id': challenge_id, 'inventory_digest': inventory_digest, -- 'source': source, 'target': target, 'method': method, -- 'plaintext_digest': plaintext_digest, 'request_id': request_id} -- -- - def keyed_source(item, details): - return dict(item, key_id=details.get('source_keys', {}).get( - item['revision'], details['old_encryption_key_id'])) -diff --git a/src/msg/security/custodial_protocol.py b/src/msg/security/custodial_protocol.py -new file mode 100644 -index 0000000..80821ca ---- /dev/null -+++ b/src/msg/security/custodial_protocol.py -@@ -0,0 +1,49 @@ -+"""Published custodial proof and statement bytes shared by both endpoints. -+ -+This module has no server-held key, transaction, current-authority or retirement -+state. Producing a proof does not verify authority or attest key destruction. -+""" -+from __future__ import annotations -+ -+import hmac -+from cryptography.hazmat.primitives.asymmetric.x25519 import X25519PublicKey -+from cryptography.hazmat.primitives.kdf.hkdf import HKDF -+from cryptography.hazmat.primitives import hashes -+from msg.core.codec import b64, canonical, unb64 -+from msg.security.age_keys import private_from_identity -+ -+ -+UPGRADE_CONTEXT_FIELDS=('subject_id','challenge_id','request_id','handle','public_key', -+ 'encryption_recipient','server_public','nonce','expires_at') -+ -+ -+def _proof(shared,challenge): -+ context={name:challenge[name] for name in UPGRADE_CONTEXT_FIELDS} -+ key=HKDF(algorithm=hashes.SHA256(),length=32, -+ salt=unb64(context['nonce'],limit=64), -+ info=b'msg-custodial-upgrade-pop-v1').derive(shared) -+ return b64(hmac.digest(key,canonical(context),'sha256')) -+ -+ -+def client_upgrade_proof(age_identity,challenge): -+ private=private_from_identity(age_identity) -+ shared=private.exchange(X25519PublicKey.from_public_bytes( -+ unb64(challenge['server_public'],limit=32))) -+ return _proof(shared,challenge) -+ -+ -+ -+def stage_statement(subject, arguments, request_id): -+ """Sign every decision field, not a caller-selected subset of the manifest.""" -+ return {'domain': 'msg-custodial-decision-v1', 'subject_id': subject, -+ 'request_id': request_id, -+ 'decision': {key: value for key, value in arguments.items() -+ if key != 'migration_ack'}} -+ -+ -+def ack_statement(subject, challenge_id, inventory_digest, source, target, -+ plaintext_digest, request_id, *, method='rewrap'): -+ return {'domain': 'msg-custodial-history-ack-v1', 'subject_id': subject, -+ 'challenge_id': challenge_id, 'inventory_digest': inventory_digest, -+ 'source': source, 'target': target, 'method': method, -+ 'plaintext_digest': plaintext_digest, 'request_id': request_id} -diff --git a/src/msg/security/vault.py b/src/msg/security/vault.py -index baff9a4..c61f454 100644 ---- a/src/msg/security/vault.py -+++ b/src/msg/security/vault.py -@@ -2,7 +2,6 @@ - from __future__ import annotations - - import os --import hmac - import shutil - import subprocess - import tempfile -@@ -11,12 +10,11 @@ from pathlib import Path - from cryptography.exceptions import InvalidTag - from cryptography.hazmat.primitives.ciphers.aead import AESGCM - from cryptography.hazmat.primitives.asymmetric.x25519 import X25519PrivateKey,X25519PublicKey --from cryptography.hazmat.primitives.kdf.hkdf import HKDF --from cryptography.hazmat.primitives import hashes - - from msg.core.codec import b64,canonical,unb64,wire - from msg.core.errors import Failure,require --from msg.security.age_keys import recipient_from_identity,private_from_identity -+from msg.security.age_keys import recipient_from_identity -+from msg.security.custodial_protocol import UPGRADE_CONTEXT_FIELDS, _proof, client_upgrade_proof - from msg.security.crypto import Ed25519Signer - - -@@ -128,25 +126,6 @@ def open_upgrade_private(app,subject,challenge_id,nonce,ciphertext): - raise Failure('custodial_upgrade_challenge_corrupt') from exc - - --UPGRADE_CONTEXT_FIELDS=('subject_id','challenge_id','request_id','handle','public_key', -- 'encryption_recipient','server_public','nonce','expires_at') -- -- --def _proof(shared,challenge): -- context={name:challenge[name] for name in UPGRADE_CONTEXT_FIELDS} -- key=HKDF(algorithm=hashes.SHA256(),length=32, -- salt=unb64(context['nonce'],limit=64), -- info=b'msg-custodial-upgrade-pop-v1').derive(shared) -- return b64(hmac.digest(key,canonical(context),'sha256')) -- -- --def client_upgrade_proof(age_identity,challenge): -- private=private_from_identity(age_identity) -- shared=private.exchange(X25519PublicKey.from_public_bytes( -- unb64(challenge['server_public'],limit=32))) -- return _proof(shared,challenge) -- -- - def server_upgrade_proof(app,subject,challenge,nonce,ciphertext): - from msg.security.age_keys import public_from_recipient - private=open_upgrade_private(app,subject,challenge['challenge_id'],nonce,ciphertext) -diff --git a/src/msg/storage/git.py b/src/msg/storage/git.py -index 9846a21..43d51ca 100644 ---- a/src/msg/storage/git.py -+++ b/src/msg/storage/git.py -@@ -17,26 +17,7 @@ from pathlib import Path - from msg.core.codec import canonical, digest, loads - from msg.core.errors import Failure, require - from msg.core.models import BlobRef -- -- --def durable_write(path: Path, data: bytes, mode: int = 0o600): -- path.parent.mkdir(parents=True,exist_ok=True) -- fd,temporary=tempfile.mkstemp(prefix='.pending-',dir=path.parent) -- try: -- os.fchmod(fd,mode) -- with os.fdopen(fd,'wb') as stream: -- stream.write(data) -- stream.flush() -- os.fsync(stream.fileno()) -- os.replace(temporary,path) -- directory=os.open(path.parent,os.O_RDONLY|os.O_DIRECTORY) -- try: -- os.fsync(directory) -- finally: -- os.close(directory) -- finally: -- if os.path.exists(temporary): -- os.unlink(temporary) -+from msg.atomic_file import durable_write - - - class LFSObjectStore: -diff --git a/src/msg/transports/client.py b/src/msg/transports/client.py -index dee3f63..546fbcf 100644 ---- a/src/msg/transports/client.py -+++ b/src/msg/transports/client.py -@@ -163,7 +163,7 @@ class MCPHTTPTransport(HTTPTransport): - async def call(self, request): - self._secure_delivery(request) - await self._effect(request.operation) -- from msg.transports.mcp import PROTOCOL_VERSION -+ from msg.transports.mcp_protocol import PROTOCOL_VERSION - tool_name = request.operation if request.contract_version==1 else ( - f'{request.operation}@{request.contract_version}') - value = await self._json('POST','/-/mcp',body={'jsonrpc':'2.0','id':request.request_id, -diff --git a/src/msg/transports/mcp.py b/src/msg/transports/mcp.py -index 8474f57..aa81f77 100644 ---- a/src/msg/transports/mcp.py -+++ b/src/msg/transports/mcp.py -@@ -9,11 +9,10 @@ from dataclasses import replace - from msg import __version__ - from msg.core.codec import canonical,digest,loads - from msg.core.errors import Failure,require --from msg.core.executor import result_wire -+from msg.core.codec import result_wire - from msg.transports.packet import decode_packet,REQUEST_SCHEMA,RESULT_SCHEMA - --PROTOCOL_VERSION='2025-11-25' --SUPPORTED_VERSIONS=frozenset({PROTOCOL_VERSION,'2025-06-18','2025-03-26'}) -+from msg.transports.mcp_protocol import PROTOCOL_VERSION, SUPPORTED_VERSIONS - - - class MCPServer: -diff --git a/src/msg/transports/mcp_protocol.py b/src/msg/transports/mcp_protocol.py -new file mode 100644 -index 0000000..7eca1c5 ---- /dev/null -+++ b/src/msg/transports/mcp_protocol.py -@@ -0,0 +1,4 @@ -+"""Published MCP negotiation versions; no server or client runtime imports.""" -+ -+PROTOCOL_VERSION='2025-11-25' -+SUPPORTED_VERSIONS=frozenset({PROTOCOL_VERSION,'2025-06-18','2025-03-26'}) diff --git a/src/msg/atomic_file.py b/src/msg/atomic_file.py new file mode 100644 index 00000000..beaf7034 --- /dev/null +++ b/src/msg/atomic_file.py @@ -0,0 +1,26 @@ +"""Atomic local-file replacement with caller-selected mode and durable syncs.""" +from __future__ import annotations + +import os +from pathlib import Path +import tempfile + + +def durable_write(path: Path, data: bytes, mode: int = 0o600): + path.parent.mkdir(parents=True,exist_ok=True) + fd,temporary=tempfile.mkstemp(prefix='.pending-',dir=path.parent) + try: + os.fchmod(fd,mode) + with os.fdopen(fd,'wb') as stream: + stream.write(data) + stream.flush() + os.fsync(stream.fileno()) + os.replace(temporary,path) + directory=os.open(path.parent,os.O_RDONLY|os.O_DIRECTORY) + try: + os.fsync(directory) + finally: + os.close(directory) + finally: + if os.path.exists(temporary): + os.unlink(temporary) diff --git a/src/msg/cli.py b/src/msg/cli.py index 9913a596..2f3d74f0 100644 --- a/src/msg/cli.py +++ b/src/msg/cli.py @@ -10,7 +10,7 @@ from msg.client import ClientState,MsgClient from msg.core.codec import canonical,loads,wire from msg.core.errors import Failure,require -from msg.core.executor import result_wire +from msg.core.codec import result_wire from msg.core.models import ResourceRef,OperationResult from msg.transports.client import TRANSPORTS diff --git a/src/msg/client.py b/src/msg/client.py index 144c0ff0..326ad17f 100644 --- a/src/msg/client.py +++ b/src/msg/client.py @@ -20,8 +20,8 @@ from msg.security.crypto import Ed25519Signer, subject_id, key_id from msg.security.age_keys import (generate_age_key,recipient_from_identity, public_from_recipient,encryption_key_id) -from msg.security.vault import client_upgrade_proof -from msg.storage.git import durable_write +from msg.security.custodial_protocol import client_upgrade_proof +from msg.atomic_file import durable_write from msg.client_tokens import read_journal, remove_journal, token_operation from msg.transports.client import HTTPTransport, GraphQLTransport, MCPHTTPTransport diff --git a/src/msg/client_certificates.py b/src/msg/client_certificates.py index d8b9c30e..75fa957c 100644 --- a/src/msg/client_certificates.py +++ b/src/msg/client_certificates.py @@ -12,7 +12,7 @@ from msg.core.models import CertificateRequest,Certificate,Signature,CapabilityGrant,IssuancePolicy from msg.security.certificates import csr_body,verify_csr,sign_certificate from msg.security.crypto import Ed25519Signer,key_id -from msg.storage.git import durable_write +from msg.atomic_file import durable_write def local_key(path, *, create=False): diff --git a/src/msg/client_custodial.py b/src/msg/client_custodial.py index f79430eb..5c162dd0 100644 --- a/src/msg/client_custodial.py +++ b/src/msg/client_custodial.py @@ -10,9 +10,9 @@ from msg.core.codec import canonical, digest, loads, wire from msg.core.errors import Failure, require from msg.security.age_keys import encryption_key_id, public_from_recipient, recipient_from_identity -from msg.security.custodial_migration import ack_statement, stage_statement -from msg.security.vault import client_upgrade_proof -from msg.storage.git import durable_write +from msg.security.custodial_protocol import ack_statement, stage_statement +from msg.security.custodial_protocol import client_upgrade_proof +from msg.atomic_file import durable_write def journal_for(state): diff --git a/src/msg/client_upgrade.py b/src/msg/client_upgrade.py index 03c5c924..b68e8872 100644 --- a/src/msg/client_upgrade.py +++ b/src/msg/client_upgrade.py @@ -8,7 +8,7 @@ from msg.core.codec import b64, canonical, loads, wire from msg.core.errors import Failure, require from msg.security.crypto import Ed25519Signer -from msg.storage.git import durable_write +from msg.atomic_file import durable_write @contextmanager diff --git a/src/msg/core/codec.py b/src/msg/core/codec.py index 133d30ef..c3cdd9a5 100644 --- a/src/msg/core/codec.py +++ b/src/msg/core/codec.py @@ -251,3 +251,11 @@ def decorate(cls): cls.__post_init__ = validate_record return dataclasses.dataclass(**kwargs)(cls) return decorate + + +def result_wire(result): + value=wire(result,compact=True) + for name in ('replayed','prefer_cli'): + if not value.get(name): + value.pop(name,None) + return value diff --git a/src/msg/core/executor.py b/src/msg/core/executor.py index d00ac08d..5eb6ba46 100644 --- a/src/msg/core/executor.py +++ b/src/msg/core/executor.py @@ -4,7 +4,7 @@ import time from dataclasses import replace from uuid import uuid4 -from msg.core.codec import wire,digest +from msg.core.codec import wire,digest,result_wire from msg.core.errors import Failure,require from msg.core.models import ExecutionContext,HandlerOutput,OperationResult,OperationError,Event,AccessRequirement from msg.core.requests import SECRET_DELIVERY_MIN_VERSION,receipt_bytes @@ -186,11 +186,3 @@ async def _independent(self,request,spec,entry): request_id=request.request_id,actor=principal.actor,subject=principal.subject,resources=result.resources,data={'operation':spec.name})) await tx.save_result(principal.subject,request.payload_digest,result) return replace(result,prefer_cli=request.source!='msg',cli_url='/AGENTS.md' if request.source!='msg' else None) - - -def result_wire(result): - value=wire(result,compact=True) - for name in ('replayed','prefer_cli'): - if not value.get(name): - value.pop(name,None) - return value diff --git a/src/msg/daemon.py b/src/msg/daemon.py index 637715da..fc6b26b8 100644 --- a/src/msg/daemon.py +++ b/src/msg/daemon.py @@ -1,6 +1,7 @@ """msgd: local installation, serving, workers, diagnostics and restricted SSH.""" from __future__ import annotations import argparse +import importlib.util import asyncio from datetime import UTC,datetime import json @@ -139,6 +140,11 @@ def parser(): def main(argv=None): args=parser().parse_args(argv) + server_modules=('starlette','uvicorn','psycopg','valkey','aiohttp','dns','graphql') + if any(importlib.util.find_spec(name) is None for name in server_modules): + print(canonical({'status':'error','error':{'code':'server_dependencies_required', + 'hint':"Install msg-lmm-best[server] before running server commands."}}).decode(),file=sys.stderr) + return 2 try: if args.command in {'doctor','selftest'}: from msg.admin.diagnostics import doctor,selftest diff --git a/src/msg/security/custodial_migration.py b/src/msg/security/custodial_migration.py index bcff8257..022d59af 100644 --- a/src/msg/security/custodial_migration.py +++ b/src/msg/security/custodial_migration.py @@ -13,6 +13,7 @@ from msg.core.errors import Failure, require from msg.core.models import ResourceRef, Signature from msg.security.crypto import key_id, verify +from msg.security.custodial_protocol import stage_statement, ack_statement from msg.security.age_keys import encryption_key_id, public_from_recipient from msg.security.backup_retirement import BackupRetirement, verified as verified_backup @@ -43,22 +44,6 @@ def inventory_commitment(subject, details, challenge): 'source_keys': details.get('source_keys', {})}) -def stage_statement(subject, arguments, request_id): - """Sign every decision field, not a caller-selected subset of the manifest.""" - return {'domain': 'msg-custodial-decision-v1', 'subject_id': subject, - 'request_id': request_id, - 'decision': {key: value for key, value in arguments.items() - if key != 'migration_ack'}} - - -def ack_statement(subject, challenge_id, inventory_digest, source, target, - plaintext_digest, request_id, *, method='rewrap'): - return {'domain': 'msg-custodial-history-ack-v1', 'subject_id': subject, - 'challenge_id': challenge_id, 'inventory_digest': inventory_digest, - 'source': source, 'target': target, 'method': method, - 'plaintext_digest': plaintext_digest, 'request_id': request_id} - - def keyed_source(item, details): return dict(item, key_id=details.get('source_keys', {}).get( item['revision'], details['old_encryption_key_id'])) diff --git a/src/msg/security/custodial_protocol.py b/src/msg/security/custodial_protocol.py new file mode 100644 index 00000000..80821ca1 --- /dev/null +++ b/src/msg/security/custodial_protocol.py @@ -0,0 +1,49 @@ +"""Published custodial proof and statement bytes shared by both endpoints. + +This module has no server-held key, transaction, current-authority or retirement +state. Producing a proof does not verify authority or attest key destruction. +""" +from __future__ import annotations + +import hmac +from cryptography.hazmat.primitives.asymmetric.x25519 import X25519PublicKey +from cryptography.hazmat.primitives.kdf.hkdf import HKDF +from cryptography.hazmat.primitives import hashes +from msg.core.codec import b64, canonical, unb64 +from msg.security.age_keys import private_from_identity + + +UPGRADE_CONTEXT_FIELDS=('subject_id','challenge_id','request_id','handle','public_key', + 'encryption_recipient','server_public','nonce','expires_at') + + +def _proof(shared,challenge): + context={name:challenge[name] for name in UPGRADE_CONTEXT_FIELDS} + key=HKDF(algorithm=hashes.SHA256(),length=32, + salt=unb64(context['nonce'],limit=64), + info=b'msg-custodial-upgrade-pop-v1').derive(shared) + return b64(hmac.digest(key,canonical(context),'sha256')) + + +def client_upgrade_proof(age_identity,challenge): + private=private_from_identity(age_identity) + shared=private.exchange(X25519PublicKey.from_public_bytes( + unb64(challenge['server_public'],limit=32))) + return _proof(shared,challenge) + + + +def stage_statement(subject, arguments, request_id): + """Sign every decision field, not a caller-selected subset of the manifest.""" + return {'domain': 'msg-custodial-decision-v1', 'subject_id': subject, + 'request_id': request_id, + 'decision': {key: value for key, value in arguments.items() + if key != 'migration_ack'}} + + +def ack_statement(subject, challenge_id, inventory_digest, source, target, + plaintext_digest, request_id, *, method='rewrap'): + return {'domain': 'msg-custodial-history-ack-v1', 'subject_id': subject, + 'challenge_id': challenge_id, 'inventory_digest': inventory_digest, + 'source': source, 'target': target, 'method': method, + 'plaintext_digest': plaintext_digest, 'request_id': request_id} diff --git a/src/msg/security/vault.py b/src/msg/security/vault.py index baff9a43..c61f454e 100644 --- a/src/msg/security/vault.py +++ b/src/msg/security/vault.py @@ -2,7 +2,6 @@ from __future__ import annotations import os -import hmac import shutil import subprocess import tempfile @@ -11,12 +10,11 @@ from cryptography.exceptions import InvalidTag from cryptography.hazmat.primitives.ciphers.aead import AESGCM from cryptography.hazmat.primitives.asymmetric.x25519 import X25519PrivateKey,X25519PublicKey -from cryptography.hazmat.primitives.kdf.hkdf import HKDF -from cryptography.hazmat.primitives import hashes from msg.core.codec import b64,canonical,unb64,wire from msg.core.errors import Failure,require -from msg.security.age_keys import recipient_from_identity,private_from_identity +from msg.security.age_keys import recipient_from_identity +from msg.security.custodial_protocol import UPGRADE_CONTEXT_FIELDS, _proof, client_upgrade_proof from msg.security.crypto import Ed25519Signer @@ -128,25 +126,6 @@ def open_upgrade_private(app,subject,challenge_id,nonce,ciphertext): raise Failure('custodial_upgrade_challenge_corrupt') from exc -UPGRADE_CONTEXT_FIELDS=('subject_id','challenge_id','request_id','handle','public_key', - 'encryption_recipient','server_public','nonce','expires_at') - - -def _proof(shared,challenge): - context={name:challenge[name] for name in UPGRADE_CONTEXT_FIELDS} - key=HKDF(algorithm=hashes.SHA256(),length=32, - salt=unb64(context['nonce'],limit=64), - info=b'msg-custodial-upgrade-pop-v1').derive(shared) - return b64(hmac.digest(key,canonical(context),'sha256')) - - -def client_upgrade_proof(age_identity,challenge): - private=private_from_identity(age_identity) - shared=private.exchange(X25519PublicKey.from_public_bytes( - unb64(challenge['server_public'],limit=32))) - return _proof(shared,challenge) - - def server_upgrade_proof(app,subject,challenge,nonce,ciphertext): from msg.security.age_keys import public_from_recipient private=open_upgrade_private(app,subject,challenge['challenge_id'],nonce,ciphertext) diff --git a/src/msg/storage/git.py b/src/msg/storage/git.py index 9846a21a..43d51ca0 100644 --- a/src/msg/storage/git.py +++ b/src/msg/storage/git.py @@ -17,26 +17,7 @@ from msg.core.codec import canonical, digest, loads from msg.core.errors import Failure, require from msg.core.models import BlobRef - - -def durable_write(path: Path, data: bytes, mode: int = 0o600): - path.parent.mkdir(parents=True,exist_ok=True) - fd,temporary=tempfile.mkstemp(prefix='.pending-',dir=path.parent) - try: - os.fchmod(fd,mode) - with os.fdopen(fd,'wb') as stream: - stream.write(data) - stream.flush() - os.fsync(stream.fileno()) - os.replace(temporary,path) - directory=os.open(path.parent,os.O_RDONLY|os.O_DIRECTORY) - try: - os.fsync(directory) - finally: - os.close(directory) - finally: - if os.path.exists(temporary): - os.unlink(temporary) +from msg.atomic_file import durable_write class LFSObjectStore: diff --git a/src/msg/transports/client.py b/src/msg/transports/client.py index dee3f632..546fbcf2 100644 --- a/src/msg/transports/client.py +++ b/src/msg/transports/client.py @@ -163,7 +163,7 @@ class MCPHTTPTransport(HTTPTransport): async def call(self, request): self._secure_delivery(request) await self._effect(request.operation) - from msg.transports.mcp import PROTOCOL_VERSION + from msg.transports.mcp_protocol import PROTOCOL_VERSION tool_name = request.operation if request.contract_version==1 else ( f'{request.operation}@{request.contract_version}') value = await self._json('POST','/-/mcp',body={'jsonrpc':'2.0','id':request.request_id, diff --git a/src/msg/transports/mcp.py b/src/msg/transports/mcp.py index 8474f57e..aa81f77d 100644 --- a/src/msg/transports/mcp.py +++ b/src/msg/transports/mcp.py @@ -9,11 +9,10 @@ from msg import __version__ from msg.core.codec import canonical,digest,loads from msg.core.errors import Failure,require -from msg.core.executor import result_wire +from msg.core.codec import result_wire from msg.transports.packet import decode_packet,REQUEST_SCHEMA,RESULT_SCHEMA -PROTOCOL_VERSION='2025-11-25' -SUPPORTED_VERSIONS=frozenset({PROTOCOL_VERSION,'2025-06-18','2025-03-26'}) +from msg.transports.mcp_protocol import PROTOCOL_VERSION, SUPPORTED_VERSIONS class MCPServer: diff --git a/src/msg/transports/mcp_protocol.py b/src/msg/transports/mcp_protocol.py new file mode 100644 index 00000000..7eca1c5b --- /dev/null +++ b/src/msg/transports/mcp_protocol.py @@ -0,0 +1,4 @@ +"""Published MCP negotiation versions; no server or client runtime imports.""" + +PROTOCOL_VERSION='2025-11-25' +SUPPORTED_VERSIONS=frozenset({PROTOCOL_VERSION,'2025-06-18','2025-03-26'}) From ee26deca16978ac2e8e4b42095dc17cfe10c175f Mon Sep 17 00:00:00 2001 From: LIghtJUNction Date: Tue, 29 Sep 2026 05:26:48 +0800 Subject: [PATCH 017/136] test(git): characterize supported smart fetch without enabling dumb HTTP --- tests/test_git_transport_boundaries.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/tests/test_git_transport_boundaries.py b/tests/test_git_transport_boundaries.py index 19f9c907..d5a9624d 100644 --- a/tests/test_git_transport_boundaries.py +++ b/tests/test_git_transport_boundaries.py @@ -107,14 +107,19 @@ async def test_public_git_read_and_rejected_push_preserve_all_business_state(ins assert created.status=='ok',wire(created) rid=created.resources[0].id repo=NativeGitStore(app).path(rid) - seed_ref(repo) + commit=seed_ref(repo) before=await business_snapshot(app) refs=refs_snapshot(repo) async with httpx.AsyncClient(transport=httpx.ASGITransport(app=create_app(app)), base_url=app.settings.service_url) as client: path='/@git-boundary-read/code.git' - head=await client.get(path+'/HEAD') - assert head.status_code==200 and b'refs/heads/main' in head.content + # Dumb HTTP HEAD is intentionally disabled by http.getanyfile=false. + # Exercise the supported smart fetch, not a new endpoint contract. + want=f'want {commit}\n'.encode() + packet=f'{len(want)+4:04x}'.encode()+want+b'00000009done\n' + fetched=await client.post(path+'/git-upload-pack',content=packet, + headers={'Content-Type':'application/x-git-upload-pack-request'}) + assert fetched.status_code==200 and b'PACK' in fetched.content, fetched.text advertised=await client.get(path+'/info/refs?service=git-upload-pack') assert advertised.status_code==200 and b'git-upload-pack' in advertised.content rejected=await client.post(path+'/git-receive-pack',content=b'0000') From e7f4da1947c5cfca90378e1e0efc9be2b5efc64c Mon Sep 17 00:00:00 2001 From: LIghtJUNction Date: Tue, 29 Sep 2026 05:27:47 +0800 Subject: [PATCH 018/136] ci: assemble audited Git boundaries without executing project code locally --- .github/tools/assemble_git_boundaries.py | 298 ++++++++++++++++++ .github/workflows/assemble-git-boundaries.yml | 71 +++++ 2 files changed, 369 insertions(+) create mode 100644 .github/tools/assemble_git_boundaries.py create mode 100644 .github/workflows/assemble-git-boundaries.yml diff --git a/.github/tools/assemble_git_boundaries.py b/.github/tools/assemble_git_boundaries.py new file mode 100644 index 00000000..49fdba4f --- /dev/null +++ b/.github/tools/assemble_git_boundaries.py @@ -0,0 +1,298 @@ +"""Exact-source Git boundary extraction; imports no project module.""" +from pathlib import Path +import ast +import copy +import difflib +import hashlib +import json +import re +import subprocess +import textwrap + +ROOT=Path.cwd() +FILES={ + 'repo':'src/msg/extensions/repositories.py', + 'ssh':'src/msg/extensions/ssh_git.py', + 'routes':'src/msg/transports/http_routes.py', +} +EXPECTED={ + 'repo':'940a1a1bd987a2d97273734a8c0646097abf3c83', + 'ssh':'d6465ce37fd7e4743773942144e91e10a7c847af', + 'routes':'04a7de0431f1211893111e887ef87661ef877c0f', +} +HTTP_METHODS=('_lfs_packet','_lfs_authorize','_lfs_preflight_publish','http_lfs', + '_repo_read_path','http','http_push','_http_push_receive','_job') +PUBLICATION=('ReferenceGuard','hook_program','relay_bounded_stdin','guarded_command') +HTTP_HELPERS=('body_bytes','json_response','error_status') +DELEGATED={'path','lfs','root','env'} +before={path:(ROOT/path).read_text() for path in FILES.values()} +after={} +audit=[] +for name,path in FILES.items(): + actual=subprocess.check_output(['git','hash-object',path],text=True).strip() + assert actual==EXPECTED[name],(path,actual) + + +def definitions(text): + return {node.name:node for node in ast.parse(text).body + if isinstance(node,(ast.ClassDef,ast.FunctionDef,ast.AsyncFunctionDef))} + + +def assignments(text): + return {target.id:node for node in ast.parse(text).body if isinstance(node,ast.Assign) + for target in node.targets if isinstance(target,ast.Name)} + + +def methods(node): + return {item.name:item for item in node.body + if isinstance(item,(ast.FunctionDef,ast.AsyncFunctionDef))} + + +def source(text,node): + # Complete physical lines keep method indentation and decorators intact. + begin=min([node.lineno]+[d.lineno for d in getattr(node,'decorator_list',[])])-1 + return ''.join(text.splitlines(True)[begin:node.end_lineno]).rstrip() + + +def delete_nodes(text,nodes): + lines=text.splitlines(True) + for node in sorted(nodes,key=lambda n:n.lineno,reverse=True): + begin=min([node.lineno]+[d.lineno for d in getattr(node,'decorator_list',[])])-1 + del lines[begin:node.end_lineno] + return ''.join(lines) + + +def once(text,old,new): + assert text.count(old)==1,old + return text.replace(old,new) + + +def clean(text): + return re.sub(r'\n{4,}','\n\n\n',text).rstrip()+'\n' + + +repo=before[FILES['repo']] +repo_nodes=definitions(repo) +store=repo_nodes['NativeGitStore'] +old_methods=methods(store) +slots=assignments(repo) +# Remove HTTP methods and stream decoding from the storage/registration module. +new_repo=delete_nodes(repo,[old_methods[name] for name in HTTP_METHODS]+[ + repo_nodes['spool_git_pack'],repo_nodes['spool_lfs_object'], + slots['_GIT_UPLOAD_SLOTS'],slots['_LFS_UPLOAD_SLOTS']]) +new_repo=once(new_repo,'from starlette.requests import ClientDisconnect\n','') +new_repo=once(new_repo,'from starlette.responses import Response,StreamingResponse\n','') +new_repo=once(new_repo,' self.app=app\n',' self.blob_root=app.settings.server.blob_dir\n') +assert new_repo.count('self.app.settings.server.blob_dir')==2 +new_repo=new_repo.replace('self.app.settings.server.blob_dir','self.blob_root') +assert 'self.app' not in source(new_repo,definitions(new_repo)['NativeGitStore']) +after[FILES['repo']]=clean(new_repo) + +# Common HTTP primitives do not depend on the router or Git adapter. +routes=before[FILES['routes']] +route_defs=definitions(routes) +base_headers=assignments(routes)['BASE_HEADERS'] +common='''"""Small HTTP response/body primitives shared without importing the router.""" +from __future__ import annotations +from starlette.responses import Response +from msg.core.codec import canonical +from msg.core.errors import require +from msg.transports.packet import gunzip + +'''+source(routes,base_headers)+'\n\n'+ '\n\n'.join(source(routes,route_defs[name]) for name in HTTP_HELPERS)+'\n' +common_path='src/msg/transports/http_common.py' +assert not (ROOT/common_path).exists() +before[common_path]='' +after[common_path]=clean(common) + +# The HTTP adapter composes the store; no catch-all attribute forwarding or +# inherited storage class keeps a hidden Request/Response dependency. +http='''"""Native Git/LFS HTTP adaptation over the shared store and publication guard.""" +from __future__ import annotations +import asyncio +import base64 +from datetime import timedelta +import hashlib +import os +from pathlib import Path +import re +import tempfile +import time +from uuid import uuid4 + +from starlette.requests import ClientDisconnect +from starlette.responses import Response, StreamingResponse +from msg.core.codec import canonical, wire, unb64 +from msg.core.errors import Failure, require +from msg.core.models import ExecutionContext +from msg.core.requests import request_for +from msg.plugins.common import check_access, resolve +from msg.extensions.repositories import ( + NativeGitStore, MAX_GIT_PACK_BYTES, MAX_GIT_UPLOAD_SECONDS, MAX_LFS_OBJECT_BYTES, + require_git_repository_capacity, _HTTP_RECEIVE, _LFS_STAGED, +) +from msg.extensions.git_publication import guarded_command +from msg.transports.http_common import body_bytes, json_response, error_status +from msg.transports.packet import path_packet + +''' +http+=source(repo,slots['_GIT_UPLOAD_SLOTS'])+'\n'+source(repo,slots['_LFS_UPLOAD_SLOTS'])+'\n\n' +http+='\n\n'.join(source(repo,repo_nodes[name]) for name in ('spool_git_pack','spool_lfs_object'))+'\n\n' +http+='''class GitHTTPAdapter: + def __init__(self, app, *, store: NativeGitStore | None = None): + self.app = app + self.store = store if store is not None else NativeGitStore(app) + +''' +for name in HTTP_METHODS: + body=source(repo,old_methods[name]) + for line in ( + ' from msg.transports.http import body_bytes,json_response\n', + ' from msg.transports.http import body_bytes\n', + ' from msg.transports.http import json_response,error_status\n', + ' from msg.transports.packet import path_packet\n', + ' from msg.extensions.ssh_git import guarded_command\n', + ): + body=body.replace(line,'') + body=re.sub(r'\bself\.(path|lfs|root|env)\b',r'self.store.\1',body) + http+=body+'\n\n' +http_path='src/msg/transports/git_http.py' +assert not (ROOT/http_path).exists() +before[http_path]='' +after[http_path]=clean(http) + +# The publication module is the old trusted guard implementation verbatim, +# minus the SSH-specific entry function. SSH exports compatible object aliases. +ssh=before[FILES['ssh']] +ssh_defs=definitions(ssh) +publication=delete_nodes(ssh,[ssh_defs['receive_pack']]) +publication_path='src/msg/extensions/git_publication.py' +assert not (ROOT/publication_path).exists() +before[publication_path]='' +after[publication_path]=clean(publication) +after[FILES['ssh']]='''"""SSH receive-pack entrypoint over the shared guarded Git publication path.""" +from __future__ import annotations +from msg.core.errors import require +from msg.extensions.repositories import ( + NativeGitStore, MAX_GIT_PACK_BYTES, MAX_GIT_UPLOAD_SECONDS, + require_git_repository_capacity, +) +# Historical imports are the exact shared objects, never alternate wrappers. +from msg.extensions.git_publication import ( + ReferenceGuard, hook_program, relay_bounded_stdin, guarded_command, +) + + +'''+source(ssh,ssh_defs['receive_pack'])+'\n' + +# Routing only selects the HTTP adapter; the outer ingress/recovery boundary +# stays in transports.http and is intentionally not modified. +new_routes=delete_nodes(routes,[base_headers]+[route_defs[name] for name in HTTP_HELPERS]) +new_routes=once(new_routes,'from starlette.routing import Route\n', + 'from starlette.routing import Route\nfrom msg.transports.http_common import BASE_HEADERS, body_bytes, json_response, error_status\n') +assert new_routes.count('from msg.extensions.repositories import NativeGitStore')==3 +assert new_routes.count('NativeGitStore(service)')==4 +new_routes=new_routes.replace('from msg.extensions.repositories import NativeGitStore', + 'from msg.transports.git_http import GitHTTPAdapter') +new_routes=new_routes.replace('NativeGitStore(service)','GitHTTPAdapter(service)') +after[FILES['routes']]=clean(new_routes) + +# Audit all moved algorithm bodies, old registrations, worker publication and +# router dispatch, allowing only the explicitly enumerated dependency rewiring. +class NoImports(ast.NodeTransformer): + def visit_ImportFrom(self,node): return None + def visit_Import(self,node): return None + +class DelegateStore(ast.NodeTransformer): + def visit_Attribute(self,node): + node=self.generic_visit(node) + if isinstance(node.value,ast.Name) and node.value.id=='self' and node.attr in DELEGATED: + node.value=ast.Attribute(value=ast.Name(id='self',ctx=ast.Load()),attr='store',ctx=ast.Load()) + return node + +class RouteAdapter(ast.NodeTransformer): + def visit_Name(self,node): + if node.id=='NativeGitStore':node.id='GitHTTPAdapter' + return node + + +def dump(node): return ast.dump(node,include_attributes=False) +def without_imports(node): return NoImports().visit(copy.deepcopy(node)) + +new_store=methods(definitions(after[FILES['repo']])['NativeGitStore']) +assert set(new_store)==set(old_methods)-set(HTTP_METHODS) +for name,node in new_store.items(): + old=source(repo,old_methods[name]) + if name=='__init__':old=old.replace('self.app=app','self.blob_root=app.settings.server.blob_dir') + old=old.replace('self.app.settings.server.blob_dir','self.blob_root') + assert dump(ast.parse(textwrap.dedent(old)).body[0])==dump(node),name + audit.append('store.'+name+' algorithm unchanged except finite settings snapshot') +for name in ('register','execute_push','git_repository_bytes','require_git_repository_capacity'): + assert dump(repo_nodes[name])==dump(definitions(after[FILES['repo']])[name]),name + audit.append('repository.'+name+' unchanged') +http_methods=methods(definitions(after[http_path])['GitHTTPAdapter']) +for name in HTTP_METHODS: + old=DelegateStore().visit(without_imports(old_methods[name])) + new=without_imports(http_methods[name]) + assert dump(old)==dump(new),name + audit.append('http.'+name+' unchanged except explicit store delegation/imports') +for name in ('spool_git_pack','spool_lfs_object'): + assert dump(repo_nodes[name])==dump(definitions(after[http_path])[name]),name + audit.append(name+' unchanged') +for name in PUBLICATION: + assert dump(ssh_defs[name])==dump(definitions(after[publication_path])[name]),name + audit.append('publication.'+name+' unchanged') +assert dump(ssh_defs['receive_pack'])==dump(definitions(after[FILES['ssh']])['receive_pack']) +audit.append('ssh.receive_pack unchanged') +for name in HTTP_HELPERS: + assert dump(route_defs[name])==dump(definitions(after[common_path])[name]),name + audit.append('http_common.'+name+' unchanged') +assert dump(base_headers)==dump(assignments(after[common_path])['BASE_HEADERS']) +for name,node in route_defs.items(): + if name in HTTP_HELPERS:continue + expected=RouteAdapter().visit(without_imports(node)) + actual=without_imports(definitions(after[FILES['routes']])[name]) + assert dump(expected)==dump(actual),name + audit.append('http_routes.'+name+' unchanged except adapter selection') +for path in (FILES['repo'],publication_path): + imports=[node.module or '' for node in ast.walk(ast.parse(after[path])) if isinstance(node,ast.ImportFrom)] + assert not [name for name in imports if name.startswith(('starlette','msg.transports.http','msg.transports.git_http','msg.extensions.ssh_git'))],path +for path in (http_path,common_path): + assert not [node.module for node in ast.walk(ast.parse(after[path])) if isinstance(node,ast.ImportFrom) + and node.module in {'msg.transports.http','msg.transports.http_routes'}],path + +notes='docs/GIT_BOUNDARY_PROGRESS.md' +before[notes]=(ROOT/notes).read_text() +after[notes]=before[notes]+''' + +## Implemented extraction and source audit + +NativeGitStore no longer owns any HTTP handler, Request/Response conversion, +body spooler or Application property. It snapshots repository/blob paths and +keeps its original physical store algorithms. GitHTTPAdapter explicitly composes +that store and retains existing HTTP/LFS behavior. ContextVar identities used by +registered operations remain singular; upload semaphores move with HTTP spooling. + +The original ReferenceGuard, hook program, bounded stdin relay and guarded Git +command are unchanged in git_publication. SSH exposes same-object aliases and its +original receive_pack entry. HTTP calls the same shared guard. The bundle worker +algorithm, including current_attempt and transaction fencing, is untouched. + +HTTP helpers/base headers have one owner in http_common; routes and Git HTTP use +it. transports.http and its outer passive/recovery boundary are unchanged. The +assembly audits every moved/retained method, every existing registration and +bundle worker body, all shared publication functions, helper functions/headers, +and route functions. Only settings snapshots, explicit store attribute delegation +and import/adapter selection are permitted differences. Source-only AST evidence +is not a successful project test. Final source and patch are uploaded; one-shot +tooling is removed from the submitted tree before focused/full validation. +''' + +out=Path('/tmp/git-boundary-assembly');out.mkdir(parents=True,exist_ok=True) +changed={path:clean(text) for path,text in after.items() if text!=before.get(path,'')} +patch=''.join(''.join(difflib.unified_diff(before.get(path,'').splitlines(True),text.splitlines(True), + fromfile='a/'+path if before.get(path) else '/dev/null',tofile='b/'+path)) for path,text in sorted(changed.items())) +for path,text in changed.items():(ROOT/path).write_text(text) +(out/'changes.patch').write_text(patch) +(out/'audit.json').write_text(json.dumps({'baseline_blobs':EXPECTED,'checked_invariants':audit, + 'changed_paths':sorted(changed),'patch_sha256':hashlib.sha256(patch.encode()).hexdigest()},indent=2)+'\n') diff --git a/.github/workflows/assemble-git-boundaries.yml b/.github/workflows/assemble-git-boundaries.yml new file mode 100644 index 00000000..7e8af931 --- /dev/null +++ b/.github/workflows/assemble-git-boundaries.yml @@ -0,0 +1,71 @@ +name: Assemble audited Git boundary extraction +on: + push: + branches: [build/git-boundaries-170] + paths: + - .github/tools/assemble_git_boundaries.py + - .github/workflows/assemble-git-boundaries.yml +permissions: + contents: write + actions: write +jobs: + assemble: + if: github.repository == 'TokenNotIncluded/msg.lmm.best' && github.ref == 'refs/heads/build/git-boundaries-170' + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - uses: actions/checkout@v7 + with: + fetch-depth: 0 + - name: Avoid duplicate full validation of this source-only helper head + env: + GH_TOKEN: ${{ github.token }} + REPO: ${{ github.repository }} + INPUT_SHA: ${{ github.sha }} + run: | + gh api -X GET "repos/$REPO/actions/runs" -f head_sha="$INPUT_SHA" -f per_page=100 --jq '.workflow_runs[] | select(.path == ".github/workflows/ci.yml" and .event == "push" and .head_branch == "build/git-boundaries-170" and (.status == "queued" or .status == "in_progress")) | .id' | + while read -r run; do + gh api -X POST "repos/$REPO/actions/runs/$run/cancel" || true + done + - name: Require exact regression-first source + env: + INPUT_SHA: ${{ github.sha }} + run: | + test "$(git rev-parse HEAD)" = "$INPUT_SHA" + git merge-base --is-ancestor ee26deca16978ac2e8e4b42095dc17cfe10c175f HEAD + git diff --exit-code ee26deca16978ac2e8e4b42095dc17cfe10c175f -- src tests docs + - name: Assemble and audit source without executing the project + run: python .github/tools/assemble_git_boundaries.py + - name: Commit only audited source and notes + run: | + git add src/msg/extensions/repositories.py src/msg/extensions/ssh_git.py src/msg/extensions/git_publication.py src/msg/transports/git_http.py src/msg/transports/http_common.py src/msg/transports/http_routes.py docs/GIT_BOUNDARY_PROGRESS.md + git diff --cached --check + python - <<'PY' + import json,subprocess + from pathlib import Path + expected=set(json.loads(Path('/tmp/git-boundary-assembly/audit.json').read_text())['changed_paths']) + actual=set(subprocess.check_output(['git','diff','--cached','--name-only'],text=True).splitlines()) + assert actual==expected,(actual-expected,expected-actual) + PY + git -c user.name='github-actions[bot]' -c user.email='41898282+github-actions[bot]@users.noreply.github.com' commit -m 'refactor(git): separate HTTP adaptation and share guarded publication' + - name: Publish only the build branch without force + env: + INPUT_SHA: ${{ github.sha }} + run: | + test "$(git ls-remote origin refs/heads/build/git-boundaries-170 | cut -f1)" = "$INPUT_SHA" + git push origin HEAD:refs/heads/build/git-boundaries-170 + python - <<'PY' + import json,os,subprocess + from pathlib import Path + git=lambda *args:subprocess.check_output(['git',*args],text=True).strip() + Path('/tmp/git-boundary-assembly/source.json').write_text(json.dumps({ + 'input':os.environ['INPUT_SHA'],'commit':git('rev-parse','HEAD'), + 'tree':git('rev-parse','HEAD^{tree}'),'dirty':bool(git('status','--porcelain'))},indent=2)+'\n') + PY + git archive --format=tar.gz -o /tmp/git-boundary-assembly/source.tar.gz HEAD + - uses: actions/upload-artifact@v4 + if: always() + with: + name: git-boundary-assembly + path: /tmp/git-boundary-assembly/ + if-no-files-found: error From b1b6628484bcb50d98e6e1686bd7f9d24cb8d753 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 28 Sep 2026 21:43:12 +0000 Subject: [PATCH 019/136] refactor(git): separate HTTP adaptation and share guarded publication --- docs/GIT_BOUNDARY_PROGRESS.md | 23 ++ src/msg/extensions/git_publication.py | 291 +++++++++++++++++ src/msg/extensions/repositories.py | 418 +------------------------ src/msg/extensions/ssh_git.py | 300 +----------------- src/msg/transports/git_http.py | 434 ++++++++++++++++++++++++++ src/msg/transports/http_common.py | 41 +++ src/msg/transports/http_routes.py | 52 +-- 7 files changed, 810 insertions(+), 749 deletions(-) create mode 100644 src/msg/extensions/git_publication.py create mode 100644 src/msg/transports/git_http.py create mode 100644 src/msg/transports/http_common.py diff --git a/docs/GIT_BOUNDARY_PROGRESS.md b/docs/GIT_BOUNDARY_PROGRESS.md index 21aa088f..50cf39d5 100644 --- a/docs/GIT_BOUNDARY_PROGRESS.md +++ b/docs/GIT_BOUNDARY_PROGRESS.md @@ -54,3 +54,26 @@ HTTP/SSH/bundle/LFS/read-only and completion-fence tests. The full four-shard ex node-ID/conformance/build and applicable Recovery safety gates remain required on the final combined tree. No local project execution, skipped test, weakened assertion, production credentials, real funds/outbound delivery or recovery promotion. + + +## Implemented extraction and source audit + +NativeGitStore no longer owns any HTTP handler, Request/Response conversion, +body spooler or Application property. It snapshots repository/blob paths and +keeps its original physical store algorithms. GitHTTPAdapter explicitly composes +that store and retains existing HTTP/LFS behavior. ContextVar identities used by +registered operations remain singular; upload semaphores move with HTTP spooling. + +The original ReferenceGuard, hook program, bounded stdin relay and guarded Git +command are unchanged in git_publication. SSH exposes same-object aliases and its +original receive_pack entry. HTTP calls the same shared guard. The bundle worker +algorithm, including current_attempt and transaction fencing, is untouched. + +HTTP helpers/base headers have one owner in http_common; routes and Git HTTP use +it. transports.http and its outer passive/recovery boundary are unchanged. The +assembly audits every moved/retained method, every existing registration and +bundle worker body, all shared publication functions, helper functions/headers, +and route functions. Only settings snapshots, explicit store attribute delegation +and import/adapter selection are permitted differences. Source-only AST evidence +is not a successful project test. Final source and patch are uploaded; one-shot +tooling is removed from the submitted tree before focused/full validation. diff --git a/src/msg/extensions/git_publication.py b/src/msg/extensions/git_publication.py new file mode 100644 index 00000000..de5e65f1 --- /dev/null +++ b/src/msg/extensions/git_publication.py @@ -0,0 +1,291 @@ +"""Bridge Git's reference transaction to a current-authority metadata transaction. + +Pack upload happens without holding a PostgreSQL write transaction. The +reference hook reauthorizes at ref preparation. Git and PostgreSQL are not one +atomic storage engine: a lost commit acknowledgement is recorded as uncertain. +""" +from __future__ import annotations + +import asyncio +from dataclasses import replace +import hashlib +import hmac +import os +from pathlib import Path +import secrets +import shlex +import sys +import tempfile + +from msg.core.codec import b64, canonical, loads, wire +from msg.core.errors import Failure, require +from msg.extensions.repositories import (NativeGitStore, MAX_GIT_PACK_BYTES, MAX_GIT_UPLOAD_SECONDS, + require_git_repository_capacity) +from msg.plugins.common import check_access +from msg.storage.git import durable_write +from msg.workers.effects import current_principal, worker_context, effect_request + + +class ReferenceGuard: + def __init__(self, app, job): + self.app, self.job = app, job + self.queue = asyncio.Queue() + self.changed = False + self.uncertain = False + self.failures = [] + + async def authorize(self, tx): + current = await tx.job(self.job.id) + require(current.state == 'running' and current.attempts == self.job.attempts, 'job_lease_lost') + principal = await current_principal(self.app, self.job.principal, tx) + context, request = worker_context(self.app, self.job, principal), effect_request(self.app, self.job, principal) + await check_access(self.app, context, request, tx, self.job.arguments['id'], 'write') + require(tx.setting('runtime_config', {}).get('accept_writes', True), 'writes_paused') + return principal + + @staticmethod + def validate_changes(text): + require(isinstance(text, str) and len(text.encode()) <= 65536, 'git_ref_batch_too_large') + rows = [] + import re + for line in text.splitlines(): + fields = line.split(' ') + require(len(fields) == 3, 'invalid_git_change') + old, new, ref = fields + require(re.fullmatch(r'[0-9a-f]{40}', old) is not None and re.fullmatch(r'[0-9a-f]{40}', new) is not None, + 'invalid_git_change') + require(ref == 'HEAD' or re.fullmatch(r'refs/(heads|tags)/[^\s]+', ref) is not None, 'invalid_git_ref') + rows.append({'ref':ref, 'old':old, 'new':new}) + require(0 < len(rows) <= 128 and len({r['ref'] for r in rows}) == len(rows), 'invalid_git_change') + return rows + + async def run(self): + """One task owns both halves of each PostgreSQL transaction.""" + manager = tx = None + preparing = prepared = None + while True: + message, answer = await self.queue.get() + try: + state = message['state'] + if state == 'stop': + if manager is not None: + self.uncertain = True + await manager.__aexit__(Failure, Failure('external_uncertain'), None) + answer.set_result({'ok':True}) + return + changes = self.validate_changes(message['changes']) + if state == 'preparing': + require(manager is None, 'git_transaction_already_prepared') + # Git 2.54+ invokes this phase before taking ref locks. + # Preflight current authority without holding a PostgreSQL write + # transaction, then re-check inside the prepared phase. + async with self.app.metadata.transaction(write=False) as preflight: + await self.authorize(preflight) + preparing = changes + elif state == 'prepared': + require(manager is None, 'git_transaction_already_prepared') + require(preparing is None or changes == preparing, 'git_transaction_mismatch') + manager = self.app.metadata.transaction(write=True) + tx = await manager.__aenter__() + try: + await self.authorize(tx) + except BaseException: + await manager.__aexit__(*sys.exc_info()) + manager = tx = None + raise + prepared = changes + elif state in {'committed', 'aborted'}: + require(manager is not None and changes == prepared, 'git_transaction_mismatch') + active = manager + manager = None + if state == 'aborted': + await active.__aexit__(Failure, Failure('git_transaction_aborted'), None) + else: + self.changed = True + try: + resource = await tx.resource(self.job.arguments['id']) + await tx.replace(replace(resource, generation=resource.generation + 1, + modified_at=self.app.clock(), modified_by=self.job.principal.actor), resource.generation) + history = tx.setting('git_receive:' + self.job.id, []) + history.append({'time':wire(self.app.clock()), 'refs':changes}) + tx.set_setting('git_receive:' + self.job.id, history) + from msg.core.models import Event, AuditEvent, ResourceRef + from msg.core.codec import digest + from msg.plugins.common import new_id + event = Event(id=new_id('event'), type='git.refs_committed', time=self.app.clock(), + request_id=self.job.arguments['request_id'], actor=self.job.principal.actor, + subject=self.job.principal.subject, resources=(), data={'job_id':self.job.id, 'refs':changes}) + await tx.append_event(event) + await tx.append_audit(AuditEvent(event=event, + authority=tuple(ResourceRef(id=c) for c in self.job.principal.certificates), + before_digest=digest(resource), after_digest=digest(changes), + previous_digest=None, entry_digest='', result='refs_committed')) + await active.__aexit__(None, None, None) + except BaseException: + self.uncertain = True + if not tx.closed: + await active.__aexit__(*sys.exc_info()) + raise + tx = preparing = prepared = None + else: + raise Failure('invalid_git_hook_state') + answer.set_result({'ok':True}) + except BaseException as exc: + code = exc.code if isinstance(exc, Failure) else 'git_guard_failed' + self.failures.append(code) + if not answer.done(): + answer.set_result({'ok':False, 'code':code}) + + async def message(self, value): + answer = asyncio.get_running_loop().create_future() + await self.queue.put((value, answer)) + return await asyncio.shield(answer) + + +def hook_program(socket_path, secret): + """Generated from trusted code, not from a repository or user command.""" + return ('#!' + sys.executable + '\n' + + 'import json,socket,sys\n' + 'data=sys.stdin.buffer.read(65537)\n' + 'if len(data)>65536: sys.exit(1)\n' + 's=socket.socket(socket.AF_UNIX,socket.SOCK_STREAM);s.settimeout(45)\n' + + 's.connect(' + repr(str(socket_path)) + ')\n' + + 's.sendall((json.dumps({"secret":' + repr(secret) + ',"state":sys.argv[1],"changes":data.decode("ascii")})+"\\n").encode())\n' + 'r=s.makefile("rb").readline(8193)\n' + 'sys.exit(0 if r and len(r)<=8192 and json.loads(r).get("ok") else 1)\n').encode() + + +async def relay_bounded_stdin(process, stream, *, limit, timeout): + """Copy stream to process stdin, rejecting once limit bytes have been seen.""" + loop=asyncio.get_running_loop() + total=0 + async def pump(): + nonlocal total + while True: + chunk=await loop.run_in_executor(None, stream.read, 65536) + if not chunk: + break + total+=len(chunk) + require(total<=limit,'request_too_large') + process.stdin.write(chunk) + await process.stdin.drain() + process.stdin.close() + await asyncio.wait_for(pump(), timeout) + return await asyncio.wait_for(process.wait(), timeout) + + +async def guarded_command(app, job, command_factory, *, stdin=None, stdout=None, stderr=None, timeout=600, + input_data=None,input_file=None,capture_output=False,output_limit=None,cache_result_key=None, + stdin_byte_limit=None,stdin_stream=None): + """Run a fixed Git command with a private reference-transaction hook. + + command_factory is installed adapter code, never a value accepted from wire. + Tests exercise this bridge with real update-ref; the SSH adapter uses receive-pack. + """ + store = NativeGitStore(app) + require(input_data is None or input_file is None,'ambiguous_git_input') + require(stdin_byte_limit is None or (input_data is None and input_file is None),'ambiguous_git_input') + app.settings.server.staging_dir.mkdir(parents=True, exist_ok=True) + # Unix socket path has a small OS limit; the random private directory is not + # an account directory and contains no application credentials. + with tempfile.TemporaryDirectory(prefix='msg-git-') as temporary: + directory = Path(temporary) + socket_path = directory/'guard.sock' + secret = secrets.token_hex(32) + guard = ReferenceGuard(app, job) + task = asyncio.create_task(guard.run()) + async def connection(reader, writer): + try: + raw = await asyncio.wait_for(reader.readline(), 45) + require(len(raw) <= 70000, 'git_hook_message_too_large') + value = loads(raw) + require(hmac.compare_digest(value.get('secret',''), secret), 'git_hook_authentication_failed') + result = await guard.message(value) + writer.write(canonical(result) + b'\n') + await writer.drain() + except Exception: + writer.write(b'{"ok":false}\n') + try:await writer.drain() + except (BrokenPipeError, ConnectionError):pass + finally: + writer.close() + await writer.wait_closed() + server = await asyncio.start_unix_server(connection, str(socket_path), limit=70001) + durable_write(directory/'reference-transaction', hook_program(socket_path, secret), mode=0o700) + env = dict(store.env) + command = ['git','--git-dir',str(store.path(job.arguments['id'])),'-c','core.fsync=all', + '-c','core.hooksPath='+str(directory),'-c','receive.fsckObjects=true', + '-c','receive.denyNonFastForwards=true',*command_factory(store)] + process = None + code = None + output = b'' + try: + process = await asyncio.create_subprocess_exec(*command, + stdin=asyncio.subprocess.PIPE if input_data is not None or input_file is not None or stdin_byte_limit is not None else stdin, + stdout=asyncio.subprocess.PIPE if capture_output else stdout, + stderr=asyncio.subprocess.DEVNULL if capture_output else stderr, + env=env,start_new_session=True) + if input_data is not None or input_file is not None: + async def exchange(): + async def feed(): + try: + if input_file is None: + process.stdin.write(input_data) + await process.stdin.drain() + else: + with open(input_file,'rb') as source: + while chunk:=source.read(65536): + process.stdin.write(chunk) + await process.stdin.drain() + except (BrokenPipeError,ConnectionResetError): + pass + finally: + process.stdin.close() + sender=asyncio.create_task(feed()) + try: + chunks=[] + size=0 + while chunk:=await process.stdout.read(65536): + size+=len(chunk) + require(output_limit is None or size<=output_limit,'response_too_large') + chunks.append(chunk) + await sender + await process.wait() + return b''.join(chunks) + finally: + if not sender.done():sender.cancel() + await asyncio.gather(sender,return_exceptions=True) + output=await asyncio.wait_for(exchange(),timeout) + code=process.returncode + else: + if stdin_byte_limit is not None: + code = await relay_bounded_stdin(process, sys.stdin.buffer if stdin_stream is None else stdin_stream, + limit=stdin_byte_limit, timeout=timeout) + else: + code = await asyncio.wait_for(process.wait(), timeout) + finally: + if process is not None and process.returncode is None: + import signal + os.killpg(process.pid, signal.SIGKILL) + await process.wait() + guard.uncertain = True + await guard.message({'state':'stop'}) + await task + server.close() + await server.wait_closed() + async with app.metadata.transaction(write=True) as tx: + current = await tx.job(job.id) + if current.state == 'running': + # Git ref transactions may have partially succeeded without + # --atomic. Audit the actual committed refs, never invent rollback. + uncertain=guard.uncertain or (guard.changed and (code != 0 or bool(guard.failures))) + state = 'uncertain' if uncertain else 'done' if code == 0 and not guard.failures else 'failed' + if state=='done' and cache_result_key is not None: + require(capture_output,'git_result_cache_requires_output') + tx.set_setting(cache_result_key,b64(output)) + await tx.save_job(replace(current, state=state, lease_until=None)) + tx.set_setting('job_status:'+job.id, {'code':'external_uncertain' if uncertain else + guard.failures[0] if guard.failures else 'ok' if state=='done' else 'git_operation_failed', + 'refs_changed':guard.changed}) + result=code if code is not None and not guard.uncertain and not guard.failures else 1 + return (result,output,guard.changed) if capture_output else result diff --git a/src/msg/extensions/repositories.py b/src/msg/extensions/repositories.py index 940a1a1b..eea41eee 100644 --- a/src/msg/extensions/repositories.py +++ b/src/msg/extensions/repositories.py @@ -16,8 +16,6 @@ from uuid import uuid4 from urllib.parse import urlsplit -from starlette.requests import ClientDisconnect -from starlette.responses import Response,StreamingResponse from msg.core.codec import canonical,decode,wire,unb64,b64 from msg.core.errors import Failure,require from msg.core.models import HandlerOutput,ResourceRef,EffectJob,ExecutionContext @@ -36,12 +34,10 @@ MAX_GIT_PACK_BYTES=32*1024*1024 MAX_GIT_UPLOAD_SECONDS=120 MAX_GIT_REPOSITORY_BYTES=2*1024*1024*1024 -_GIT_UPLOAD_SLOTS=asyncio.Semaphore(2) MAX_LFS_OBJECT_BYTES=256*1024*1024 # Deployment-wide, not per-account. Operators may lower this for their shared # volume; PostgreSQL records the value so workers with divergent settings fail. DEFAULT_LFS_DEPLOYMENT_BYTES=4*1024*1024*1024 -_LFS_UPLOAD_SLOTS=asyncio.Semaphore(2) _LFS_STAGED=ContextVar('msg_lfs_staged',default=None) @@ -66,48 +62,9 @@ def require_git_repository_capacity(root,incoming=MAX_GIT_PACK_BYTES): require(git_repository_bytes(root)+incoming<=MAX_GIT_REPOSITORY_BYTES,'storage_capacity_exceeded') -async def spool_git_pack(request,path): - require(request.headers.get('content-encoding','identity')=='identity','unknown_encoding') - length=request.headers.get('content-length') - if length is not None: - require(length.isdecimal() and int(length)<=MAX_GIT_PACK_BYTES,'request_too_large') - hasher=hashlib.sha256() - size=0 - try: - with path.open('xb') as stream: - async for chunk in request.stream(): - size+=len(chunk) - require(size<=MAX_GIT_PACK_BYTES,'request_too_large') - hasher.update(chunk) - stream.write(chunk) - stream.flush() - os.fsync(stream.fileno()) - except ClientDisconnect as exc: - raise Failure('request_incomplete') from exc - return 'sha256:'+hasher.hexdigest(),size - - -async def spool_lfs_object(request,path,expected_size): - require(request.headers.get('content-encoding','identity')=='identity','unknown_encoding') - length=request.headers.get('content-length') - if length is not None: - require(length.isdecimal() and int(length)==expected_size,'lfs_size_mismatch') - hasher=hashlib.sha256();size=0 - try: - with path.open('xb') as stream: - async for chunk in request.stream(): - size+=len(chunk) - require(size<=expected_size,'lfs_size_mismatch') - hasher.update(chunk);stream.write(chunk) - stream.flush();os.fsync(stream.fileno()) - except ClientDisconnect as exc:raise Failure('request_incomplete') from exc - require(size==expected_size,'lfs_size_mismatch') - return hasher.hexdigest(),size - - class NativeGitStore: def __init__(self,app): - self.app=app + self.blob_root=app.settings.server.blob_dir self.root=app.settings.server.repositories_dir self.env={'PATH':os.environ.get('PATH','/usr/bin:/bin'),'LANG':'C.UTF-8','HOME':'/nonexistent', 'GIT_CONFIG_NOSYSTEM':'1','GIT_CONFIG_GLOBAL':'/dev/null','GIT_TERMINAL_PROMPT':'0', @@ -118,7 +75,7 @@ def path(self,id): return self.root/(id+'.git') def lfs(self,id): - return LFSObjectStore(self.path(id),self.app.settings.server.blob_dir) + return LFSObjectStore(self.path(id),self.blob_root) def lfs_capacity(self,tx): configured=os.environ.get('MSG_LFS_DEPLOYMENT_MAX_BYTES',str(DEFAULT_LFS_DEPLOYMENT_BYTES)) @@ -126,7 +83,7 @@ def lfs_capacity(self,tx): limit=int(configured) previous=tx.setting('lfs_deployment_capacity_bytes') require(previous is None or previous==limit,'lfs_deployment_capacity_mismatch') - for root,key in ((self.app.settings.server.blob_dir,'lfs_shared_store_id'), + for root,key in ((self.blob_root,'lfs_shared_store_id'), (self.root,'lfs_shared_repositories_id')): root.mkdir(parents=True,exist_ok=True) sentinel=root/'.msg-shared-store-id' @@ -141,163 +98,6 @@ def lfs_capacity(self,tx): if previous is None:tx.set_setting('lfs_deployment_capacity_bytes',limit) return limit - def _lfs_packet(self,request,operation,arguments): - from msg.transports.packet import path_packet - limits=self.app.settings.server.limits - signed=request.headers.get('x-msg-request') - authorization=request.headers.get('authorization','') - require(not (signed and authorization),'ambiguous_proof') - if signed: - packet=path_packet(signed,'j',limits.max_request_bytes) - require(packet.operation==operation and canonical(packet.arguments)==canonical(arguments), - 'representation_mismatch') - return packet - if authorization: - require(authorization.startswith('Basic '),'authentication_required') - try: - user,password=base64.b64decode(authorization[6:],validate=True).decode('ascii').split(':',1) - token=unb64(password) - except (ValueError,UnicodeDecodeError,Failure) as exc: - raise Failure('invalid_token') from exc - return request_for(operation,arguments,self.app.settings.service_url,token=(user,token), - request_id=uuid4().hex if operation=='git.lfs_publish' else None, - expires_at=self.app.clock()+timedelta(seconds=120)) - return request_for(operation,arguments,self.app.settings.service_url) - - async def _lfs_authorize(self,request,operation,arguments): - result=await self.app.executor.execute(self._lfs_packet(request,operation,arguments),entry='network') - require(result.status=='ok',result.error.code if result.error else 'permission_denied') - return result.data['resource_id'] - - async def _lfs_preflight_publish(self,request,arguments): - """Validate the exact signed publish envelope without committing it.""" - packet=self._lfs_packet(request,'git.lfs_publish',arguments) - spec=self.app.registry.operation(packet.operation,packet.contract_version) - require('network' in spec.entries,'entry_not_allowed') - self.app.registry.validate(spec.input_schema,packet.arguments) - async with self.app.metadata.transaction(write=False) as tx: - principal=await self.app.authenticator.authenticate(packet,tx,entry='network') - require(principal.method=='signature','authentication_required') - context=ExecutionContext(request_id=packet.request_id,principal=principal,entry='network', - now=self.app.clock(),deadline_monotonic=time.monotonic()+30) - rid=await resolve(tx,arguments['id']) - await check_access(self.app,context,packet,tx,rid,'write') - resource=await tx.resource(rid) - require(resource.type=='repo' and resource.state=='active','not_a_repository') - return rid - - async def http_lfs(self,request,repo_id,suffix,*,write=False): - """LFS batch is a read/planning query; only /-/ object PUT commits bytes.""" - from msg.transports.http import body_bytes,json_response - require(not request.query_params,'invalid_lfs_query') - object_match=re.fullmatch(r'objects/([0-9a-f]{64})(?:/([0-9]+))?',suffix) - if object_match: - oid,size_text=object_match.groups() - if write: - require(request.method=='PUT' and size_text is not None,'method_not_allowed') - size=int(size_text) - require(size<=MAX_LFS_OBJECT_BYTES,'request_too_large') - publish_args={'id':repo_id,'oid':oid,'size':size} - if request.headers.get('x-msg-request'): - await self._lfs_preflight_publish(request,publish_args) - else: - await self._lfs_authorize(request,'git.lfs_write_authorize',{'id':repo_id}) - self.app.settings.server.staging_dir.mkdir(parents=True,exist_ok=True) - async with _LFS_UPLOAD_SLOTS: - with tempfile.TemporaryDirectory(prefix='msg-lfs-',dir=self.app.settings.server.staging_dir) as temp: - staged=Path(temp)/'object' - try: - async with asyncio.timeout(MAX_GIT_UPLOAD_SECONDS): - digest,actual=await spool_lfs_object(request,staged,size) - except TimeoutError as exc: - raise Failure('request_incomplete') from exc - require(digest==oid,'lfs_digest_mismatch') - marker=_LFS_STAGED.set(staged) - try: - await self._lfs_authorize(request,'git.lfs_publish',publish_args) - finally:_LFS_STAGED.reset(marker) - return Response(status_code=200,headers={'Cache-Control':'no-store'}) - require(request.method in {'GET','HEAD'} and size_text is None,'method_not_allowed') - rid=await self._lfs_authorize(request,'git.lfs_read',{'id':repo_id,'oid':oid}) - path=self.lfs(rid).path(oid) - require(path.is_file(),'not_found') - size=path.stat().st_size - etag='"sha256:'+oid+'"' - begin,end,status=0,size,200 - requested=request.headers.get('range') - if requested and request.headers.get('if-range') in {None,etag}: - match=re.fullmatch(r'bytes=(\d*)-(\d*)',requested) if len(requested)<=128 else None - if match and any(match.groups()): - left,right=match.groups() - if left: - begin=int(left) - end=min(size,int(right)+1) if right else size - valid=begin0 and begin65536: sys.exit(1)\n' - 's=socket.socket(socket.AF_UNIX,socket.SOCK_STREAM);s.settimeout(45)\n' + - 's.connect(' + repr(str(socket_path)) + ')\n' + - 's.sendall((json.dumps({"secret":' + repr(secret) + ',"state":sys.argv[1],"changes":data.decode("ascii")})+"\\n").encode())\n' - 'r=s.makefile("rb").readline(8193)\n' - 'sys.exit(0 if r and len(r)<=8192 and json.loads(r).get("ok") else 1)\n').encode() - - -async def relay_bounded_stdin(process, stream, *, limit, timeout): - """Copy stream to process stdin, rejecting once limit bytes have been seen.""" - loop=asyncio.get_running_loop() - total=0 - async def pump(): - nonlocal total - while True: - chunk=await loop.run_in_executor(None, stream.read, 65536) - if not chunk: - break - total+=len(chunk) - require(total<=limit,'request_too_large') - process.stdin.write(chunk) - await process.stdin.drain() - process.stdin.close() - await asyncio.wait_for(pump(), timeout) - return await asyncio.wait_for(process.wait(), timeout) - - -async def guarded_command(app, job, command_factory, *, stdin=None, stdout=None, stderr=None, timeout=600, - input_data=None,input_file=None,capture_output=False,output_limit=None,cache_result_key=None, - stdin_byte_limit=None,stdin_stream=None): - """Run a fixed Git command with a private reference-transaction hook. - - command_factory is installed adapter code, never a value accepted from wire. - Tests exercise this bridge with real update-ref; the SSH adapter uses receive-pack. - """ - store = NativeGitStore(app) - require(input_data is None or input_file is None,'ambiguous_git_input') - require(stdin_byte_limit is None or (input_data is None and input_file is None),'ambiguous_git_input') - app.settings.server.staging_dir.mkdir(parents=True, exist_ok=True) - # Unix socket path has a small OS limit; the random private directory is not - # an account directory and contains no application credentials. - with tempfile.TemporaryDirectory(prefix='msg-git-') as temporary: - directory = Path(temporary) - socket_path = directory/'guard.sock' - secret = secrets.token_hex(32) - guard = ReferenceGuard(app, job) - task = asyncio.create_task(guard.run()) - async def connection(reader, writer): - try: - raw = await asyncio.wait_for(reader.readline(), 45) - require(len(raw) <= 70000, 'git_hook_message_too_large') - value = loads(raw) - require(hmac.compare_digest(value.get('secret',''), secret), 'git_hook_authentication_failed') - result = await guard.message(value) - writer.write(canonical(result) + b'\n') - await writer.drain() - except Exception: - writer.write(b'{"ok":false}\n') - try:await writer.drain() - except (BrokenPipeError, ConnectionError):pass - finally: - writer.close() - await writer.wait_closed() - server = await asyncio.start_unix_server(connection, str(socket_path), limit=70001) - durable_write(directory/'reference-transaction', hook_program(socket_path, secret), mode=0o700) - env = dict(store.env) - command = ['git','--git-dir',str(store.path(job.arguments['id'])),'-c','core.fsync=all', - '-c','core.hooksPath='+str(directory),'-c','receive.fsckObjects=true', - '-c','receive.denyNonFastForwards=true',*command_factory(store)] - process = None - code = None - output = b'' - try: - process = await asyncio.create_subprocess_exec(*command, - stdin=asyncio.subprocess.PIPE if input_data is not None or input_file is not None or stdin_byte_limit is not None else stdin, - stdout=asyncio.subprocess.PIPE if capture_output else stdout, - stderr=asyncio.subprocess.DEVNULL if capture_output else stderr, - env=env,start_new_session=True) - if input_data is not None or input_file is not None: - async def exchange(): - async def feed(): - try: - if input_file is None: - process.stdin.write(input_data) - await process.stdin.drain() - else: - with open(input_file,'rb') as source: - while chunk:=source.read(65536): - process.stdin.write(chunk) - await process.stdin.drain() - except (BrokenPipeError,ConnectionResetError): - pass - finally: - process.stdin.close() - sender=asyncio.create_task(feed()) - try: - chunks=[] - size=0 - while chunk:=await process.stdout.read(65536): - size+=len(chunk) - require(output_limit is None or size<=output_limit,'response_too_large') - chunks.append(chunk) - await sender - await process.wait() - return b''.join(chunks) - finally: - if not sender.done():sender.cancel() - await asyncio.gather(sender,return_exceptions=True) - output=await asyncio.wait_for(exchange(),timeout) - code=process.returncode - else: - if stdin_byte_limit is not None: - code = await relay_bounded_stdin(process, sys.stdin.buffer if stdin_stream is None else stdin_stream, - limit=stdin_byte_limit, timeout=timeout) - else: - code = await asyncio.wait_for(process.wait(), timeout) - finally: - if process is not None and process.returncode is None: - import signal - os.killpg(process.pid, signal.SIGKILL) - await process.wait() - guard.uncertain = True - await guard.message({'state':'stop'}) - await task - server.close() - await server.wait_closed() - async with app.metadata.transaction(write=True) as tx: - current = await tx.job(job.id) - if current.state == 'running': - # Git ref transactions may have partially succeeded without - # --atomic. Audit the actual committed refs, never invent rollback. - uncertain=guard.uncertain or (guard.changed and (code != 0 or bool(guard.failures))) - state = 'uncertain' if uncertain else 'done' if code == 0 and not guard.failures else 'failed' - if state=='done' and cache_result_key is not None: - require(capture_output,'git_result_cache_requires_output') - tx.set_setting(cache_result_key,b64(output)) - await tx.save_job(replace(current, state=state, lease_until=None)) - tx.set_setting('job_status:'+job.id, {'code':'external_uncertain' if uncertain else - guard.failures[0] if guard.failures else 'ok' if state=='done' else 'git_operation_failed', - 'refs_changed':guard.changed}) - result=code if code is not None and not guard.uncertain and not guard.failures else 1 - return (result,output,guard.changed) if capture_output else result +from msg.core.errors import require +from msg.extensions.repositories import ( + NativeGitStore, MAX_GIT_PACK_BYTES, MAX_GIT_UPLOAD_SECONDS, + require_git_repository_capacity, +) +# Historical imports are the exact shared objects, never alternate wrappers. +from msg.extensions.git_publication import ( + ReferenceGuard, hook_program, relay_bounded_stdin, guarded_command, +) async def receive_pack(app, job_id, *, stdin_stream=None): diff --git a/src/msg/transports/git_http.py b/src/msg/transports/git_http.py new file mode 100644 index 00000000..934708ef --- /dev/null +++ b/src/msg/transports/git_http.py @@ -0,0 +1,434 @@ +"""Native Git/LFS HTTP adaptation over the shared store and publication guard.""" +from __future__ import annotations +import asyncio +import base64 +from datetime import timedelta +import hashlib +import os +from pathlib import Path +import re +import tempfile +import time +from uuid import uuid4 + +from starlette.requests import ClientDisconnect +from starlette.responses import Response, StreamingResponse +from msg.core.codec import canonical, wire, unb64 +from msg.core.errors import Failure, require +from msg.core.models import ExecutionContext +from msg.core.requests import request_for +from msg.plugins.common import check_access, resolve +from msg.extensions.repositories import ( + NativeGitStore, MAX_GIT_PACK_BYTES, MAX_GIT_UPLOAD_SECONDS, MAX_LFS_OBJECT_BYTES, + require_git_repository_capacity, _HTTP_RECEIVE, _LFS_STAGED, +) +from msg.extensions.git_publication import guarded_command +from msg.transports.http_common import body_bytes, json_response, error_status +from msg.transports.packet import path_packet + +_GIT_UPLOAD_SLOTS=asyncio.Semaphore(2) +_LFS_UPLOAD_SLOTS=asyncio.Semaphore(2) + +async def spool_git_pack(request,path): + require(request.headers.get('content-encoding','identity')=='identity','unknown_encoding') + length=request.headers.get('content-length') + if length is not None: + require(length.isdecimal() and int(length)<=MAX_GIT_PACK_BYTES,'request_too_large') + hasher=hashlib.sha256() + size=0 + try: + with path.open('xb') as stream: + async for chunk in request.stream(): + size+=len(chunk) + require(size<=MAX_GIT_PACK_BYTES,'request_too_large') + hasher.update(chunk) + stream.write(chunk) + stream.flush() + os.fsync(stream.fileno()) + except ClientDisconnect as exc: + raise Failure('request_incomplete') from exc + return 'sha256:'+hasher.hexdigest(),size + +async def spool_lfs_object(request,path,expected_size): + require(request.headers.get('content-encoding','identity')=='identity','unknown_encoding') + length=request.headers.get('content-length') + if length is not None: + require(length.isdecimal() and int(length)==expected_size,'lfs_size_mismatch') + hasher=hashlib.sha256();size=0 + try: + with path.open('xb') as stream: + async for chunk in request.stream(): + size+=len(chunk) + require(size<=expected_size,'lfs_size_mismatch') + hasher.update(chunk);stream.write(chunk) + stream.flush();os.fsync(stream.fileno()) + except ClientDisconnect as exc:raise Failure('request_incomplete') from exc + require(size==expected_size,'lfs_size_mismatch') + return hasher.hexdigest(),size + +class GitHTTPAdapter: + def __init__(self, app, *, store: NativeGitStore | None = None): + self.app = app + self.store = store if store is not None else NativeGitStore(app) + + def _lfs_packet(self,request,operation,arguments): + limits=self.app.settings.server.limits + signed=request.headers.get('x-msg-request') + authorization=request.headers.get('authorization','') + require(not (signed and authorization),'ambiguous_proof') + if signed: + packet=path_packet(signed,'j',limits.max_request_bytes) + require(packet.operation==operation and canonical(packet.arguments)==canonical(arguments), + 'representation_mismatch') + return packet + if authorization: + require(authorization.startswith('Basic '),'authentication_required') + try: + user,password=base64.b64decode(authorization[6:],validate=True).decode('ascii').split(':',1) + token=unb64(password) + except (ValueError,UnicodeDecodeError,Failure) as exc: + raise Failure('invalid_token') from exc + return request_for(operation,arguments,self.app.settings.service_url,token=(user,token), + request_id=uuid4().hex if operation=='git.lfs_publish' else None, + expires_at=self.app.clock()+timedelta(seconds=120)) + return request_for(operation,arguments,self.app.settings.service_url) + + async def _lfs_authorize(self,request,operation,arguments): + result=await self.app.executor.execute(self._lfs_packet(request,operation,arguments),entry='network') + require(result.status=='ok',result.error.code if result.error else 'permission_denied') + return result.data['resource_id'] + + async def _lfs_preflight_publish(self,request,arguments): + """Validate the exact signed publish envelope without committing it.""" + packet=self._lfs_packet(request,'git.lfs_publish',arguments) + spec=self.app.registry.operation(packet.operation,packet.contract_version) + require('network' in spec.entries,'entry_not_allowed') + self.app.registry.validate(spec.input_schema,packet.arguments) + async with self.app.metadata.transaction(write=False) as tx: + principal=await self.app.authenticator.authenticate(packet,tx,entry='network') + require(principal.method=='signature','authentication_required') + context=ExecutionContext(request_id=packet.request_id,principal=principal,entry='network', + now=self.app.clock(),deadline_monotonic=time.monotonic()+30) + rid=await resolve(tx,arguments['id']) + await check_access(self.app,context,packet,tx,rid,'write') + resource=await tx.resource(rid) + require(resource.type=='repo' and resource.state=='active','not_a_repository') + return rid + + async def http_lfs(self,request,repo_id,suffix,*,write=False): + """LFS batch is a read/planning query; only /-/ object PUT commits bytes.""" + require(not request.query_params,'invalid_lfs_query') + object_match=re.fullmatch(r'objects/([0-9a-f]{64})(?:/([0-9]+))?',suffix) + if object_match: + oid,size_text=object_match.groups() + if write: + require(request.method=='PUT' and size_text is not None,'method_not_allowed') + size=int(size_text) + require(size<=MAX_LFS_OBJECT_BYTES,'request_too_large') + publish_args={'id':repo_id,'oid':oid,'size':size} + if request.headers.get('x-msg-request'): + await self._lfs_preflight_publish(request,publish_args) + else: + await self._lfs_authorize(request,'git.lfs_write_authorize',{'id':repo_id}) + self.app.settings.server.staging_dir.mkdir(parents=True,exist_ok=True) + async with _LFS_UPLOAD_SLOTS: + with tempfile.TemporaryDirectory(prefix='msg-lfs-',dir=self.app.settings.server.staging_dir) as temp: + staged=Path(temp)/'object' + try: + async with asyncio.timeout(MAX_GIT_UPLOAD_SECONDS): + digest,actual=await spool_lfs_object(request,staged,size) + except TimeoutError as exc: + raise Failure('request_incomplete') from exc + require(digest==oid,'lfs_digest_mismatch') + marker=_LFS_STAGED.set(staged) + try: + await self._lfs_authorize(request,'git.lfs_publish',publish_args) + finally:_LFS_STAGED.reset(marker) + return Response(status_code=200,headers={'Cache-Control':'no-store'}) + require(request.method in {'GET','HEAD'} and size_text is None,'method_not_allowed') + rid=await self._lfs_authorize(request,'git.lfs_read',{'id':repo_id,'oid':oid}) + path=self.store.lfs(rid).path(oid) + require(path.is_file(),'not_found') + size=path.stat().st_size + etag='"sha256:'+oid+'"' + begin,end,status=0,size,200 + requested=request.headers.get('range') + if requested and request.headers.get('if-range') in {None,etag}: + match=re.fullmatch(r'bytes=(\d*)-(\d*)',requested) if len(requested)<=128 else None + if match and any(match.groups()): + left,right=match.groups() + if left: + begin=int(left) + end=min(size,int(right)+1) if right else size + valid=begin0 and begin @@ -374,41 +373,6 @@ def path_read_proof(encoded,operation,args,service,limit): return packet -def json_response(value,status=200,headers=None): - return Response(canonical(value),status_code=status,media_type='application/json',headers={**BASE_HEADERS,**(headers or {})}) - - -def error_status(code): - if code=='range_not_satisfiable':return 416 - if code in {'not_found','resource_purged','revision_not_found','csr_not_found', - 'certificate_not_found','listing_not_found','package_not_found', - 'bounty_not_found','order_not_found','delivery_not_found', - 'offer_not_found'}: return 404 - if code in {'authentication_required','invalid_token','invalid_signature','credential_revoked','credential_expired','request_expired'}: return 401 - if code in {'permission_denied','local_only','credential_ceiling','certificate_gate','tool_certificate_required','forbidden_origin','forbidden_host','passive_client_forbidden','query_ref_principal_mismatch','cursor_principal_mismatch'}: return 403 - if code in {'generation_conflict','revision_conflict','idempotency_conflict','chunk_conflict','constraint_conflict'}: return 409 - if code in {'request_too_large','path_too_large','response_too_large','use_transfer','part_too_large'}: return 413 - if code in {'method_not_allowed','effect_mismatch'}: return 405 - if code=='secure_channel_required': return 400 - if code in {'server_busy','issuer_not_ready','dependency_unavailable','service_restart_required','writes_paused'}: return 503 - if code=='storage_capacity_exceeded': return 507 - if code=='internal_error': return 500 - return 400 - - -async def body_bytes(request,limit): - length=request.headers.get('content-length') - if length is not None: - require(length.isdecimal() and int(length)<=limit,'request_too_large') - body=bytearray() - async for data in request.stream(): - require(len(body)+len(data)<=limit,'request_too_large') - body.extend(data) - encoding=request.headers.get('content-encoding','identity') - require(encoding in {'identity','gzip'},'unknown_encoding') - return gunzip(bytes(body),limit) if encoding=='gzip' else bytes(body) - - def describe_resource(data): if 'content' in data: content=data['content'] @@ -496,26 +460,26 @@ async def dispatch(request:Request): native=re.fullmatch(r'(/[@&][^/]+/[^/]+\.git)/(.*)',path) if native: require(service.registry.operation('git.refs').effect=='read','effect_mismatch') - from msg.extensions.repositories import NativeGitStore + from msg.transports.git_http import GitHTTPAdapter if native.group(2).startswith('info/lfs/'): require(service.registry.operation('git.lfs_read').effect=='read' and service.registry.operation('git.lfs_read_batch').effect=='read', 'effect_mismatch') - return await NativeGitStore(service).http_lfs(request,native.group(1), + return await GitHTTPAdapter(service).http_lfs(request,native.group(1), native.group(2)[9:]) - return await NativeGitStore(service).http(request,native.group(1),native.group(2)) + return await GitHTTPAdapter(service).http(request,native.group(1),native.group(2)) # git-lfs derives .git/info/lfs even when the advertised # push URL is /-/git/; both spellings remain inside /-/. lfs_write=re.fullmatch(r'/-/git/([A-Za-z0-9_-]{1,128})(?:\.git)?/info/lfs/(objects(?:/batch|/[0-9a-f]{64}/[0-9]+))',path) if lfs_write: require(raw_path==path.encode('ascii'),'not_found') - from msg.extensions.repositories import NativeGitStore - return await NativeGitStore(service).http_lfs(request,*lfs_write.groups(),write=True) + from msg.transports.git_http import GitHTTPAdapter + return await GitHTTPAdapter(service).http_lfs(request,*lfs_write.groups(),write=True) git_push=re.fullmatch(r'/-/git/([A-Za-z0-9_-]{1,128})/(info/refs|git-receive-pack)',path) if git_push: require(raw_path==path.encode('ascii'),'not_found') - from msg.extensions.repositories import NativeGitStore - return await NativeGitStore(service).http_push(request,*git_push.groups()) + from msg.transports.git_http import GitHTTPAdapter + return await GitHTTPAdapter(service).http_push(request,*git_push.groups()) if path.startswith(('/!','/~','/run/j/','/run/gz/')) or path=='/mcp': raise Failure('not_found') if request.method=='OPTIONS': From 08852c080dc322aa97da973c591f4b75e2b52213 Mon Sep 17 00:00:00 2001 From: LIghtJUNction Date: Tue, 29 Sep 2026 18:07:26 +0800 Subject: [PATCH 020/136] ci: remove completed Git boundary assembly tooling --- .github/tools/assemble_git_boundaries.py | 298 ------------------ .github/workflows/assemble-git-boundaries.yml | 71 ----- 2 files changed, 369 deletions(-) delete mode 100644 .github/tools/assemble_git_boundaries.py delete mode 100644 .github/workflows/assemble-git-boundaries.yml diff --git a/.github/tools/assemble_git_boundaries.py b/.github/tools/assemble_git_boundaries.py deleted file mode 100644 index 49fdba4f..00000000 --- a/.github/tools/assemble_git_boundaries.py +++ /dev/null @@ -1,298 +0,0 @@ -"""Exact-source Git boundary extraction; imports no project module.""" -from pathlib import Path -import ast -import copy -import difflib -import hashlib -import json -import re -import subprocess -import textwrap - -ROOT=Path.cwd() -FILES={ - 'repo':'src/msg/extensions/repositories.py', - 'ssh':'src/msg/extensions/ssh_git.py', - 'routes':'src/msg/transports/http_routes.py', -} -EXPECTED={ - 'repo':'940a1a1bd987a2d97273734a8c0646097abf3c83', - 'ssh':'d6465ce37fd7e4743773942144e91e10a7c847af', - 'routes':'04a7de0431f1211893111e887ef87661ef877c0f', -} -HTTP_METHODS=('_lfs_packet','_lfs_authorize','_lfs_preflight_publish','http_lfs', - '_repo_read_path','http','http_push','_http_push_receive','_job') -PUBLICATION=('ReferenceGuard','hook_program','relay_bounded_stdin','guarded_command') -HTTP_HELPERS=('body_bytes','json_response','error_status') -DELEGATED={'path','lfs','root','env'} -before={path:(ROOT/path).read_text() for path in FILES.values()} -after={} -audit=[] -for name,path in FILES.items(): - actual=subprocess.check_output(['git','hash-object',path],text=True).strip() - assert actual==EXPECTED[name],(path,actual) - - -def definitions(text): - return {node.name:node for node in ast.parse(text).body - if isinstance(node,(ast.ClassDef,ast.FunctionDef,ast.AsyncFunctionDef))} - - -def assignments(text): - return {target.id:node for node in ast.parse(text).body if isinstance(node,ast.Assign) - for target in node.targets if isinstance(target,ast.Name)} - - -def methods(node): - return {item.name:item for item in node.body - if isinstance(item,(ast.FunctionDef,ast.AsyncFunctionDef))} - - -def source(text,node): - # Complete physical lines keep method indentation and decorators intact. - begin=min([node.lineno]+[d.lineno for d in getattr(node,'decorator_list',[])])-1 - return ''.join(text.splitlines(True)[begin:node.end_lineno]).rstrip() - - -def delete_nodes(text,nodes): - lines=text.splitlines(True) - for node in sorted(nodes,key=lambda n:n.lineno,reverse=True): - begin=min([node.lineno]+[d.lineno for d in getattr(node,'decorator_list',[])])-1 - del lines[begin:node.end_lineno] - return ''.join(lines) - - -def once(text,old,new): - assert text.count(old)==1,old - return text.replace(old,new) - - -def clean(text): - return re.sub(r'\n{4,}','\n\n\n',text).rstrip()+'\n' - - -repo=before[FILES['repo']] -repo_nodes=definitions(repo) -store=repo_nodes['NativeGitStore'] -old_methods=methods(store) -slots=assignments(repo) -# Remove HTTP methods and stream decoding from the storage/registration module. -new_repo=delete_nodes(repo,[old_methods[name] for name in HTTP_METHODS]+[ - repo_nodes['spool_git_pack'],repo_nodes['spool_lfs_object'], - slots['_GIT_UPLOAD_SLOTS'],slots['_LFS_UPLOAD_SLOTS']]) -new_repo=once(new_repo,'from starlette.requests import ClientDisconnect\n','') -new_repo=once(new_repo,'from starlette.responses import Response,StreamingResponse\n','') -new_repo=once(new_repo,' self.app=app\n',' self.blob_root=app.settings.server.blob_dir\n') -assert new_repo.count('self.app.settings.server.blob_dir')==2 -new_repo=new_repo.replace('self.app.settings.server.blob_dir','self.blob_root') -assert 'self.app' not in source(new_repo,definitions(new_repo)['NativeGitStore']) -after[FILES['repo']]=clean(new_repo) - -# Common HTTP primitives do not depend on the router or Git adapter. -routes=before[FILES['routes']] -route_defs=definitions(routes) -base_headers=assignments(routes)['BASE_HEADERS'] -common='''"""Small HTTP response/body primitives shared without importing the router.""" -from __future__ import annotations -from starlette.responses import Response -from msg.core.codec import canonical -from msg.core.errors import require -from msg.transports.packet import gunzip - -'''+source(routes,base_headers)+'\n\n'+ '\n\n'.join(source(routes,route_defs[name]) for name in HTTP_HELPERS)+'\n' -common_path='src/msg/transports/http_common.py' -assert not (ROOT/common_path).exists() -before[common_path]='' -after[common_path]=clean(common) - -# The HTTP adapter composes the store; no catch-all attribute forwarding or -# inherited storage class keeps a hidden Request/Response dependency. -http='''"""Native Git/LFS HTTP adaptation over the shared store and publication guard.""" -from __future__ import annotations -import asyncio -import base64 -from datetime import timedelta -import hashlib -import os -from pathlib import Path -import re -import tempfile -import time -from uuid import uuid4 - -from starlette.requests import ClientDisconnect -from starlette.responses import Response, StreamingResponse -from msg.core.codec import canonical, wire, unb64 -from msg.core.errors import Failure, require -from msg.core.models import ExecutionContext -from msg.core.requests import request_for -from msg.plugins.common import check_access, resolve -from msg.extensions.repositories import ( - NativeGitStore, MAX_GIT_PACK_BYTES, MAX_GIT_UPLOAD_SECONDS, MAX_LFS_OBJECT_BYTES, - require_git_repository_capacity, _HTTP_RECEIVE, _LFS_STAGED, -) -from msg.extensions.git_publication import guarded_command -from msg.transports.http_common import body_bytes, json_response, error_status -from msg.transports.packet import path_packet - -''' -http+=source(repo,slots['_GIT_UPLOAD_SLOTS'])+'\n'+source(repo,slots['_LFS_UPLOAD_SLOTS'])+'\n\n' -http+='\n\n'.join(source(repo,repo_nodes[name]) for name in ('spool_git_pack','spool_lfs_object'))+'\n\n' -http+='''class GitHTTPAdapter: - def __init__(self, app, *, store: NativeGitStore | None = None): - self.app = app - self.store = store if store is not None else NativeGitStore(app) - -''' -for name in HTTP_METHODS: - body=source(repo,old_methods[name]) - for line in ( - ' from msg.transports.http import body_bytes,json_response\n', - ' from msg.transports.http import body_bytes\n', - ' from msg.transports.http import json_response,error_status\n', - ' from msg.transports.packet import path_packet\n', - ' from msg.extensions.ssh_git import guarded_command\n', - ): - body=body.replace(line,'') - body=re.sub(r'\bself\.(path|lfs|root|env)\b',r'self.store.\1',body) - http+=body+'\n\n' -http_path='src/msg/transports/git_http.py' -assert not (ROOT/http_path).exists() -before[http_path]='' -after[http_path]=clean(http) - -# The publication module is the old trusted guard implementation verbatim, -# minus the SSH-specific entry function. SSH exports compatible object aliases. -ssh=before[FILES['ssh']] -ssh_defs=definitions(ssh) -publication=delete_nodes(ssh,[ssh_defs['receive_pack']]) -publication_path='src/msg/extensions/git_publication.py' -assert not (ROOT/publication_path).exists() -before[publication_path]='' -after[publication_path]=clean(publication) -after[FILES['ssh']]='''"""SSH receive-pack entrypoint over the shared guarded Git publication path.""" -from __future__ import annotations -from msg.core.errors import require -from msg.extensions.repositories import ( - NativeGitStore, MAX_GIT_PACK_BYTES, MAX_GIT_UPLOAD_SECONDS, - require_git_repository_capacity, -) -# Historical imports are the exact shared objects, never alternate wrappers. -from msg.extensions.git_publication import ( - ReferenceGuard, hook_program, relay_bounded_stdin, guarded_command, -) - - -'''+source(ssh,ssh_defs['receive_pack'])+'\n' - -# Routing only selects the HTTP adapter; the outer ingress/recovery boundary -# stays in transports.http and is intentionally not modified. -new_routes=delete_nodes(routes,[base_headers]+[route_defs[name] for name in HTTP_HELPERS]) -new_routes=once(new_routes,'from starlette.routing import Route\n', - 'from starlette.routing import Route\nfrom msg.transports.http_common import BASE_HEADERS, body_bytes, json_response, error_status\n') -assert new_routes.count('from msg.extensions.repositories import NativeGitStore')==3 -assert new_routes.count('NativeGitStore(service)')==4 -new_routes=new_routes.replace('from msg.extensions.repositories import NativeGitStore', - 'from msg.transports.git_http import GitHTTPAdapter') -new_routes=new_routes.replace('NativeGitStore(service)','GitHTTPAdapter(service)') -after[FILES['routes']]=clean(new_routes) - -# Audit all moved algorithm bodies, old registrations, worker publication and -# router dispatch, allowing only the explicitly enumerated dependency rewiring. -class NoImports(ast.NodeTransformer): - def visit_ImportFrom(self,node): return None - def visit_Import(self,node): return None - -class DelegateStore(ast.NodeTransformer): - def visit_Attribute(self,node): - node=self.generic_visit(node) - if isinstance(node.value,ast.Name) and node.value.id=='self' and node.attr in DELEGATED: - node.value=ast.Attribute(value=ast.Name(id='self',ctx=ast.Load()),attr='store',ctx=ast.Load()) - return node - -class RouteAdapter(ast.NodeTransformer): - def visit_Name(self,node): - if node.id=='NativeGitStore':node.id='GitHTTPAdapter' - return node - - -def dump(node): return ast.dump(node,include_attributes=False) -def without_imports(node): return NoImports().visit(copy.deepcopy(node)) - -new_store=methods(definitions(after[FILES['repo']])['NativeGitStore']) -assert set(new_store)==set(old_methods)-set(HTTP_METHODS) -for name,node in new_store.items(): - old=source(repo,old_methods[name]) - if name=='__init__':old=old.replace('self.app=app','self.blob_root=app.settings.server.blob_dir') - old=old.replace('self.app.settings.server.blob_dir','self.blob_root') - assert dump(ast.parse(textwrap.dedent(old)).body[0])==dump(node),name - audit.append('store.'+name+' algorithm unchanged except finite settings snapshot') -for name in ('register','execute_push','git_repository_bytes','require_git_repository_capacity'): - assert dump(repo_nodes[name])==dump(definitions(after[FILES['repo']])[name]),name - audit.append('repository.'+name+' unchanged') -http_methods=methods(definitions(after[http_path])['GitHTTPAdapter']) -for name in HTTP_METHODS: - old=DelegateStore().visit(without_imports(old_methods[name])) - new=without_imports(http_methods[name]) - assert dump(old)==dump(new),name - audit.append('http.'+name+' unchanged except explicit store delegation/imports') -for name in ('spool_git_pack','spool_lfs_object'): - assert dump(repo_nodes[name])==dump(definitions(after[http_path])[name]),name - audit.append(name+' unchanged') -for name in PUBLICATION: - assert dump(ssh_defs[name])==dump(definitions(after[publication_path])[name]),name - audit.append('publication.'+name+' unchanged') -assert dump(ssh_defs['receive_pack'])==dump(definitions(after[FILES['ssh']])['receive_pack']) -audit.append('ssh.receive_pack unchanged') -for name in HTTP_HELPERS: - assert dump(route_defs[name])==dump(definitions(after[common_path])[name]),name - audit.append('http_common.'+name+' unchanged') -assert dump(base_headers)==dump(assignments(after[common_path])['BASE_HEADERS']) -for name,node in route_defs.items(): - if name in HTTP_HELPERS:continue - expected=RouteAdapter().visit(without_imports(node)) - actual=without_imports(definitions(after[FILES['routes']])[name]) - assert dump(expected)==dump(actual),name - audit.append('http_routes.'+name+' unchanged except adapter selection') -for path in (FILES['repo'],publication_path): - imports=[node.module or '' for node in ast.walk(ast.parse(after[path])) if isinstance(node,ast.ImportFrom)] - assert not [name for name in imports if name.startswith(('starlette','msg.transports.http','msg.transports.git_http','msg.extensions.ssh_git'))],path -for path in (http_path,common_path): - assert not [node.module for node in ast.walk(ast.parse(after[path])) if isinstance(node,ast.ImportFrom) - and node.module in {'msg.transports.http','msg.transports.http_routes'}],path - -notes='docs/GIT_BOUNDARY_PROGRESS.md' -before[notes]=(ROOT/notes).read_text() -after[notes]=before[notes]+''' - -## Implemented extraction and source audit - -NativeGitStore no longer owns any HTTP handler, Request/Response conversion, -body spooler or Application property. It snapshots repository/blob paths and -keeps its original physical store algorithms. GitHTTPAdapter explicitly composes -that store and retains existing HTTP/LFS behavior. ContextVar identities used by -registered operations remain singular; upload semaphores move with HTTP spooling. - -The original ReferenceGuard, hook program, bounded stdin relay and guarded Git -command are unchanged in git_publication. SSH exposes same-object aliases and its -original receive_pack entry. HTTP calls the same shared guard. The bundle worker -algorithm, including current_attempt and transaction fencing, is untouched. - -HTTP helpers/base headers have one owner in http_common; routes and Git HTTP use -it. transports.http and its outer passive/recovery boundary are unchanged. The -assembly audits every moved/retained method, every existing registration and -bundle worker body, all shared publication functions, helper functions/headers, -and route functions. Only settings snapshots, explicit store attribute delegation -and import/adapter selection are permitted differences. Source-only AST evidence -is not a successful project test. Final source and patch are uploaded; one-shot -tooling is removed from the submitted tree before focused/full validation. -''' - -out=Path('/tmp/git-boundary-assembly');out.mkdir(parents=True,exist_ok=True) -changed={path:clean(text) for path,text in after.items() if text!=before.get(path,'')} -patch=''.join(''.join(difflib.unified_diff(before.get(path,'').splitlines(True),text.splitlines(True), - fromfile='a/'+path if before.get(path) else '/dev/null',tofile='b/'+path)) for path,text in sorted(changed.items())) -for path,text in changed.items():(ROOT/path).write_text(text) -(out/'changes.patch').write_text(patch) -(out/'audit.json').write_text(json.dumps({'baseline_blobs':EXPECTED,'checked_invariants':audit, - 'changed_paths':sorted(changed),'patch_sha256':hashlib.sha256(patch.encode()).hexdigest()},indent=2)+'\n') diff --git a/.github/workflows/assemble-git-boundaries.yml b/.github/workflows/assemble-git-boundaries.yml deleted file mode 100644 index 7e8af931..00000000 --- a/.github/workflows/assemble-git-boundaries.yml +++ /dev/null @@ -1,71 +0,0 @@ -name: Assemble audited Git boundary extraction -on: - push: - branches: [build/git-boundaries-170] - paths: - - .github/tools/assemble_git_boundaries.py - - .github/workflows/assemble-git-boundaries.yml -permissions: - contents: write - actions: write -jobs: - assemble: - if: github.repository == 'TokenNotIncluded/msg.lmm.best' && github.ref == 'refs/heads/build/git-boundaries-170' - runs-on: ubuntu-latest - timeout-minutes: 5 - steps: - - uses: actions/checkout@v7 - with: - fetch-depth: 0 - - name: Avoid duplicate full validation of this source-only helper head - env: - GH_TOKEN: ${{ github.token }} - REPO: ${{ github.repository }} - INPUT_SHA: ${{ github.sha }} - run: | - gh api -X GET "repos/$REPO/actions/runs" -f head_sha="$INPUT_SHA" -f per_page=100 --jq '.workflow_runs[] | select(.path == ".github/workflows/ci.yml" and .event == "push" and .head_branch == "build/git-boundaries-170" and (.status == "queued" or .status == "in_progress")) | .id' | - while read -r run; do - gh api -X POST "repos/$REPO/actions/runs/$run/cancel" || true - done - - name: Require exact regression-first source - env: - INPUT_SHA: ${{ github.sha }} - run: | - test "$(git rev-parse HEAD)" = "$INPUT_SHA" - git merge-base --is-ancestor ee26deca16978ac2e8e4b42095dc17cfe10c175f HEAD - git diff --exit-code ee26deca16978ac2e8e4b42095dc17cfe10c175f -- src tests docs - - name: Assemble and audit source without executing the project - run: python .github/tools/assemble_git_boundaries.py - - name: Commit only audited source and notes - run: | - git add src/msg/extensions/repositories.py src/msg/extensions/ssh_git.py src/msg/extensions/git_publication.py src/msg/transports/git_http.py src/msg/transports/http_common.py src/msg/transports/http_routes.py docs/GIT_BOUNDARY_PROGRESS.md - git diff --cached --check - python - <<'PY' - import json,subprocess - from pathlib import Path - expected=set(json.loads(Path('/tmp/git-boundary-assembly/audit.json').read_text())['changed_paths']) - actual=set(subprocess.check_output(['git','diff','--cached','--name-only'],text=True).splitlines()) - assert actual==expected,(actual-expected,expected-actual) - PY - git -c user.name='github-actions[bot]' -c user.email='41898282+github-actions[bot]@users.noreply.github.com' commit -m 'refactor(git): separate HTTP adaptation and share guarded publication' - - name: Publish only the build branch without force - env: - INPUT_SHA: ${{ github.sha }} - run: | - test "$(git ls-remote origin refs/heads/build/git-boundaries-170 | cut -f1)" = "$INPUT_SHA" - git push origin HEAD:refs/heads/build/git-boundaries-170 - python - <<'PY' - import json,os,subprocess - from pathlib import Path - git=lambda *args:subprocess.check_output(['git',*args],text=True).strip() - Path('/tmp/git-boundary-assembly/source.json').write_text(json.dumps({ - 'input':os.environ['INPUT_SHA'],'commit':git('rev-parse','HEAD'), - 'tree':git('rev-parse','HEAD^{tree}'),'dirty':bool(git('status','--porcelain'))},indent=2)+'\n') - PY - git archive --format=tar.gz -o /tmp/git-boundary-assembly/source.tar.gz HEAD - - uses: actions/upload-artifact@v4 - if: always() - with: - name: git-boundary-assembly - path: /tmp/git-boundary-assembly/ - if-no-files-found: error From bc7c2123312bd6270f85e4958909ddcbc45691a3 Mon Sep 17 00:00:00 2001 From: LIghtJUNction Date: Tue, 29 Sep 2026 18:11:27 +0800 Subject: [PATCH 021/136] fix: allow achievement restart after round expiry --- src/msg/plugins/achievements.py | 5 ++++- tests/test_achievements.py | 26 ++++++++++++++++++++++++++ 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/src/msg/plugins/achievements.py b/src/msg/plugins/achievements.py index 2572d41b..ea99b06e 100644 --- a/src/msg/plugins/achievements.py +++ b/src/msg/plugins/achievements.py @@ -214,7 +214,10 @@ async def start(ctx, request, tx): # expired attempts may be replaced; a live challenge is not superseded. for (raw,) in tx.execute("SELECT body FROM achievement_ceremonies WHERE subject=? AND state='active'", (subject,)): - require(ctx.now >= parse_time(loads(raw)['expires_at']), 'achievement_ceremony_active') + previous = loads(raw) + require(ctx.now >= min(parse_time(previous['expires_at']), + parse_time(previous['round_expires_at'])), + 'achievement_ceremony_active') tx.execute('DELETE FROM achievement_ceremonies WHERE subject=?', (subject,), write=True) state = {'id': new_id('achc'), 'subject_id': subject, 'achievement_id': I_AM_NOT_HUMAN.id, 'spec_version': I_AM_NOT_HUMAN.version, 'status': 'active', 'round': 1, diff --git a/tests/test_achievements.py b/tests/test_achievements.py index d7dd183e..8a2a8ca5 100644 --- a/tests/test_achievements.py +++ b/tests/test_achievements.py @@ -134,3 +134,29 @@ async def test_total_ttl_and_replayed_nonce(installed): 'statement': FINAL_STATEMENT, }, key=key, subject=subject) assert expired.data['reason'] == 'ceremony_expired' + + +@pytest.mark.asyncio +async def test_expired_round_can_restart_without_submitting_old_answer(installed): + app, _ = installed + key, subject, _ = await register(app, 'achievement-round-restart') + started = await call(app, 'achievement.start', {}, key=key, subject=subject) + assert started.status == 'ok', wire(started) + app.executor.clock = lambda: NOW + timedelta(seconds=59) + live = await call(app, 'achievement.start', {}, key=key, subject=subject) + assert live.status == 'error' and live.error.code == 'achievement_ceremony_active' + + # At the inclusive round deadline no answer can succeed. Restart must not + # require a doomed answer or waiting for the longer ceremony deadline. + app.executor.clock = lambda: NOW + timedelta(seconds=60) + restarted = await call(app, 'achievement.start', {}, key=key, subject=subject) + assert restarted.status == 'ok', wire(restarted) + assert restarted.data['round'] == 1 + assert restarted.data['challenge_id'] != started.data['challenge_id'] + assert restarted.data['nonce'] != started.data['nonce'] + stale = await advance(app, key, subject, started.data) + assert stale.status == 'error' and stale.error.code == 'achievement_ceremony_not_found' + advanced = await advance(app, key, subject, restarted.data) + assert advanced.status == 'ok' and advanced.data['round'] == 2 + async with app.metadata.transaction(write=False) as tx: + assert tx.one('SELECT COUNT(*) FROM achievement_grants WHERE subject=?', (subject,))[0] == 0 From 91c347afdc1aa0e355158da598064dcd1aa4f74a Mon Sep 17 00:00:00 2001 From: LIghtJUNction Date: Tue, 29 Sep 2026 18:13:23 +0800 Subject: [PATCH 022/136] fix: expose console backup retirement with checked file boundaries --- .github/workflows/issue-recovery.yml | 1 + docs/CUSTODIAL_HISTORY.md | 10 ++- src/msg/admin/backup_retirement.py | 37 ++++++++++- src/msg/admin/root.py | 14 ++-- src/msg/daemon.py | 8 +++ tests/test_backup_retirement_cli.py | 99 ++++++++++++++++++++++++++++ 6 files changed, 158 insertions(+), 11 deletions(-) create mode 100644 tests/test_backup_retirement_cli.py diff --git a/.github/workflows/issue-recovery.yml b/.github/workflows/issue-recovery.yml index 262ce8cb..9b0b0f83 100644 --- a/.github/workflows/issue-recovery.yml +++ b/.github/workflows/issue-recovery.yml @@ -72,6 +72,7 @@ jobs: run: >- python -m pytest tests/test_recovery_current_authority.py tests/test_task_a_recovery_checkpoint.py tests/test_backup_retirement_attestation.py + tests/test_backup_retirement_cli.py --junitxml=artifacts/recovery-current.xml - uses: actions/upload-artifact@v4 if: always() diff --git a/docs/CUSTODIAL_HISTORY.md b/docs/CUSTODIAL_HISTORY.md index c051f56c..6a39d4c4 100644 --- a/docs/CUSTODIAL_HISTORY.md +++ b/docs/CUSTODIAL_HISTORY.md @@ -92,8 +92,8 @@ application without the isolated selftest namespace. `history_recoverable` covers **enumerated and client-verified revisions only**. `online_key_retired` describes online vault material only. `backup_retired` stays false unless a root-signed backup retirement record was imported at the physical -console (`RootAdmin.sign_backup_retirement` / `RootAdmin.import_backup_retirement`, -both behind `require_local_console`). The record binds the subject, both old key +console (`msgd root backup-retirement sign SOURCE DESTINATION` followed by +`msgd root backup-retirement import RECORD`, both behind `require_local_console`). The record binds the subject, both old key IDs, a non-empty list of named backup sets and a validity window of at most 400 days; it is signed with purpose `custodial-backup-retirement-v1`. Every read re-verifies it against the active, unrevoked root credential, so an altered, @@ -106,7 +106,11 @@ does not prove that no other copy exists and it does not claim that production backups were destroyed. No network operation writes, accepts or forwards it; a `backup_retirement` value inside the network-writable migration body is ignored. Neither a boolean supplied by a network caller nor a green test run can establish -that old backups lost a key. There is no `msg-admin` CLI wiring yet. Restore/ +that old backups lost a key. Input JSON must be a single-link regular file owned +by the local operator, mode 0600, at most 1 MiB; symlinks and special files are +rejected. Signing publishes a complete 0600 record without replacing an existing +destination. Both commands require a digest-bound interactive confirmation; +signing additionally unlocks Root with a non-echoed PIN prompt. Restore/ revocation replay and real operator acceptance are tracked separately in issues #69 and #70. diff --git a/src/msg/admin/backup_retirement.py b/src/msg/admin/backup_retirement.py index ef0ca393..f91e764c 100644 --- a/src/msg/admin/backup_retirement.py +++ b/src/msg/admin/backup_retirement.py @@ -8,13 +8,46 @@ from __future__ import annotations from msg.constants import ROOT_SUBJECT -from msg.core.codec import digest, loads -from msg.core.errors import require +import os +from pathlib import Path +import tempfile + +from msg.core.codec import canonical, digest, loads +from msg.core.errors import Failure, require from msg.core.models import AuditEvent, Event, ResourceRef from msg.plugins.common import new_id from msg.security.backup_retirement import FIELDS, DOMAIN, check, root_verifier, setting_key +def read_statement(path): + from msg.security.root_files import read_private + return loads(read_private(path, limit=1024 * 1024)) + + +def write_record(path, record): + """Publish a complete 0600 record without replacing an existing destination.""" + path = Path(path) + fd, temporary = tempfile.mkstemp(prefix='.retirement-', dir=path.parent) + try: + with os.fdopen(fd, 'wb') as stream: + stream.write(canonical(record)) + stream.flush() + os.fsync(stream.fileno()) + try: + os.link(temporary, path) + except FileExistsError as exc: + raise Failure('backup_retirement_destination_exists') from exc + os.unlink(temporary) + directory = os.open(path.parent, os.O_RDONLY | os.O_DIRECTORY) + try: + os.fsync(directory) + finally: + os.close(directory) + finally: + if os.path.exists(temporary): + os.unlink(temporary) + + def unsigned_statement(value): require(isinstance(value, dict) and set(value) == FIELDS and value.get('domain') == DOMAIN, 'backup_retirement_invalid') diff --git a/src/msg/admin/root.py b/src/msg/admin/root.py index c77107fa..8c300977 100644 --- a/src/msg/admin/root.py +++ b/src/msg/admin/root.py @@ -265,26 +265,28 @@ async def read(): def sign_backup_retirement(self,source,destination): """Root-sign an operator's statement that listed backup sets lost one old key.""" require_local_console(self.config_dir) - from msg.admin.backup_retirement import unsigned_statement + from msg.admin.backup_retirement import read_statement, unsigned_statement, write_record from msg.security.backup_retirement import sign_record target=Path(destination) require(not target.exists() and not target.is_symlink(),'backup_retirement_destination_exists') - body=unsigned_statement(loads(Path(source).read_bytes())) + body=unsigned_statement(read_statement(source)) print(canonical({'statement':body,'digest':digest(body), 'claim':'listed_backup_sets_only'}).decode()) require(input('Type ATTEST BACKUP RETIREMENT '+digest(body)+': ')== 'ATTEST BACKUP RETIREMENT '+digest(body),'approval_cancelled') - private=open_private_key(loads(root_envelope(self.config_dir).read_bytes()),getpass.getpass('Root PIN/passphrase: ')) + from msg.security.root_files import read_private + envelope=loads(read_private(root_envelope(self.config_dir))) + private=open_private_key(envelope,getpass.getpass('Root PIN/passphrase: ')) record=sign_record(body,Ed25519Signer.from_bytes(private)) - durable_write(target,canonical(record),mode=0o600) + write_record(target,record) return {'status':'backup_retirement_signed','path':str(target),'digest':digest(record)} def import_backup_retirement(self,source): """Only this console path persists a record; reads re-verify it every time.""" operator=require_local_console(self.config_dir) - from msg.admin.backup_retirement import import_record + from msg.admin.backup_retirement import import_record, read_statement app=self._app() - record=loads(Path(source).read_bytes()) + record=read_statement(source) print(canonical({'record':record,'digest':digest(record)}).decode()) require(input('Type IMPORT BACKUP RETIREMENT '+digest(record)+': ')== 'IMPORT BACKUP RETIREMENT '+digest(record),'approval_cancelled') diff --git a/src/msg/daemon.py b/src/msg/daemon.py index 637715da..449b1572 100644 --- a/src/msg/daemon.py +++ b/src/msg/daemon.py @@ -98,6 +98,11 @@ def parser(): rotate=rs.add_parser('rotate');rotate.add_argument('--lost-key',action='store_true');rotate.add_argument('--resume',action='store_true') rb=rs.add_parser('backup');rb.add_argument('destination',type=Path) rr=rs.add_parser('recover');rr.add_argument('source',type=Path) + retirement=rs.add_parser('backup-retirement',help='Physical-console attestation for listed backup sets') + retirement_sub=retirement.add_subparsers(dest='retirement_command',required=True) + retirement_sign=retirement_sub.add_parser('sign') + retirement_sign.add_argument('source',type=Path);retirement_sign.add_argument('destination',type=Path) + retirement_sub.add_parser('import').add_argument('source',type=Path) money=sub.add_parser('money',help='Physical-console central bank administration') ms=money.add_subparsers(dest='money_command',required=True) ms.add_parser('mint').add_argument('amount') @@ -168,6 +173,9 @@ def main(argv=None): admin.change_pin();result={'status':'pin_changed'} elif args.root_command=='rotate':result=admin.rotate(lost_key=args.lost_key,resume=args.resume) elif args.root_command=='backup':result=admin.backup(args.destination) + elif args.root_command=='backup-retirement': + result=(admin.sign_backup_retirement(args.source,args.destination) + if args.retirement_command=='sign' else admin.import_backup_retirement(args.source)) else:result=admin.recover(args.source) emit(result);return 0 if args.command=='market': diff --git a/tests/test_backup_retirement_cli.py b/tests/test_backup_retirement_cli.py new file mode 100644 index 00000000..25a5014a --- /dev/null +++ b/tests/test_backup_retirement_cli.py @@ -0,0 +1,99 @@ +"""Installed administrative entry and real descriptor boundaries, not console success.""" +import json +import os +from pathlib import Path +import subprocess +import sys + +import pytest + +from msg.core.errors import Failure +from msg.daemon import parser + + +@pytest.mark.parametrize('action', ['sign', 'import']) +def test_retirement_command_parser(action, tmp_path): + command = ['root', 'backup-retirement', action, str(tmp_path/'statement.json')] + if action == 'sign': + command.append(str(tmp_path/'record.json')) + args = parser().parse_args(command) + assert args.retirement_command == action + assert args.source == tmp_path/'statement.json' + + +@pytest.mark.parametrize('action', ['sign', 'import']) +def test_retirement_cli_refuses_nonconsole_before_reading_files(action, tmp_path): + command = [str(Path(sys.executable).with_name('msgd')), + '--config-dir', str(tmp_path), 'root', 'backup-retirement', action, + str(tmp_path/'missing.json')] + if action == 'sign': + command.append(str(tmp_path/'output.json')) + env = dict(os.environ, PYTHONPATH=str(Path(__file__).resolve().parents[1]/'src')) + for key in ('SSH_CONNECTION', 'SSH_CLIENT', 'SSH_TTY'): + env.pop(key, None) + result = subprocess.run(command, env=env, capture_output=True, text=True) + assert result.returncode == 1 + expected = 'local_os_administrator_required' if os.geteuid() else 'local_console_required' + assert json.loads(result.stderr) == {'status': 'error', 'error': {'code': expected}} + assert result.stdout == '' + assert not (tmp_path/'output.json').exists() + + +@pytest.mark.parametrize('kind', ['symlink', 'hardlink', 'permissions', 'fifo', 'oversize']) +def test_statement_file_boundary(kind, tmp_path): + from msg.admin.backup_retirement import read_statement + path = tmp_path/'statement.json' + path.write_bytes(b'{}') + path.chmod(0o600) + if kind == 'symlink': + target = tmp_path/'target' + path.rename(target) + path.symlink_to(target) + elif kind == 'hardlink': + os.link(path, tmp_path/'other') + elif kind == 'permissions': + path.chmod(0o644) + elif kind == 'fifo': + path.unlink() + os.mkfifo(path, 0o600) + else: + path.write_bytes(b' ' * (1024 * 1024 + 1)) + with pytest.raises(Failure, match='unsafe_root_private_file'): + read_statement(path) + + +def test_statement_file_valid(tmp_path): + from msg.admin.backup_retirement import read_statement + path = tmp_path/'statement.json' + path.write_bytes(b'{"statement": {}}') + path.chmod(0o600) + assert read_statement(path) == {'statement': {}} + + +@pytest.mark.parametrize('existing', [False, True]) +def test_record_output_never_overwrites(existing, tmp_path): + from msg.admin.backup_retirement import write_record + target = tmp_path/'record.json' + if existing: + target.write_bytes(b'preserve') + with pytest.raises(Failure, match='backup_retirement_destination_exists'): + write_record(target, {'statement': {}}) + assert target.read_bytes() == b'preserve' + else: + write_record(target, {'statement': {}}) + assert json.loads(target.read_bytes()) == {'statement': {}} + assert target.stat().st_mode & 0o777 == 0o600 + assert not list(tmp_path.glob('.retirement-*')) + + +def test_record_output_refuses_symlink(tmp_path): + from msg.admin.backup_retirement import write_record + victim = tmp_path/'victim.json' + victim.write_bytes(b'preserve') + target = tmp_path/'record.json' + target.symlink_to(victim) + with pytest.raises(Failure, match='backup_retirement_destination_exists'): + write_record(target, {'statement': {}}) + assert target.is_symlink() + assert victim.read_bytes() == b'preserve' + assert not list(tmp_path.glob('.retirement-*')) From 00b17457ce8c22319db3fa10289a0990cfaac531 Mon Sep 17 00:00:00 2001 From: LIghtJUNction Date: Tue, 29 Sep 2026 18:15:05 +0800 Subject: [PATCH 023/136] fix(tui): render frozen client search snippets safely --- src/msg/tui.py | 2 +- tests/test_tui.py | 3 +++ tests/test_tui_transport_boundaries.py | 8 ++++++++ 3 files changed, 12 insertions(+), 1 deletion(-) diff --git a/src/msg/tui.py b/src/msg/tui.py index 23605b41..df408870 100644 --- a/src/msg/tui.py +++ b/src/msg/tui.py @@ -166,7 +166,7 @@ def _show_items(self, items): name = item.get('title') or item.get('name') or item.get('path') or item.get('source') or rid self._write(f'{index}. {name} {rid}') snippet = item.get('snippet') - if isinstance(snippet, dict) and snippet.get('text'): + if isinstance(snippet, Mapping) and isinstance(snippet.get('text'), str) and snippet['text']: self._write(' ' + snippet['text']) async def _show_page(self, operation, arguments, title): diff --git a/tests/test_tui.py b/tests/test_tui.py index 708f923b..13288295 100644 --- a/tests/test_tui.py +++ b/tests/test_tui.py @@ -86,7 +86,10 @@ async def test_tui_real_pg_read_and_revocation_no_business_effect(installed, tmp ('resources', 'revisions', 'messages', 'events', 'reactions')) await ui.home() await ui.read(rid) + output.seek(0) + output.truncate(0) await ui.search('/main', 'TUI public') + assert 'TUI public text' in output.getvalue() await ui.thread(rid) async with app.metadata.transaction(write=False) as tx: after = tuple(tx.one(f'SELECT COUNT(*) FROM {table}')[0] for table in diff --git a/tests/test_tui_transport_boundaries.py b/tests/test_tui_transport_boundaries.py index bf1471cb..f359ae94 100644 --- a/tests/test_tui_transport_boundaries.py +++ b/tests/test_tui_transport_boundaries.py @@ -179,3 +179,11 @@ async def switch_during_read(operation, arguments): assert ui._last_read is None assert 'NEVER DISPLAY' not in ui.stdout.getvalue() assert len(client.calls) == 1 + + +@pytest.mark.parametrize('text', [7, {'nested': 'text'}, ['text'], None]) +def test_malformed_snippet_text_does_not_crash_or_display(text): + output = StringIO() + ui = TerminalUI(Client([]), stdout=output) + ui._show_items([{'id': 'r_item', 'snippet': {'text': text}}]) + assert output.getvalue().strip() == '1. r_item r_item' From edee1e2c4ed823ef4fe3ad596b3209b22c79147d Mon Sep 17 00:00:00 2001 From: LIghtJUNction Date: Tue, 29 Sep 2026 18:15:05 +0800 Subject: [PATCH 024/136] test(mail): exercise isolated TLS SMTP socket outcomes --- tests/test_smtp_socket_acceptance.py | 125 +++++++++++++++++++++++++++ 1 file changed, 125 insertions(+) create mode 100644 tests/test_smtp_socket_acceptance.py diff --git a/tests/test_smtp_socket_acceptance.py b/tests/test_smtp_socket_acceptance.py new file mode 100644 index 00000000..4c1a2e47 --- /dev/null +++ b/tests/test_smtp_socket_acceptance.py @@ -0,0 +1,125 @@ +"""Exercise SMTP over a real, certificate-verified loopback TLS socket.""" +import socket +import ssl +import subprocess +from contextlib import contextmanager +from email import policy +from email.parser import BytesParser +from threading import Thread +from types import SimpleNamespace + +import pytest + +from msg.core.errors import Failure +from msg.workers.mail import SmtpSender + + +@pytest.fixture +def tls_contexts(tmp_path): + cert, key = tmp_path / 'cert.pem', tmp_path / 'key.pem' + subprocess.run(['openssl', 'req', '-x509', '-newkey', 'rsa:2048', '-nodes', + '-keyout', str(key), '-out', str(cert), '-days', '1', + '-subj', '/CN=localhost', '-addext', 'subjectAltName=IP:127.0.0.1'], + check=True, capture_output=True) + server = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + server.load_cert_chain(cert, key) + client = ssl.create_default_context(cafile=str(cert)) + return server, client + + +@contextmanager +def smtp_sink(context, outcome): + commands, messages, errors = [], [], [] + with socket.socket() as listener: + listener.bind(('127.0.0.1', 0)) + listener.listen(1) + listener.settimeout(5) + + def serve(): + try: + conn, _ = listener.accept() + with context.wrap_socket(conn, server_side=True) as conn: + conn.settimeout(5) + with conn.makefile('rwb') as wire: + def reply(value): + wire.write(value + b'\r\n') + wire.flush() + reply(b'220 localhost isolated test sink') + while line := wire.readline(): + commands.append(line.strip()) + verb = line.split(b' ', 1)[0].strip().upper() + if verb in (b'EHLO', b'HELO'): + reply(b'250 localhost') + elif verb == b'MAIL': + reply(b'250 sender accepted') + elif verb == b'RCPT': + reply(b'550 recipient refused' if outcome == 'refused' else b'250 recipient accepted') + elif verb == b'RSET': + reply(b'250 reset') + elif verb == b'DATA': + reply(b'354 send message') + body = [] + while (line := wire.readline()) != b'.\r\n': + if not line: + raise AssertionError('EOF during DATA') + body.append(line[1:] if line.startswith(b'..') else line) + messages.append(b''.join(body)) + if outcome == 'disconnect': + return + reply(b'250 accepted') + else: + raise AssertionError(f'unexpected command {verb!r}') + except (OSError, AssertionError) as exc: + errors.append(exc) + + thread = Thread(target=serve, daemon=True) + thread.start() + try: + yield listener.getsockname()[1], commands, messages + finally: + thread.join(6) + assert not thread.is_alive(), 'SMTP sink did not finish' + assert not errors, (errors, commands) + + +def sender(port): + return SmtpSender(SimpleNamespace(enabled=True, host='127.0.0.1', port=port, + sender='sender@example.test', tls='tls', credential_file=None)) + + +def job(): + return SimpleNamespace(id='isolated-smtp-acceptance', arguments={ + 'recipient': 'reader@example.test', 'subject': 'minimal notice', + 'text': 'Open your inbox.\n.dot-stuffed line'}) + + +@pytest.mark.asyncio +@pytest.mark.parametrize(('outcome', 'expected'), [ + ('accepted', 'sent'), ('disconnect', 'uncertain'), ('refused', 'uncertain')]) +async def test_real_tls_smtp_success_refusal_and_lost_ack(tls_contexts, monkeypatch, outcome, expected): + server, client = tls_contexts + monkeypatch.setattr(ssl, 'create_default_context', lambda: client) + # Keep EHLO hostname discovery independent of the host machine's DNS. + monkeypatch.setattr(socket, 'getfqdn', lambda: 'sink.example.test') + with smtp_sink(server, outcome) as (port, commands, messages): + assert await sender(port).send(job()) == expected + assert sum(command.upper().startswith(b'MAIL ') for command in commands) == 1 + assert [command.lower() for command in commands if command.upper().startswith(b'RCPT ')] == [ + b'rcpt to:'] + assert len(messages) == (0 if outcome == 'refused' else 1) + if messages: + message = BytesParser(policy=policy.default).parsebytes(messages[0]) + assert message['To'] == 'reader@example.test' + assert message['Message-ID'] == '' + assert '.dot-stuffed line' in message.get_content() + + +@pytest.mark.asyncio +async def test_real_smtp_connection_failure_is_retryable_before_data(): + # Bind without listening to reserve a port which refuses TCP connections. + with socket.socket() as reserved: + reserved.bind(('127.0.0.1', 0)) + with pytest.raises(Failure) as caught: + await sender(reserved.getsockname()[1]).send(job()) + assert caught.value.code == 'mail_connection_failed' + assert caught.value.retryable From 5e3525bab5c0f820efaf30b2b4c6ad1be0cc720a Mon Sep 17 00:00:00 2001 From: LIghtJUNction Date: Tue, 29 Sep 2026 18:13:17 +0800 Subject: [PATCH 025/136] fix(registry): reject dangling resource and capability references --- src/msg/core/registry.py | 10 ++- tests/test_registry_contract_references.py | 79 ++++++++++++++++++++++ 2 files changed, 87 insertions(+), 2 deletions(-) create mode 100644 tests/test_registry_contract_references.py diff --git a/src/msg/core/registry.py b/src/msg/core/registry.py index ecb95941..fc1c578e 100644 --- a/src/msg/core/registry.py +++ b/src/msg/core/registry.py @@ -86,10 +86,16 @@ def freeze(self): require('identity' in self._plugins,'identity_plugin_required') for spec in self._operations.values(): require(spec.input_schema.id in self._schemas and spec.output_schema.id in self._schemas,'missing_schema') + operation_ids={f'{s.name}@{s.version}' for s in self._operations.values()} + for resource in self._types.values(): + require(resource.content_schema is None or + resource.content_schema.id in self._schemas,'missing_schema') + require(resource.operations<=operation_ids,'unknown_resource_operation') for cap in self._capabilities.values(): + require(cap.constraints_schema is None or + cap.constraints_schema.id in self._schemas,'missing_schema') require(cap.scope_types<=set(n for n,v in self._types),'unknown_scope_type') - require(all(op in {f'{s.name}@{s.version}' for s in self._operations.values()} - for op in cap.operations),'unknown_capability_operation') + require(cap.operations<=operation_ids,'unknown_capability_operation') self._frozen=True @property diff --git a/tests/test_registry_contract_references.py b/tests/test_registry_contract_references.py new file mode 100644 index 00000000..6a1aafb7 --- /dev/null +++ b/tests/test_registry_contract_references.py @@ -0,0 +1,79 @@ +"""Registry freeze rejects dangling references before serving any contract.""" +from dataclasses import replace + +import pytest + +from msg.core.errors import Failure +from msg.core.models import ( + CapabilitySpec, OperationSpec, PluginManifest, ResourceRef, ResourceTypeSpec, +) +from msg.core.registry import Registry + + +REF = ResourceRef(id='schema_contract_reference') + + +async def unused(*args): + raise AssertionError('Registry validation must not invoke application code') + + +def declarations(): + resource = ResourceTypeSpec(name='document', version=1, container=False, + content_schema=None, operations=frozenset(), relations=frozenset()) + capability = CapabilitySpec(name='document.read', version=1, + scope_types=frozenset({'document'}), operations=frozenset({'document.read@1'}), + replaces_checks=frozenset(), delegatable=False, ca_only=False, + constraints_schema=None) + operation = OperationSpec(name='document.read', version=1, input_schema=REF, + output_schema=REF, effect='read', entries=frozenset({'network'}), + require_signature=False, requirements=unused, handler=unused) + return resource, capability, operation + + +def registry(resource, capability, operation): + result = Registry() + result.add_schema(REF, {'type': 'object'}) + result.add(PluginManifest(name='identity', version='1', dependencies=(), + resource_types=(resource,), capabilities=(capability,), + operations=(operation,), migrations=())) + return result + + +@pytest.mark.parametrize('owner', ['resource', 'capability']) +def test_freeze_rejects_unregistered_optional_schema_and_can_retry(owner): + resource, capability, operation = declarations() + missing = ResourceRef(id='schema_missing') + if owner == 'resource': + resource = replace(resource, content_schema=missing) + else: + capability = replace(capability, constraints_schema=missing) + installed = registry(resource, capability, operation) + with pytest.raises(Failure, match='^missing_schema$'): + installed.freeze() + assert installed.frozen is False + # Forward references are permitted while assembling the complete registry. + installed.add_schema(missing, {'type': 'object'}) + installed.freeze() + assert installed.frozen is True + + +@pytest.mark.parametrize('reference', ['document.unknown@1', 'document.read@2', 'document.read']) +def test_freeze_rejects_unknown_resource_operation_including_wrong_version(reference): + resource, capability, operation = declarations() + resource = replace(resource, operations=frozenset({reference})) + installed = registry(resource, capability, operation) + with pytest.raises(Failure, match='^unknown_resource_operation$'): + installed.freeze() + assert installed.frozen is False + + +def test_freeze_accepts_registered_content_constraints_and_operation_references(): + resource, capability, operation = declarations() + resource = replace(resource, content_schema=REF, + operations=frozenset({'document.read@1'})) + capability = replace(capability, constraints_schema=REF) + installed = registry(resource, capability, operation) + installed.freeze() + assert installed.resource_type('document') == resource + assert installed.capability('document.read') == capability + assert installed.operation('document.read') == operation From 7dfa254d5f8f2e1b7de2af54c4c0d1ffc3633320 Mon Sep 17 00:00:00 2001 From: LIghtJUNction Date: Tue, 29 Sep 2026 18:15:16 +0800 Subject: [PATCH 026/136] refactor(registry): give operations ownership of rule dependencies --- src/msg/core/models.py | 3 ++ src/msg/core/registry.py | 25 +++-------- src/msg/plugins/common.py | 26 ++++++++++- tests/test_registry_operation_rules.py | 62 ++++++++++++++++++++++++++ 4 files changed, 94 insertions(+), 22 deletions(-) create mode 100644 tests/test_registry_operation_rules.py diff --git a/src/msg/core/models.py b/src/msg/core/models.py index 84e0ba9f..bab47f1c 100644 --- a/src/msg/core/models.py +++ b/src/msg/core/models.py @@ -354,6 +354,9 @@ class OperationSpec: [ExecutionContext, OperationRequest, MetadataSession], Awaitable[HandlerOutput] ] + requires_rules: tuple[str, ...] = ('msg.protocol',) + + @dataclass(frozen=True, slots=True, kw_only=True) class TransportLimits: max_request_bytes: int diff --git a/src/msg/core/registry.py b/src/msg/core/registry.py index fc1c578e..958cd2ad 100644 --- a/src/msg/core/registry.py +++ b/src/msg/core/registry.py @@ -83,8 +83,12 @@ def add(self,manifest): self._plugins[manifest.name]=manifest def freeze(self): + from msg.bootstrap import RULE_PATHS require('identity' in self._plugins,'identity_plugin_required') for spec in self._operations.values(): + require(spec.requires_rules and len(set(spec.requires_rules))==len(spec.requires_rules) + and all(rule_id in RULE_PATHS for rule_id in spec.requires_rules), + 'dangling_requires_rules') require(spec.input_schema.id in self._schemas and spec.output_schema.id in self._schemas,'missing_schema') operation_ids={f'{s.name}@{s.version}' for s in self._operations.values()} for resource in self._types.values(): @@ -143,26 +147,7 @@ def validate(self,ref,value): def describe(self,spec): from msg.bootstrap import RULE_PATHS - name=spec.name - if name.startswith('identity.'): - rules=('identity','auth') - elif name.startswith('content.topic_') or name.startswith('discussion.'): - rules=('topics','read-write') - elif name.startswith('content.'): - rules=('read-write',) - elif name.startswith(('transfer.','keystore.','git.')): - rules=('files','protocol') - elif name.startswith(('cert.','group.')): - rules=('auth',) - elif name.startswith(('system.','tool.')): - rules=('security','protocol') - elif name.startswith('discovery.'): - rules=('read-write','protocol') - else: - rules=('protocol',) - rule_ids=tuple('msg.'+rule for rule in rules) - require(len(set(rule_ids))==len(rule_ids) and all(rid in RULE_PATHS for rid in rule_ids), - 'dangling_requires_rules',name) + rule_ids=spec.requires_rules result={'name':spec.name,'version':spec.version,'effect':spec.effect, 'entries':sorted(spec.entries),'require_signature':spec.require_signature, 'input_schema':wire(spec.input_schema),'output_schema':wire(spec.output_schema), diff --git a/src/msg/plugins/common.py b/src/msg/plugins/common.py index 7cbd811e..13a0fcc3 100644 --- a/src/msg/plugins/common.py +++ b/src/msg/plugins/common.py @@ -194,6 +194,27 @@ async def topic_policy(tx,resource): return tx.setting('policy:'+topic.id,{}) +def default_operation_rules(name): + """Compatibility defaults captured when constructing an operation.""" + if name.startswith('identity.'): + rules=('identity','auth') + elif name.startswith('content.topic_') or name.startswith('discussion.'): + rules=('topics','read-write') + elif name.startswith('content.'): + rules=('read-write',) + elif name.startswith(('transfer.','keystore.','git.')): + rules=('files','protocol') + elif name.startswith(('cert.','group.')): + rules=('auth',) + elif name.startswith(('system.','tool.')): + rules=('security','protocol') + elif name.startswith('discovery.'): + rules=('read-write','protocol') + else: + rules=('protocol',) + return tuple('msg.'+rule for rule in rules) + + def registration(app,name,dependencies=()): from msg.core.models import OperationSpec,PluginManifest,ResourceRef from msg.plugins.schemas import OUTPUT @@ -201,13 +222,14 @@ def registration(app,name,dependencies=()): output_ref=ResourceRef(id='schema:operation-result') if output_ref.id not in app.registry._schemas: app.registry.add_schema(output_ref,OUTPUT) - def operation(opname,schema, *, effect='transaction',requirements=no_requirements,signature=False,version=1): + def operation(opname,schema, *, effect='transaction',requirements=no_requirements,signature=False,version=1,requires_rules=None): def decorate(handler): ref=ResourceRef(id='schema:'+opname+':'+str(version)) app.registry.add_schema(ref,schema) operations.append(OperationSpec(name=opname,version=version,input_schema=ref,output_schema=output_ref, effect=effect,entries=frozenset({'network','worker'}),require_signature=signature, - requirements=requirements,handler=handler)) + requirements=requirements,handler=handler, + requires_rules=default_operation_rules(opname) if requires_rules is None else tuple(requires_rules))) return handler return decorate def finish(resource_types=()): diff --git a/tests/test_registry_operation_rules.py b/tests/test_registry_operation_rules.py new file mode 100644 index 00000000..a4c138a7 --- /dev/null +++ b/tests/test_registry_operation_rules.py @@ -0,0 +1,62 @@ +"""Operations own rule dependencies; publication does not infer them anew.""" +from dataclasses import replace +from types import SimpleNamespace + +import pytest + +from msg.bootstrap import RULE_PATHS +from msg.core.errors import Failure +from msg.core.registry import Registry +from msg.plugins.common import registration + + +async def unused(*args): + raise AssertionError('contract assembly must not execute handlers') + + +def assemble(name, **options): + registry = Registry() + op, finish = registration(SimpleNamespace(registry=registry), 'identity') + op(name, {'type': 'object'}, **options)(unused) + finish() + return registry + + +@pytest.mark.parametrize(('name', 'ids'), [ + ('identity.register', ('msg.identity', 'msg.auth')), + ('content.topic_create', ('msg.topics', 'msg.read-write')), + ('discussion.reply', ('msg.topics', 'msg.read-write')), + ('content.post_create', ('msg.read-write',)), + ('transfer.begin', ('msg.files', 'msg.protocol')), + ('keystore.write', ('msg.files', 'msg.protocol')), + ('git.push', ('msg.files', 'msg.protocol')), + ('cert.issue', ('msg.auth',)), + ('group.join', ('msg.auth',)), + ('system.get', ('msg.security', 'msg.protocol')), + ('tool.invoke', ('msg.security', 'msg.protocol')), + ('discovery.get', ('msg.read-write', 'msg.protocol')), + ('orders.buy', ('msg.protocol',)), +]) +def test_existing_default_rule_descriptions_remain_identical(name, ids): + registry = assemble(name) + registry.freeze() + assert registry.describe(registry.operation(name))['requires_rules'] == [ + {'rule_id': rid, 'path': RULE_PATHS[rid]} for rid in ids] + + +def test_explicit_rule_dependencies_override_name_and_are_owned_by_spec(): + registry = assemble('identity.custom', requires_rules=('msg.recovery', 'msg.security')) + registry.freeze() + spec = registry.operation('identity.custom') + assert spec.requires_rules == ('msg.recovery', 'msg.security') + # Renaming an already assembled declaration must not change its rules. + assert registry.describe(replace(spec, name='content.custom'))['requires_rules'] == [ + {'rule_id': rid, 'path': RULE_PATHS[rid]} for rid in spec.requires_rules] + + +@pytest.mark.parametrize('ids', [('msg.unknown',), ('msg.auth', 'msg.auth'), ()]) +def test_freeze_rejects_missing_duplicate_or_empty_operation_rules(ids): + registry = assemble('identity.custom', requires_rules=ids) + with pytest.raises(Failure, match='^dangling_requires_rules$'): + registry.freeze() + assert not registry.frozen From 1ab30e65a1c9da24f506ce3c68e1d98a9f9822eb Mon Sep 17 00:00:00 2001 From: LIghtJUNction Date: Tue, 29 Sep 2026 18:15:40 +0800 Subject: [PATCH 027/136] fix: sign and attribute custodial achievement confirmations --- src/msg/plugins/achievements.py | 33 +++++++++--- src/msg/security/authentication.py | 8 ++- tests/test_achievements.py | 82 ++++++++++++++++++++++++++++++ 3 files changed, 116 insertions(+), 7 deletions(-) diff --git a/src/msg/plugins/achievements.py b/src/msg/plugins/achievements.py index ea99b06e..1cd33286 100644 --- a/src/msg/plugins/achievements.py +++ b/src/msg/plugins/achievements.py @@ -277,9 +277,28 @@ async def complete(ctx, request, tx): return _failure(state, tx, reason) if args['statement'] != FINAL_STATEMENT or args['ceremony_digest'] != ceremony_digest(state): return _failure(state, tx, 'final_confirmation_failed') - # AuthenticationService already verified the signed request bytes with - # the subject's active signing key. Token-only actors cannot reach here. - require(ctx.principal.method == 'signature', 'self_custody_signature_required') + signature_source = 'self-custody' + if ctx.principal.method == 'token': + require((await tx.subject(subject)).kind == 'custodial', + 'self_custody_signature_required') + from msg.security.vault import open_signer + signer = open_signer(app, tx, subject) + primary = tx.one('SELECT key_id FROM identity_keys WHERE subject=? AND is_primary=1', + (subject,)) + require(primary is not None and primary[0] == signer.key_id, + 'custodial_vault_key_mismatch') + credential = await tx.credential(signer.key_id) + require(credential.revoked_at is None and credential.not_before <= ctx.now and + (credential.expires_at is None or ctx.now < credential.expires_at), + 'credential_revoked') + state['final_confirmation'] = {'subject_id': subject, 'challenge_id': state['id'], + 'ceremony_digest': args['ceremony_digest'], 'statement': args['statement'], + 'request_id': request.request_id, 'signature_source': 'custodial'} + state['final_signature'] = wire(signer.sign(canonical(state['final_confirmation']), + purpose='achievement-confirmation')) + signature_source = 'custodial' + else: + require(ctx.principal.method == 'signature', 'self_custody_signature_required') evidence = digest({'challenge_id': state['id'], 'answers': state['answers'], 'strategy': state['strategy'], 'strategy_version': state['strategy_version'], 'ceremony_digest': args['ceremony_digest']}) @@ -289,8 +308,10 @@ async def complete(ctx, request, tx): grant = {'id': new_id('achg'), 'subject_id': subject, 'achievement_id': I_AM_NOT_HUMAN.id, 'spec_version': I_AM_NOT_HUMAN.version, 'issuer': app.receipt_signer.key_id, 'issued_at': wire(ctx.now), 'claim': I_AM_NOT_HUMAN.claim, - 'auth_method': 'signature', 'evidence_digest': evidence, 'automatic': True, + 'auth_method': ctx.principal.method, 'evidence_digest': evidence, 'automatic': True, 'revoked_at': None, 'metadata': {'protocol_passed': True}} + if signature_source == 'custodial': + grant['metadata']['signature_source'] = signature_source grant['signature'] = wire(app.receipt_signer.sign(canonical(grant), purpose='achievement-grant')) tx.execute('INSERT INTO achievement_grants (id,subject,achievement_id,spec_version,body) VALUES (?,?,?,?,?)', (grant['id'], subject, grant['achievement_id'], grant['spec_version'], @@ -301,7 +322,7 @@ async def complete(ctx, request, tx): data={'achievement_id': grant['achievement_id'], 'challenge_id': state['id'], 'strategy': state['strategy'], 'strategy_version': state['strategy_version'], 'round_digests': [digest(item) for item in state['answers']], - 'auth_method': 'signature', 'signature_source': 'self-custody', + 'auth_method': ctx.principal.method, 'signature_source': signature_source, 'evidence_digest': evidence, 'automatic': True, 'certificate_id': grant['id']}) await tx.append_audit(AuditEvent(event=audit, authority=(ResourceRef(id=subject),), @@ -314,6 +335,6 @@ async def complete(ctx, request, tx): state.pop('expected_digest', None) state.pop('nonce_digest', None) _save(tx, state) - return HandlerOutput(data={'grant': public_grant(grant), 'signature_source': 'self-custody'}) + return HandlerOutput(data={'grant': public_grant(grant), 'signature_source': signature_source}) finish() diff --git a/src/msg/security/authentication.py b/src/msg/security/authentication.py index 3a7291b1..8885287d 100644 --- a/src/msg/security/authentication.py +++ b/src/msg/security/authentication.py @@ -20,6 +20,7 @@ 'identity.custodial_upgrade_start','identity.custodial_upgrade_finish', 'identity.custodial_rewrap_entry','identity.custodial_rewrap_revision', 'identity.custodial_rewrap_ack', + 'achievement.start','achievement.answer','achievement.finish', }) @@ -127,7 +128,12 @@ async def authenticate(self,request,session, *, entry): request.arguments.get('direction')=='download') require(spec.name in CUSTODIAL_SIGNED_WRITES or download_only, 'custodial_operation_not_supported') - require(not spec.require_signature or method=='signature','signature_required') + # The achievement evaluator signs the explicit final confirmation with + # the current vault key; ordinary token subjects still need a signature. + custodial_confirmation=(actor.kind=='custodial' and method=='token' and + spec.name=='achievement.finish') + require(not spec.require_signature or method=='signature' or custodial_confirmation, + 'signature_required') valid=[] for cid in ids: cert=await self.certificates.validate(cid,session) diff --git a/tests/test_achievements.py b/tests/test_achievements.py index 8a2a8ca5..1aaa133c 100644 --- a/tests/test_achievements.py +++ b/tests/test_achievements.py @@ -160,3 +160,85 @@ async def test_expired_round_can_restart_without_submitting_old_answer(installed assert advanced.status == 'ok' and advanced.data['round'] == 2 async with app.metadata.transaction(write=False) as tx: assert tx.one('SELECT COUNT(*) FROM achievement_grants WHERE subject=?', (subject,))[0] == 0 + + +@pytest.mark.asyncio +@pytest.mark.parametrize('retired', [False, True]) +async def test_custodial_ceremony_signs_final_request_and_labels_source(installed, retired): + import os + from msg.core.codec import b64, canonical, decode, unb64 + from msg.core.models import Signature + from msg.security.crypto import verify + + app, _ = installed + created = await call(app, 'identity.custodial_create', { + 'handle': 'achievement-custodial', 'nonce': b64(os.urandom(32)), + 'recovery_secret': b64(os.urandom(32)), + }, contract_version=2) + assert created.status == 'ok', wire(created) + subject = created.data['subject_id'] + token = (created.data['credential_id'], unb64(created.data['token'])) + challenge = await call(app, 'achievement.start', {}, subject=subject, token=token) + assert challenge.status == 'ok', wire(challenge) + for _ in range(4): + data = challenge.data + challenge = await call(app, 'achievement.answer', { + 'challenge_id': data['challenge_id'], 'round': data['round'], + 'question_digest': data['question_digest'], 'nonce': data['nonce'], + 'answer': answer_for(data), + }, subject=subject, token=token) + assert challenge.status == 'ok', wire(challenge) + data = challenge.data + args = {name: data[name] for name in ('challenge_id', 'question_digest', 'nonce', 'ceremony_digest')} + args['statement'] = FINAL_STATEMENT + if retired: + async with app.metadata.transaction(write=True) as tx: + tx.execute("UPDATE custodial_vault SET status='decrypt_only', signing_nonce=NULL, signing_ciphertext=NULL WHERE subject=?", (subject,), write=True) + finished = await call(app, 'achievement.finish', args, subject=subject, token=token) + if retired: + assert finished.status == 'error' and finished.error.code == 'custodial_vault_unavailable' + async with app.metadata.transaction(write=False) as tx: + assert tx.one('SELECT COUNT(*) FROM achievement_grants WHERE subject=?', (subject,))[0] == 0 + return + assert finished.status == 'ok', wire(finished) + assert finished.data['signature_source'] == 'custodial' + grant = finished.data['grant'] + assert grant['auth_method'] == 'token' + assert grant['metadata'] == {'protocol_passed': True, 'signature_source': 'custodial'} + async with app.metadata.transaction(write=False) as tx: + state = loads(tx.one('SELECT body FROM achievement_ceremonies WHERE id=?', (data['challenge_id'],))[0]) + credential = await tx.credential(state['final_signature']['key_id']) + verify(credential.verifier, canonical(state['final_confirmation']), + decode(Signature, state['final_signature']), purpose='achievement-confirmation') + assert state['final_confirmation']['ceremony_digest'] == data['ceremony_digest'] + audit = [loads(row[0]) for row in tx.rows('SELECT body FROM audit')] + issued = [row for row in audit if row['event']['type'] == 'achievement.auto.issue'] + assert len(issued) == 1 + assert issued[0]['event']['data']['signature_source'] == 'custodial' + assert issued[0]['event']['data']['auth_method'] == 'token' + repeat = await call(app, 'achievement.start', {}, subject=subject, token=token) + assert repeat.status == 'error' and repeat.error.code == 'achievement_already_granted' + private = await call(app, 'discovery.get', {'id': '/private'}, subject=subject, token=token) + assert private.status == 'error' + + +@pytest.mark.asyncio +async def test_non_custodial_token_cannot_finish_ceremony(installed): + from msg.core.codec import unb64 + from test_service import temporary_v3_args + app, _ = installed + args, rid, _, _ = temporary_v3_args() + created = await call(app, 'identity.temporary', args, rid=rid, contract_version=3) + assert created.status == 'ok', wire(created) + subject = created.data['subject_id'] + token = (created.data['credential_id'], unb64(created.data['token'])) + started = await call(app, 'achievement.start', {}, subject=subject, token=token) + assert started.status == 'ok', wire(started) + denied = await call(app, 'achievement.finish', { + 'challenge_id': started.data['challenge_id'], + 'question_digest': started.data['question_digest'], 'nonce': started.data['nonce'], + 'ceremony_digest': 'sha256:' + '0' * 64, 'statement': FINAL_STATEMENT, + }, subject=subject, token=token) + assert denied.status == 'error' and denied.error.code == 'signature_required' + async with app.metadata.transaction(write=False) as tx: + assert tx.one('SELECT COUNT(*) FROM achievement_grants WHERE subject=?', (subject,))[0] == 0 From 63417ac4ed93ffcfa2c71aa0fee910353fc48df2 Mon Sep 17 00:00:00 2001 From: LIghtJUNction Date: Tue, 29 Sep 2026 18:16:03 +0800 Subject: [PATCH 028/136] fix: validate legacy offer catalog before new redemption --- docs/OFFER_ORDER_COMPATIBILITY.md | 69 ++++++++++++++++++++++ src/msg/plugins/offers.py | 15 ++++- tests/test_offer_compatibility_boundary.py | 60 +++++++++++++++++++ 3 files changed, 142 insertions(+), 2 deletions(-) create mode 100644 docs/OFFER_ORDER_COMPATIBILITY.md create mode 100644 tests/test_offer_compatibility_boundary.py diff --git a/docs/OFFER_ORDER_COMPATIBILITY.md b/docs/OFFER_ORDER_COMPATIBILITY.md new file mode 100644 index 00000000..7262ee8a --- /dev/null +++ b/docs/OFFER_ORDER_COMPATIBILITY.md @@ -0,0 +1,69 @@ +# Offer and purchase compatibility boundary + +This audit uses the market integration at `65b7330`. It does **not** claim +ServerOffer → Listing or Purchase → Order convergence is implemented. + +## Current authoritative records + +- `plugins/offers.py` reads `server_offers` for `money.offers@1` and + `money.redeem@1/@2`. `admin/money.py:apply_offer` alone changes local offers, + retaining the Root signature/audit boundary. Offers are not Resource revisions. +- Redemption freezes the offer in `money_purchases.offer_snapshot`, allocates a + `purchase_escrow` LedgerAccount, and posts named `purchase_fund` and + `purchase_final` legs through the shared `_post_transfer` function. It does not + allocate a `store_orders` row or a `store_deliveries` row. +- `money.redeem@1` completes synchronously; @2 may defer. Existing pending + purchases settle against their frozen quote even after catalog edits or + disabling. They can refund only while funds remain in escrow. Settled funds + belong to Root and `money.purchase_cancel` rejects settled purchases. +- `market/order_records.py:read_order` reads only `store_orders`. It is also used + by mutation paths, so adding a legacy fallback there would accidentally expose + old purchases to settlement/delivery/arbitration code written for store orders. +- `resource_entitlements` retains an FK to `server_offers`; purchase escrow + validation and backup restore also rely on the original purchase source ID. + +## Safe compatibility gate + +Old enabled catalog rows now pass the same `validate_local_offer` constraints as +new console-created offers before advertisement or a *new* redemption. An invalid +unit, quantity range or duration is invisible and cannot fund an escrow. This is +a read-time filter, not data repair. It does not change the signed snapshot of an +existing purchase or its ability to settle against that snapshot. + +`tests/test_offer_compatibility_boundary.py` exercises those cases against the +normal PostgreSQL fixture, verifies no ledger/account/purchase/entitlement +changes for rejection, and verifies the original funding receipt and locked +snapshot survive a later catalog edit. Existing redemption/backup tests remain +required regression coverage. + +## Required next migration contract + +The design requires a compatible read or explicit migration; neither exists yet. +The implementation must define these mappings before migrating any live rows: + +1. Preserve `pur_*`/offer IDs and their references. Define whether the original + IDs are the canonical compatibility identities or aliases for new `ord_*` + and Resource IDs; aliases must not silently rewrite old signed requests or + escrow source IDs. Reject collisions, unknown states and incomplete sources. +2. Define an explicit **legacy entitlement policy**, separate from new store + signature-acceptance policy. Legacy settled purchases contain no buyer + acceptance fact. A migrated/read-projected order must never synthesize one, + nor fabricate a Delivery claim or signature from a grant. +3. Define how immutable offer snapshots become revision/terms references. Current + offers are mutable rows and a historical snapshot may differ from them. A + migration cannot substitute the current price revision or recompute historical + payment/receipt digests as though they were original signed facts. +4. Preserve owner-only legacy reads and Root/local-only mutations. A generic + seller view or public Listing index must not widen purchase access or give + ordinary sellers an entitlement issuance path. +5. Bind each migrated escrow to exactly one authoritative business record and + settlement path. Test pending cancellation, settlement, concurrent retry, + recovery and rollback before retiring `purchase_escrow` references. A view + alone must not make both purchase and store mutation handlers spend it. +6. Store and verify migration provenance/checkpoints and provide rollback that + retains exact receipt bytes, ledger sequence, request replay results and + entitlement IDs. No startup-time silent financial rewrite. + +These are implementation obligations for the next slice, not permission to +change published request semantics. The present patch changes no DDL, request +schema, historical record, financial state machine or production data. diff --git a/src/msg/plugins/offers.py b/src/msg/plugins/offers.py index 976eb666..51e66a63 100644 --- a/src/msg/plugins/offers.py +++ b/src/msg/plugins/offers.py @@ -65,6 +65,17 @@ def validate_local_offer(app, *, resource_kind: str, entitlement_kind: str, 'invalid_offer_duration') +def _valid_catalog_offer(app, row): + """Apply current issuer bounds to old rows without rewriting signed history.""" + try: + validate_local_offer(app, resource_kind=row[1], unit=row[2], price_minor=row[3], + min_quantity=row[4], max_quantity=row[5], + entitlement_kind=row[6], duration_seconds=row[7]) + except Failure: + return False + return True + + def _public_offer(row): return dict(zip(('offer_id','resource_kind','unit','price_minor','min_quantity', 'max_quantity','entitlement_kind','duration_seconds','price_revision'),row)) | { @@ -115,7 +126,7 @@ async def offers(ctx,request,tx): max_quantity,entitlement_kind,duration_seconds,price_revision FROM server_offers WHERE enabled=TRUE ORDER BY offer_id''') return HandlerOutput(data={'currency_id':CURRENCY_ID,'offers':[ - _public_offer(row) for row in rows if purchasable(app,row[1],row[6])]}) + _public_offer(row) for row in rows if _valid_catalog_offer(app, row)]}) @op('money.redeem',obj({'offer_id':IDENTIFIER,'quantity':{'type':'integer','minimum':1,'maximum':MAX_MINOR}, 'currency_id':{'const':CURRENCY_ID},'price_revision':IDENTIFIER}, @@ -129,7 +140,7 @@ async def redeem(ctx,request,tx): row = tx.one('''SELECT offer_id,resource_kind,unit,price_minor,min_quantity, max_quantity,entitlement_kind,duration_seconds,price_revision FROM server_offers WHERE offer_id=? AND enabled=TRUE''',(args['offer_id'],)) - require(row is not None and purchasable(app,row[1],row[6]),'offer_not_found') + require(row is not None and _valid_catalog_offer(app, row),'offer_not_found') require(args['price_revision'] == row[8],'offer_price_changed') require(row[4] <= args['quantity'] <= row[5] and row[3]*args['quantity'] <= MAX_MINOR, 'invalid_offer_quantity') diff --git a/tests/test_offer_compatibility_boundary.py b/tests/test_offer_compatibility_boundary.py new file mode 100644 index 00000000..d6e2db1f --- /dev/null +++ b/tests/test_offer_compatibility_boundary.py @@ -0,0 +1,60 @@ +"""Legacy catalogue rows must satisfy the same issuer limits as new local offers.""" +import pytest + +from msg.plugins.hosting_capacity import MAX_CAPACITY_BYTES +from test_market_redemption import setup_offer +from test_service import call + + +@pytest.mark.asyncio +@pytest.mark.parametrize(('column', 'value'), [ + ('unit', 'megabyte'), + ('max_quantity', MAX_CAPACITY_BYTES + 1), + ('duration_seconds', 315360001), +]) +async def test_invalid_legacy_offer_is_hidden_and_cannot_fund(installed, column, value): + app, root = installed + key, owner, args, _ = await setup_offer(app, root) + async with app.metadata.transaction(write=True) as tx: + tx.execute(f'UPDATE server_offers SET {column}=? WHERE offer_id=?', + (value, args['offer_id']), write=True) + tables = ('money_ledger', 'money_purchases', 'resource_entitlements', 'ledger_accounts') + async with app.metadata.transaction(write=False) as tx: + before = {table: tx.rows('SELECT * FROM ' + table + ' ORDER BY id') for table in tables} + original = tx.rows('SELECT * FROM server_offers') + catalog = await call(app, 'money.offers', {}) + assert catalog.status == 'ok' and len(catalog.data['offers']) == 0 + for version in (1, 2): + result = await call(app, 'money.redeem', args, key=key, subject=owner, + contract_version=version) + assert result.status == 'error' and result.error.code == 'offer_not_found' + async with app.metadata.transaction(write=False) as tx: + assert {table: tx.rows('SELECT * FROM ' + table + ' ORDER BY id') for table in tables} == before + assert tx.rows('SELECT * FROM server_offers') == original + + +@pytest.mark.asyncio +async def test_catalog_filter_does_not_rewrite_or_invalidate_locked_purchase(installed): + app, root = installed + key, owner, args, _ = await setup_offer(app, root) + result = await call(app, 'money.redeem', {**args, 'defer': True}, key=key, + subject=owner, contract_version=2, rid='legacy-compatible-purchase') + assert result.status == 'ok' + original = result.data['purchase'] + async with app.metadata.transaction(write=True) as tx: + tx.execute('UPDATE server_offers SET unit=? WHERE offer_id=?', + ('megabyte', args['offer_id']), write=True) + signed_funding = tx.one('SELECT receipt FROM money_ledger WHERE id=?', + (original['funding_transaction_id'],)) + assert len((await call(app, 'money.offers', {})).data['offers']) == 0 + read = await call(app, 'money.purchase_get', {'purchase_id': original['id']}, + key=key, subject=owner) + assert read.data['purchase'] == original + settled = await call(app, 'money.purchase_settle', {'purchase_id': original['id']}, + key=key, subject=owner) + assert settled.status == 'ok' + assert settled.data['purchase']['state'] == 'settled' + assert settled.data['purchase']['offer_snapshot'] == original['offer_snapshot'] + async with app.metadata.transaction(write=False) as tx: + assert tx.one('SELECT receipt FROM money_ledger WHERE id=?', + (original['funding_transaction_id'],)) == signed_funding From 9517ef33e014092ca607eb121eff0034693cce0c Mon Sep 17 00:00:00 2001 From: LIghtJUNction Date: Tue, 29 Sep 2026 18:16:42 +0800 Subject: [PATCH 029/136] fix: prevent replayed topic bans from reviving old admin membership --- docs/TASK_A_RECOVERY_20260928.md | 2 +- src/msg/admin/recovery_replay.py | 11 +++++++++- tests/test_task_a_recovery_checkpoint.py | 28 ++++++++++++++++++++++++ 3 files changed, 39 insertions(+), 2 deletions(-) diff --git a/docs/TASK_A_RECOVERY_20260928.md b/docs/TASK_A_RECOVERY_20260928.md index 6f249a3d..c564e2fc 100644 --- a/docs/TASK_A_RECOVERY_20260928.md +++ b/docs/TASK_A_RECOVERY_20260928.md @@ -39,7 +39,7 @@ Refs #64, #65, #68, #69, #70, #84, #85；接续 #112，不关闭现场验收项 同序号内容变化会被精确 pin 拒绝；跨水位的已重放前缀不一致会拒绝。 支持 credential、certificate、ShareGrant v1/v2、ShareLink 撤销，普通 Membership 与 TopicMembership 移除， -IdentityKey/EncryptionSubkey 退役，以及指定账号在线 vault 销毁。只收缩现存权限：不新增身份、授权、 +IdentityKey/EncryptionSubkey 退役，以及指定账号在线 vault 销毁。 `topic_ban.apply` 同时将旧快照里的话题成员降为 member/removed，与正常禁言一致；后续 `topic_ban.lift` 不恢复旧成员资格或 admin 角色。只收缩现存权限：不新增身份、授权、 密钥、财务交易、外发 job 或任何历史正文；证书的原始签署 body 不改。 `g_public` 是虚拟成员关系，不能通过修改一条 Membership 假装移除，明确拒绝该事实。 diff --git a/src/msg/admin/recovery_replay.py b/src/msg/admin/recovery_replay.py index 573f8121..09403b35 100644 --- a/src/msg/admin/recovery_replay.py +++ b/src/msg/admin/recovery_replay.py @@ -165,6 +165,15 @@ async def _apply(tx, fact): if kind == 'topic_ban.apply': require(tx.one('SELECT id FROM resources WHERE id=?', (target,)) is not None, 'recovery_fact_missing') + # A live ban removes membership as well as denying topic participation. + # Preserve that effect when replaying an old snapshot, including when a + # later fact lifts the ban: lifting never restores an old admin role. + member = tx.one('SELECT role,status FROM topic_memberships WHERE topic=? AND subject=?', + (target, subject)) + member_changed = member is not None and member != ('member', 'removed') + if member_changed: + tx.execute("UPDATE topic_memberships SET role='member',status='removed' " + 'WHERE topic=? AND subject=?', (target, subject), write=True) row = tx.one('SELECT status,expires_at FROM topic_bans WHERE topic=? AND subject=?', (target, subject)) if row is None: tx.execute('''INSERT INTO topic_bans @@ -178,7 +187,7 @@ async def _apply(tx, fact): tx.execute("UPDATE topic_bans SET status='active',expires_at=NULL WHERE topic=? AND subject=?", (target, subject), write=True) return True - return False + return member_changed if kind == 'topic_ban.lift': row = tx.one('SELECT status FROM topic_bans WHERE topic=? AND subject=?', (target, subject)) require(row is not None, 'recovery_fact_missing') diff --git a/tests/test_task_a_recovery_checkpoint.py b/tests/test_task_a_recovery_checkpoint.py index 602159ea..3cf12439 100644 --- a/tests/test_task_a_recovery_checkpoint.py +++ b/tests/test_task_a_recovery_checkpoint.py @@ -354,3 +354,31 @@ def test_packaged_schema_and_example_match_the_replay_contract(): with pytest.raises(Failure, match='^recovery_checkpoint_pin_required$'): verify_checkpoint(example['packet'], pin=None) assert list(validator.iter_errors({})) + + +@pytest.mark.asyncio +@pytest.mark.parametrize('lift', [False, True]) +async def test_replayed_ban_removes_restored_admin_even_after_lift(restored, checkpoint, lift): + from msg.core.models import Resource + body, pin, signed = checkpoint + resource = Resource(id='t_old_admin', type='topic', type_version=1, name='old-admin', + parent=None, owner='u_owner', group='g_public', mode=0o700, generation=1, + revision=None, state='active', created_at=NOW, created_by='u_owner', + modified_at=NOW, modified_by='u_owner') + async with restored.transaction(write=True) as tx: + await tx.insert(resource) + tx.execute('INSERT INTO topic_memberships VALUES (?,?,?,?,?,NULL)', + (resource.id, 'u_owner', 'admin', 'active', wire(NOW)), write=True) + kinds = ['topic_ban.apply'] + (['topic_ban.lift'] if lift else []) + facts = [{'sequence': i, 'kind': kind, 'subject': 'u_owner', + 'target': resource.id, 'at': wire(NOW)} for i, kind in enumerate(kinds, 1)] + body = dict(body, entries=facts, sequence=len(facts)) + pin = replace(pin, digest=digest(body), sequence=len(facts)) + for _ in range(2): + await replay(restored, signed(body), pin=pin) + async with restored.transaction(write=False) as tx: + assert tx.one('SELECT role,status FROM topic_memberships WHERE topic=? AND subject=?', + (resource.id, 'u_owner')) == ('member', 'removed') + assert tx.one('SELECT status FROM topic_bans WHERE topic=? AND subject=?', + (resource.id, 'u_owner')) == ('lifted' if lift else 'active',) + assert active(tx) From 9d6c73e6564bdba92599385e9f2aa1da4a3c96d5 Mon Sep 17 00:00:00 2001 From: LIghtJUNction Date: Tue, 29 Sep 2026 18:17:01 +0800 Subject: [PATCH 030/136] fix(bootstrap): normalize explicit rule relocation declarations --- src/msg/bootstrap.py | 1 + tests/test_system_rule_migration.py | 5 +++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/msg/bootstrap.py b/src/msg/bootstrap.py index c8fb71fe..eade7760 100644 --- a/src/msg/bootstrap.py +++ b/src/msg/bootstrap.py @@ -110,6 +110,7 @@ async def sync_system_sources(tx,contents,now, *, source_root=None,namespace_roo all(isinstance(p,str) for p in move) and move[1]==source_paths.get(rule_id,next(s[4] for s in RULE_SPECS if s[0]==rule_id)), 'system_source_invalid_migration',rule_id) + migrations[rule_id]=tuple(move) for rid,name,mode in (('r_rules','_rules','0555'),('t_wiki','wiki','1777')): await seed_resource(tx,contents,dict(id=rid,type='topic',name=name,parent=namespace_root, owner=ROOT_SUBJECT,group=PUBLIC_GROUP,mode=mode),now) diff --git a/tests/test_system_rule_migration.py b/tests/test_system_rule_migration.py index 10bb62b0..0923f8c3 100644 --- a/tests/test_system_rule_migration.py +++ b/tests/test_system_rule_migration.py @@ -9,7 +9,8 @@ @pytest.mark.asyncio -async def test_source_move_preserves_public_resource_and_revision(installed,tmp_path): +@pytest.mark.parametrize("container", [tuple, list]) +async def test_source_move_preserves_public_resource_and_revision(installed,tmp_path,container): app,_=installed source=tmp_path/'system' shutil.copytree(system_source_root(),source) @@ -18,7 +19,7 @@ async def test_source_move_preserves_public_resource_and_revision(installed,tmp_ moved.parent.mkdir() original.rename(moved) paths={'msg.identity':'rules/moved/identity.md'} - declaration={'msg.identity':('rules/identity.md','rules/moved/identity.md')} + declaration={'msg.identity':container(('rules/identity.md','rules/moved/identity.md'))} async with app.metadata.transaction(write=False) as tx: before=await tx.resource('r_rule_identity') with pytest.raises(Failure,match='system_source_migration_required'): From 7c6e7f4616b5b74fa5e3da12d7acb8457f4aacc2 Mon Sep 17 00:00:00 2001 From: LIghtJUNction Date: Tue, 29 Sep 2026 18:17:45 +0800 Subject: [PATCH 031/136] feat(market): add explicit buyer acceptance checkout contract --- docs/MARKET_CLEARING.md | 2 +- docs/MARKET_CONTRACTS.md | 14 ++- src/msg/admin/diagnostics.py | 2 +- src/msg/admin/market_check.py | 34 ++++-- src/msg/client_market.py | 4 + src/msg/market/delivery.py | 4 +- src/msg/market/escrow.py | 4 +- src/msg/market/orders.py | 22 +++- src/msg/market/policy.py | 6 +- tests/test_market_account_scope.py | 2 +- tests/test_market_explicit_acceptance.py | 138 +++++++++++++++++++++++ 11 files changed, 209 insertions(+), 23 deletions(-) create mode 100644 tests/test_market_explicit_acceptance.py diff --git a/docs/MARKET_CLEARING.md b/docs/MARKET_CLEARING.md index 663ad383..3f5ea420 100644 --- a/docs/MARKET_CLEARING.md +++ b/docs/MARKET_CLEARING.md @@ -83,7 +83,7 @@ These commands use the normal signed client and Registry schemas, not a second A Operation name alone is insufficient authority: private money/market operations also check the credential's scope against the represented account, including cached replays. New operations do not automatically extend old signed CA grants or finite credential ceilings; explicit authority renewal may be needed. Doctor reports authority snapshot gaps instead of upgrading signatures silently. -`doctor`'s read-only `market_clearing` verifies conservation, nonnegative balances, signed ledger receipts, escrow/source consistency, order history, purchase entitlements and bounty nonce/Claim linkage. Default zero supply/no banks/no offers is separately visible. `selftest`'s `market_e2e` is restricted to a fresh isolated instance and Test Root: parse and execute mint 20 and **bank fund 20**, escrow a 10 MSG bounty, pay a current-key PoP Claim, publish the fixed 5 MSG bundle, then automatically deliver and settle. Its final balances are **buyer 5, bank 15, every escrow 0, supply 20 MSG**. The manifest is `msg.lmm.best store selftest`; `hello.txt` is exactly `delivery-ok\n`. It sends no external mail and does not claim SMTP acceptance testing. +`doctor`'s read-only `market_clearing` verifies conservation, nonnegative balances, signed ledger receipts, escrow/source consistency, order history, purchase entitlements and bounty nonce/Claim linkage. Default zero supply/no banks/no offers is separately visible. `selftest`'s `market_e2e` is restricted to a fresh isolated instance and Test Root: parse and execute mint 20 and **bank fund 20**, escrow a 10 MSG bounty, pay a current-key PoP Claim, publish the fixed 5 MSG bundle, then automatically deliver under `orders.buy@4`. Before acceptance it verifies buyer 5, bank 10 and order escrow 5 MSG. The buyer reads and verifies the fixed payload, then signs `delivery.accept@2` to settle. Historical automatic settlement remains covered by `market_lifecycle` and versioned checkout tests. Its final balances are **buyer 5, bank 15, every escrow 0, supply 20 MSG**. The manifest is `msg.lmm.best store selftest`; `hello.txt` is exactly `delivery-ok\n`. It sends no external mail and does not claim SMTP acceptance testing. Backup/restore tests include pending purchases, active entitlement grants, funded/unpaid/settled orders, delivery references, nonce consumption and Claim receipts. Restore intentionally pauses writes; only disposable test restores lift that guard before replay checks. Old ledger-account migration remains idempotent; history is not reconstructed from mutable catalog values. Production migration, real console/SMTP/ingress and deployment acceptance remain separate, authorized work (#84). diff --git a/docs/MARKET_CONTRACTS.md b/docs/MARKET_CONTRACTS.md index b4588d49..91e371e6 100644 --- a/docs/MARKET_CONTRACTS.md +++ b/docs/MARKET_CONTRACTS.md @@ -14,8 +14,16 @@ optional `auto_accept` under `escrow-instant-v1`; `delivery.claim@1` remains its separate signed acknowledgement. Its schema, receipts and decision journal are preserved rather than reinterpreted by the new arbitration flow. -New clients select `orders.buy@3`, or use `orders.create@1` followed by -`orders.fund@1`. The latter signs the returned `order_digest`, exact total and +New CLI checkouts select `orders.buy@4`, or use `orders.create@2` followed by +`orders.fund@1`. These pin contract version 4 and the immutable +`explicit-buyer-acceptance` policy version 1: managed delivery leaves payment in +escrow until the current buyer signs `delivery.accept@2` with the exact delivery +digest. Reading a delivery cannot settle it. Explicit CLI `--contract-version` +still selects historical behavior; existing credentials must authorize the new +operation version before using it. + +Published `orders.buy@3` and `orders.create@1` retain automatic managed settlement. +The reservation flow uses `orders.fund@1`, which signs the returned `order_digest`, exact total and currency. Read `orders.contract` or `orders.get` as the buyer or seller. The existing CLI's schema-driven invocation supports these operations; obtain each schema from discovery rather than constructing URLs containing credentials. @@ -205,7 +213,7 @@ immutable decision journals are retained. Both execution paths share the same internal OrderEscrow account check; neither path can settle the other's order. Existing `delivery.claim@1` remains an acknowledgement, not another payment. -Version-3 contract reads and execution recheck every immutable checkout column +Version-3 and version-4 contract reads and execution recheck every immutable checkout column against its pinned contract. Funding references must identify the corresponding buyer-to-escrow ledger entry. Settlement facts bind funding, receipt references, settlement time and prepared-delivery identity/content, while allowing subsequent diff --git a/src/msg/admin/diagnostics.py b/src/msg/admin/diagnostics.py index 33fa791c..7277095d 100644 --- a/src/msg/admin/diagnostics.py +++ b/src/msg/admin/diagnostics.py @@ -737,8 +737,8 @@ async def approve(key,uid,grants): from msg.admin.custodial_check import check_custodial_history checks['custodial_history_recovery']=await check_custodial_history(app,now) from msg.admin.market_check import check_market, check_market_e2e - checks['market_lifecycle']=await check_market(app,root,call,register,now) checks['market_e2e']=await check_market_e2e(app,root,call,register) + checks['market_lifecycle']=await check_market(app,root,call,register,now) checks.update(await _selftest_search_hosting(app,call,register,test_path)) except Failure as exc: checks['failure']={'code':exc.code} diff --git a/src/msg/admin/market_check.py b/src/msg/admin/market_check.py index 5d29d484..835d543f 100644 --- a/src/msg/admin/market_check.py +++ b/src/msg/admin/market_check.py @@ -67,6 +67,8 @@ async def check_market(app,root,call,register,now): from msg.market.policy import DEFAULT_POLICY from msg.workers.effects import EffectWorker require(app.selftest_run_id is not None,'selftest_namespace_required') + async with app.metadata.transaction(write=False) as tx: + baseline_supply = _supply(tx) bank,bank_id = await register('market-test-bank') buyer,buyer_id = await register('market-test-buyer') async def invoke(name,args,key=buyer,subject=buyer_id,**kw): @@ -158,7 +160,7 @@ async def send(self,job): async with app.metadata.transaction(write=False) as tx: await inspect_market(tx) require(_balance(tx,buyer_id)==5_000_000 and _balance(tx,bank_id)==15_000_000 and - tx.one("SELECT SUM(amount_minor) FROM money_ledger WHERE kind='mint'")[0]==20_000_000, + _supply(tx)==baseline_supply+20_000_000, 'market_conservation_failed') for (escrow,) in tx.rows("SELECT id FROM ledger_accounts WHERE kind IN ('order_escrow','bounty_escrow')"): require(_balance(tx,escrow)==0,'market_escrow_invariant') @@ -260,6 +262,7 @@ async def check_market_e2e(app, root, call, register): from msg.daemon import parser async with app.metadata.transaction(write=False) as tx: baseline=inspect_clearing(app,tx) + require(baseline['total_supply_minor']==0,'selftest_requires_zero_supply') bank_key,bank=await register('bank-test') buyer_key,buyer=await register('market-buyer') @@ -299,7 +302,7 @@ async def checked(name,args,key=None,subject=None,**kwargs): claimed.data['escrow_balance_minor']==0,'selftest_bounty_not_paid') listing=await checked('store.listing_create',{'name':'test-delivery','item_kind':'bundle', 'price_minor':5_000_000,'currency_id':'primary','quantity':1, - 'delivery_mode':'managed_instant','escrow_policy':'escrow-instant-v1', + 'delivery_mode':'managed_instant','escrow_policy':'escrow-v1', 'dispute_policy':'dispute-v1','terms':'Fixed text and file; immediate in-site delivery.'},bank_key,bank) async with app.metadata.transaction(write=False) as tx: files=await tx.resolve((await tx.path(bank))+'/files') @@ -313,28 +316,39 @@ async def checked(name,args,key=None,subject=None,**kwargs): bank_key,bank,expected=((listing_id,listing.data['generation']),)) quote=active.data['listing'] args={'listing_id':listing_id,'listing_revision':quote['listing_revision'],'quantity':1, - 'currency_id':'primary','total_price_minor':5_000_000, - 'package_digest':package.data['package']['digest'],'auto_accept':True} + 'currency_id':'primary','total_price_minor':5_000_000} # No seller request occurs after publishing. The signed buyer request alone - # validates the immutable deposit, delivers, then releases the exact escrow. + # validates the immutable deposit and delivers; payment remains in escrow. bought=await checked('orders.buy',args,buyer_key,buyer,request_id='market-e2e-buy', - contract_version=2) - require(bought.data['order']['state']=='settled','selftest_order_not_settled') - delivery=await checked('delivery.get',{'order_id':bought.data['order']['id']},buyer_key,buyer) + contract_version=4) + require(bought.data['order']['state']=='delivered','selftest_order_not_delivered') + oid=bought.data['order']['id'] + async with app.metadata.transaction(write=False) as tx: + escrow=tx.one('SELECT escrow_subject FROM store_orders WHERE id=?',(oid,))[0] + require((_balance(tx,buyer),_balance(tx,bank),_balance(tx,escrow)) == + (5_000_000,10_000_000,5_000_000),'selftest_awaiting_acceptance_balance_mismatch') + delivery=await checked('delivery.get',{'order_id':oid},buyer_key,buyer,contract_version=2) content=delivery.data['delivery'] require(content['manifest']=={'text':'msg.lmm.best store selftest'} and content['payloads'][0]['data']==b64(b'delivery-ok\n') and content['package_digest']==package.data['package']['digest'],'selftest_delivery_mismatch') repeated=await checked('orders.buy',args,buyer_key,buyer,request_id='market-e2e-buy', - contract_version=2) + contract_version=4) require(repeated.replayed and repeated.data==bought.data,'selftest_market_replay_failed') + require(content['state']=='prepared','selftest_false_buyer_ack') + accepted=await checked('delivery.accept',{'order_id':oid, + 'delivery_digest':content['delivery_digest']},buyer_key,buyer, + contract_version=2,request_id='market-e2e-accept') + require(accepted.data['state']=='settled','selftest_order_not_settled') + async with app.metadata.transaction(write=False) as tx: + require(_balance(tx,escrow)==0,'selftest_escrow_not_empty') buyer_balance=await checked('money.balance',{},buyer_key,buyer) bank_balance=await checked('money.balance',{},bank_key,bank) require(buyer_balance.data['balance_minor']==5_000_000 and bank_balance.data['balance_minor']==15_000_000, 'selftest_market_balance_mismatch') async with app.metadata.transaction(write=False) as tx: checked_state=inspect_clearing(app,tx) - require(checked_state['total_supply_minor']==baseline['total_supply_minor']+20_000_000 and + require(checked_state['total_supply_minor']==20_000_000 and checked_state['banks']==baseline['banks']+1 and checked_state['enabled_offers']==baseline['enabled_offers'], 'selftest_market_supply_mismatch') diff --git a/src/msg/client_market.py b/src/msg/client_market.py index c3cbbb08..35ec0ac7 100644 --- a/src/msg/client_market.py +++ b/src/msg/client_market.py @@ -96,6 +96,10 @@ async def run_command(client, args, parse_arguments): if version is not None: require(type(version) is int and 1 <= version <= 2**31 - 1, 'invalid_contract_version') kwargs['contract_version'] = version + elif operation == 'orders.buy': + kwargs['contract_version'] = 4 + elif operation == 'orders.create': + kwargs['contract_version'] = 2 elif operation=='money.redeem': kwargs['contract_version']=2 if args.command=='store' and args.action=='update': diff --git a/src/msg/market/delivery.py b/src/msg/market/delivery.py index 2c916ad7..22b0930b 100644 --- a/src/msg/market/delivery.py +++ b/src/msg/market/delivery.py @@ -110,7 +110,7 @@ async def prepare(app, tx, ctx, request, order, kind, manifest, refs, *, envelop async def automatic(app, tx, ctx, request, order): - """An authenticated payment authorizes the pinned managed-instant policy. + """Prepare automatically; release only under the pinned settlement policy. Only verified objective storage faults refund. An arbitrary exception rolls back the executor transaction, rather than turning software bugs into payouts. @@ -124,6 +124,8 @@ async def automatic(app, tx, ctx, request, order): request_id=request.request_id, reason=exc.code, refund_minor=order['total_price_minor']) return await prepare(app, tx, ctx, request, order, kind, manifest, refs) + if contract(tx, order['id'])['version'] == 4: + return await transition(tx, order, 'accepted', now=ctx.now, actor=order['buyer'], request_id=request.request_id, reason='managed_instant_verified') await settle(app, tx, order, now=ctx.now, actor=order['buyer'], diff --git a/src/msg/market/escrow.py b/src/msg/market/escrow.py index 39c4237d..9b3ece99 100644 --- a/src/msg/market/escrow.py +++ b/src/msg/market/escrow.py @@ -67,7 +67,9 @@ async def settle(app, tx, order, *, now, actor, request_id, reason, Caller owns the transaction. Any failure rolls back both legs, the decision consumption, transitions, receipts, events and request result together. """ - contract(tx, order['id']) + locked = contract(tx, order['id']) + require(not (locked['version'] == 4 and reason == 'managed_instant_verified'), + 'escrow_release_forbidden') require(tx.one('SELECT 1 FROM order_escrow_decisions WHERE order_id=?', (order['id'],)) is None, 'order_already_settled') total = order['total_price_minor'] diff --git a/src/msg/market/orders.py b/src/msg/market/orders.py index d7b7c7c7..2182968d 100644 --- a/src/msg/market/orders.py +++ b/src/msg/market/orders.py @@ -1,4 +1,4 @@ -"""New checkout contracts without changing published orders.buy@1/@2 semantics.""" +"""Versioned checkout, preserving published orders.buy@1/@2/@3 semantics.""" from __future__ import annotations from datetime import timedelta @@ -35,8 +35,8 @@ def view(tx, order, viewer): row = tx.one('SELECT digest FROM order_contracts WHERE order_id=?', (order['id'],)) if row: result['order_digest'] = row[0] - result['contract_version'] = 3 locked = contract(tx, order['id']) + result['contract_version'] = locked['version'] result['delivery_mode'] = locked['listing']['delivery_mode'] result['policy_digest'] = locked['policy']['policy_digest'] # Seller needs the *public encryption subkey*, not buyer's email. @@ -46,7 +46,7 @@ def view(tx, order, viewer): return result -async def create(app, tx, ctx, request): +async def create(app, tx, ctx, request, *, version=3): buyer, args = _subject(ctx), request.arguments listing = await _listing(app, ctx, request, tx, args['listing_id']) body, revision = await _body(app, tx, listing) @@ -95,7 +95,7 @@ async def create(app, tx, ctx, request): package_digest,args['quantity'],body['price_minor'],total,CURRENCY_ID,escrow, body['escrow_policy'],body['dispute_policy'],digest(body['terms']),canonical(target).decode(), request.payload_digest,now), write=True) - locked = {'order_id': order_id, 'version': 3, 'buyer': buyer, 'seller': listing.owner, + locked = {'order_id': order_id, 'version': version, 'buyer': buyer, 'seller': listing.owner, 'listing': body, 'listing_id': listing.id, 'escrow_subject': escrow, 'listing_revision': listing.revision, 'listing_digest': revision.content.digest, 'package_id': package_id, @@ -104,6 +104,8 @@ async def create(app, tx, ctx, request): 'terms_digest': digest(body['terms']), 'policy': policy, 'recipient_key': recipient_key, 'handle_snapshot': target['handle_snapshot'], 'buyer_principal': wire(ctx.principal), 'created_at': now} + if version == 4: + locked['settlement_policy'] = {'id': 'explicit-buyer-acceptance', 'version': 1} tx.execute('INSERT INTO order_contracts(order_id,body,digest) VALUES (?,?,?)', (order_id,canonical(locked).decode(),digest(locked)), write=True) tx.execute('INSERT INTO order_deadlines(order_id,expires_at) VALUES (?,?)', @@ -149,6 +151,11 @@ async def reserve(ctx, request, tx): order = await create(app, tx, ctx, request) return HandlerOutput(data={'order': view(tx, order, order['buyer'])}) + @op('orders.create', obj(INTENT, INTENT_REQUIRED), signature=True, version=2) + async def reserve_explicit(ctx, request, tx): + order = await create(app, tx, ctx, request, version=4) + return HandlerOutput(data={'order': view(tx, order, order['buyer'])}) + @op('orders.buy', obj(INTENT, INTENT_REQUIRED), signature=True, version=3) async def buy(ctx, request, tx): order = await create(app, tx, ctx, request) @@ -156,6 +163,13 @@ async def buy(ctx, request, tx): return HandlerOutput(data={'order': view(tx, _row(tx, order['id'], order['buyer']), order['buyer']), 'payment': receipt}) + @op('orders.buy', obj(INTENT, INTENT_REQUIRED), signature=True, version=4) + async def buy_explicit(ctx, request, tx): + order = await create(app, tx, ctx, request, version=4) + receipt = await fund(app, tx, ctx, request, order) + return HandlerOutput(data={'order': view(tx, _row(tx, order['id'], order['buyer']), order['buyer']), + 'payment': receipt}) + @op('orders.fund', obj({'order_id': IDENTIFIER, 'order_digest': HASH, 'total_price_minor': INTENT['total_price_minor'], 'currency_id': {'const': CURRENCY_ID}}, ('order_id','order_digest','total_price_minor','currency_id')), signature=True) diff --git a/src/msg/market/policy.py b/src/msg/market/policy.py index c4e0a250..3321a1c4 100644 --- a/src/msg/market/policy.py +++ b/src/msg/market/policy.py @@ -89,7 +89,11 @@ def delivery_snapshot(tx, order_id): def validate_projection(tx, locked): """A restored mutable projection cannot replace the immutable checkout facts.""" - require(locked.get('version') == 3, 'order_contract_version') + require(locked.get('version') in {3, 4}, 'order_contract_version') + if locked['version'] == 4: + require(locked.get('settlement_policy') == + {'id': 'explicit-buyer-acceptance', 'version': 1}, + 'order_contract_version') listing = locked['listing'] expected = {name: locked[name] for name in ( 'buyer', 'seller', 'listing_id', 'listing_revision', 'package_id', diff --git a/tests/test_market_account_scope.py b/tests/test_market_account_scope.py index 31e53b1e..b11e6c63 100644 --- a/tests/test_market_account_scope.py +++ b/tests/test_market_account_scope.py @@ -103,7 +103,7 @@ async def test_account_scoped_read_key_preserves_owner_projection(installed, ope @pytest.mark.asyncio -@pytest.mark.parametrize('version', [1, 2, 3]) +@pytest.mark.parametrize('version', [1, 2, 3, 4]) async def test_catalog_scope_alone_cannot_buy_in_any_published_version(installed, version): app, root = installed _, _, buyer_key, buyer, listing, package = await market(app, root) diff --git a/tests/test_market_explicit_acceptance.py b/tests/test_market_explicit_acceptance.py new file mode 100644 index 00000000..145f3bd3 --- /dev/null +++ b/tests/test_market_explicit_acceptance.py @@ -0,0 +1,138 @@ +"""V4 managed delivery keeps payment in escrow until signed buyer acceptance.""" +import pytest + +from msg.core.codec import wire +from msg.plugins.money import _balance +from test_market_lifecycle import market +from test_orders import _intent +from test_service import call + + +@pytest.mark.asyncio +async def test_v4_delivery_requires_current_buyer_and_exact_digest_before_settlement(installed): + app, root = installed + sk, seller, bk, buyer, listing, _ = await market(app, root) + args = _intent(listing) + bought = await call(app, 'orders.buy', args, key=bk, subject=buyer, + contract_version=4, rid='explicit-buy') + assert bought.status == 'ok', wire(bought) + order = bought.data['order'] + assert order['state'] == 'delivered' and order['contract_version'] == 4 + contract = await call(app, 'orders.contract', {'order_id': order['id']}, + key=bk, subject=buyer) + assert contract.status == 'ok', wire(contract) + assert contract.data['contract']['settlement_policy'] == { + 'id': 'explicit-buyer-acceptance', 'version': 1} + replay = await call(app, 'orders.buy', args, key=bk, subject=buyer, + contract_version=4, rid='explicit-buy') + assert replay.replayed and replay.data == bought.data + delivery = await call(app, 'delivery.get', {'order_id': order['id']}, + key=bk, subject=buyer, contract_version=2) + assert delivery.status == 'ok', wire(delivery) + assert delivery.data['delivery']['state'] == 'prepared' + accept = {'order_id': order['id'], + 'delivery_digest': delivery.data['delivery']['delivery_digest']} + wrong_buyer = await call(app, 'delivery.accept', accept, + key=sk, subject=seller, contract_version=2) + assert wrong_buyer.status == 'error' + wrong_digest = await call(app, 'delivery.accept', + {**accept, 'delivery_digest': 'sha256:' + '0' * 64}, + key=bk, subject=buyer, contract_version=2) + assert wrong_digest.error.code == 'delivery_mismatch' + async with app.metadata.transaction(write=False) as tx: + escrow = tx.one('SELECT escrow_subject FROM store_orders WHERE id=?', (order['id'],))[0] + assert (_balance(tx, buyer), _balance(tx, seller), _balance(tx, escrow)) == (15_000_000, 0, 5_000_000) + assert tx.one('SELECT COUNT(*) FROM order_settlements WHERE order_id=?', (order['id'],))[0] == 0 + accepted = await call(app, 'delivery.accept', accept, key=bk, subject=buyer, + contract_version=2, rid='explicit-accept') + assert accepted.status == 'ok' and accepted.data['state'] == 'settled', wire(accepted) + repeated = await call(app, 'delivery.accept', accept, key=bk, subject=buyer, + contract_version=2, rid='explicit-accept') + assert repeated.replayed and repeated.data == accepted.data + again = await call(app, 'delivery.accept', accept, key=bk, subject=buyer, + contract_version=2, rid='explicit-accept-again') + assert again.status == 'ok' and again.data['receipt'] == accepted.data['receipt'] + async with app.metadata.transaction(write=False) as tx: + assert (_balance(tx, buyer), _balance(tx, seller), _balance(tx, escrow)) == (15_000_000, 5_000_000, 0) + assert tx.one('SELECT COUNT(*) FROM order_settlements WHERE order_id=?', (order['id'],))[0] == 1 + + +@pytest.mark.asyncio +async def test_cli_default_checkout_waits_for_signed_acceptance(installed): + from test_market_cli_bindings import ExecutorClient, command + from msg.core.codec import canonical + app, root = installed + _, _, bk, buyer, listing, _ = await market(app, root) + bought = await command(ExecutorClient(app, bk, buyer), 'orders', 'buy', + canonical(_intent(listing)).decode()) + assert bought.status == 'ok', wire(bought) + assert bought.data['order']['contract_version'] == 4 + assert bought.data['order']['state'] == 'delivered' + + +@pytest.mark.asyncio +async def test_revoked_acceptance_key_cannot_replay_settlement(installed): + from test_market_account_scope import restricted_key + app, root = installed + _, _, bk, buyer, listing, _ = await market(app, root) + accept_key = await restricted_key(app, bk, buyer, 'delivery.accept', 2, account=True) + bought = await call(app, 'orders.buy', _intent(listing), key=bk, subject=buyer, + contract_version=4) + order = bought.data['order'] + delivery = await call(app, 'delivery.get', {'order_id': order['id']}, + key=bk, subject=buyer, contract_version=2) + args = {'order_id': order['id'], + 'delivery_digest': delivery.data['delivery']['delivery_digest']} + accepted = await call(app, 'delivery.accept', args, key=accept_key, subject=buyer, + contract_version=2, rid='accept-revocable') + assert accepted.status == 'ok', wire(accepted) + revoked = await call(app, 'identity.key_revoke', {'key_id': accept_key.key_id}, + key=bk, subject=buyer) + assert revoked.status == 'ok', wire(revoked) + replay = await call(app, 'delivery.accept', args, key=accept_key, subject=buyer, + contract_version=2, rid='accept-revocable') + assert replay.status == 'error' and not replay.replayed + + +@pytest.mark.asyncio +async def test_explicit_reservation_policy_survives_separate_funding(installed): + app, root = installed + _, _, bk, buyer, listing, _ = await market(app, root) + created = await call(app, 'orders.create', _intent(listing), key=bk, subject=buyer, + contract_version=2) + assert created.status == 'ok', wire(created) + order = created.data['order'] + assert order['contract_version'] == 4 + funded = await call(app, 'orders.fund', { + 'order_id': order['id'], 'order_digest': order['order_digest'], + 'total_price_minor': order['total_price_minor'], 'currency_id': 'primary'}, + key=bk, subject=buyer) + assert funded.status == 'ok' and funded.data['order']['state'] == 'delivered', wire(funded) + + +@pytest.mark.asyncio +async def test_waiting_acceptance_survives_restart_without_release(installed): + from msg.application import Application + from msg.admin.market_check import inspect_clearing + from test_service import NOW + app, root = installed + _, seller, bk, buyer, listing, _ = await market(app, root) + bought = await call(app, 'orders.buy', _intent(listing), key=bk, subject=buyer, + contract_version=4) + assert bought.status == 'ok', wire(bought) + order = bought.data['order'] + restarted = Application(app.settings, clock=lambda: NOW) + await restarted.load() + try: + delivery = await call(restarted, 'delivery.get', {'order_id': order['id']}, + key=bk, subject=buyer, contract_version=2) + assert delivery.status == 'ok' and delivery.data['delivery']['state'] == 'prepared' + async with restarted.metadata.transaction(write=False) as tx: + assert inspect_clearing(restarted, tx)['conserved'] + assert _balance(tx, seller) == 0 + accepted = await call(restarted, 'delivery.accept', { + 'order_id': order['id'], 'delivery_digest': delivery.data['delivery']['delivery_digest']}, + key=bk, subject=buyer, contract_version=2) + assert accepted.status == 'ok' and accepted.data['state'] == 'settled', wire(accepted) + finally: + await restarted.close() From 2b6ba8e05e12b8af76afe0aae869a09a90e898d2 Mon Sep 17 00:00:00 2001 From: LIghtJUNction Date: Tue, 29 Sep 2026 18:17:56 +0800 Subject: [PATCH 032/136] fix: map Unicode search snippets to original text offsets --- src/msg/plugins/discovery.py | 20 +++++++++++++++++--- tests/test_lexical_search_grep.py | 24 ++++++++++++++++++++++++ 2 files changed, 41 insertions(+), 3 deletions(-) diff --git a/src/msg/plugins/discovery.py b/src/msg/plugins/discovery.py index 6a1e23c3..19212ad1 100644 --- a/src/msg/plugins/discovery.py +++ b/src/msg/plugins/discovery.py @@ -19,6 +19,19 @@ from msg.plugins.schemas import * +def original_casefold_span(text, folded_start, folded_end): + """Translate a nonempty casefold match back to whole original characters.""" + offset = 0 + start = None + for index, char in enumerate(text): + offset += len(char.casefold()) + if start is None and offset > folded_start: + start = index + if offset >= folded_end: + return start, index + 1 + raise ValueError('casefold match outside source') + + async def visible(app,ctx,request,tx,rid): try: await check_access(app,ctx,request,tx,rid,'read') @@ -929,11 +942,12 @@ async def lexical_search(ctx,request,tx): low=lowered[key] needle=exact if exact and exact in low else next((term for term in terms if term in low),'') if needle: - start=low.index(needle) + folded_start=low.index(needle) + start,end=original_casefold_span(value,folded_start,folded_start+len(needle)) left=max(0,start-40) - right=min(len(value),start+len(needle)+40) + right=min(len(value),end+40) item['snippet']={'field':key,'text':value[left:right], - 'range':[start-left,start-left+len(needle)]} + 'range':[start-left,end-left]} break if a.get('explain')=='compact': item['rank_reason']={'matched_fields':[key for key,value in lowered.items() diff --git a/tests/test_lexical_search_grep.py b/tests/test_lexical_search_grep.py index 4c709c54..6ea7620e 100644 --- a/tests/test_lexical_search_grep.py +++ b/tests/test_lexical_search_grep.py @@ -280,3 +280,27 @@ async def test_sealed_search_query_ref_reads_same_authorized_results(installed): assert {item['ref']['id'] for item in path.json()['items']+continued.json()['items']}=={ result.resources[0].id,second.resources[0].id} assert await business_state(app)==before + + +@pytest.mark.asyncio +@pytest.mark.parametrize('prefix,matched,query', [ + ('ß' * 80, 'Target', 'target'), + ('İ' * 80, 'Straße', 'STRASSE'), + ('x' * 80, 'ß', 's'), +]) +async def test_unicode_casefold_snippet_ranges_address_original_text(installed, prefix, matched, query): + app, _ = installed + key, subject, _ = await register(app, 'unicode-search-owner') + created = await call(app, 'content.post_create', + {'parent': '/main', 'body': prefix + matched + '尾' * 80}, + key=key, subject=subject) + assert created.status == 'ok', wire(created) + result = await call(app, 'discovery.lexical_search', { + 'scope': '/main', 'terms': query, 'field': 'body', 'snippet': True, + }, key=key, subject=subject) + assert result.status == 'ok', wire(result) + item = next(item for item in result.data['items'] if item['id'] == created.resources[0].id) + snippet = item['snippet'] + start, end = snippet['range'] + assert snippet['text'][start:end] == matched + assert snippet['text'] == prefix[-40:] + matched + '尾' * 40 From 680297fe6110919c531f5dd11d224782f7957e2e Mon Sep 17 00:00:00 2001 From: LIghtJUNction Date: Tue, 29 Sep 2026 18:20:02 +0800 Subject: [PATCH 033/136] fix: link plugin manifests to owned bootstrap features --- src/msg/bootstrap.py | 2 + src/msg/core/models.py | 2 + src/msg/core/registry.py | 12 ++++ src/msg/plugins/common.py | 4 +- src/msg/plugins/features.py | 51 +++++++++++++++++ tests/test_manifest_feature_linkage.py | 78 ++++++++++++++++++++++++++ 6 files changed, 148 insertions(+), 1 deletion(-) create mode 100644 src/msg/plugins/features.py create mode 100644 tests/test_manifest_feature_linkage.py diff --git a/src/msg/bootstrap.py b/src/msg/bootstrap.py index eade7760..5e120956 100644 --- a/src/msg/bootstrap.py +++ b/src/msg/bootstrap.py @@ -198,6 +198,8 @@ def feature_manifest(definition=None): require(all(feature[name] is None or (isinstance(feature[name],str) and bool(feature[name])) for name in ('doctor_check','selftest_case')),'feature_manifest_invalid') + from msg.plugins.features import validate_feature_sources + validate_feature_sources(features) return tuple(features) diff --git a/src/msg/core/models.py b/src/msg/core/models.py index bab47f1c..d82aa2f8 100644 --- a/src/msg/core/models.py +++ b/src/msg/core/models.py @@ -474,6 +474,7 @@ class PluginManifest: capabilities: tuple[CapabilitySpec, ...] operations: tuple[OperationSpec, ...] migrations: tuple[str, ...] + feature_ids: tuple[str, ...] = () @dataclass(frozen=True, slots=True, kw_only=True) @@ -483,6 +484,7 @@ class BootstrapManifest: resources: tuple[JsonMap, ...] # 经固定 seed schema 校验 organizations: tuple[JsonMap, ...] templates: tuple[BlobRef, ...] + features: tuple[JsonMap, ...] = () @dataclass(frozen=True, slots=True, kw_only=True) diff --git a/src/msg/core/registry.py b/src/msg/core/registry.py index 958cd2ad..0a05cb8f 100644 --- a/src/msg/core/registry.py +++ b/src/msg/core/registry.py @@ -60,6 +60,8 @@ def add_schema(self,ref,schema): self._validators[ref.id]=validator def add(self,manifest): + from msg.plugins.features import validate_feature_claims + validate_feature_claims(manifest) require(not self._frozen,'registry_frozen') require(manifest.name not in self._plugins,'duplicate_plugin') require(all(d in self._plugins for d in manifest.dependencies),'missing_plugin_dependency') @@ -82,7 +84,17 @@ def add(self,manifest): self.add_operation(item) self._plugins[manifest.name]=manifest + def features(self,plugin_name): + """Resolve a plugin's defaults/checks from the sole bootstrap inventory.""" + from msg.bootstrap import feature_manifest + require(plugin_name in self._plugins,'unknown_plugin') + claims=self._plugins[plugin_name].feature_ids + return tuple(deepcopy(row) for row in feature_manifest() + if row['feature_id'] in claims) + def freeze(self): + from msg.bootstrap import feature_manifest + feature_manifest() from msg.bootstrap import RULE_PATHS require('identity' in self._plugins,'identity_plugin_required') for spec in self._operations.values(): diff --git a/src/msg/plugins/common.py b/src/msg/plugins/common.py index 13a0fcc3..37f99924 100644 --- a/src/msg/plugins/common.py +++ b/src/msg/plugins/common.py @@ -233,7 +233,9 @@ def decorate(handler): return handler return decorate def finish(resource_types=()): + from msg.plugins.features import feature_ids manifest=PluginManifest(name=name,version='1',dependencies=tuple(dependencies), - resource_types=tuple(resource_types),capabilities=(),operations=tuple(operations),migrations=()) + resource_types=tuple(resource_types),capabilities=(),operations=tuple(operations),migrations=(), + feature_ids=feature_ids(name)) app.registry.add(manifest) return operation,finish diff --git a/src/msg/plugins/features.py b/src/msg/plugins/features.py new file mode 100644 index 00000000..a1c3841a --- /dev/null +++ b/src/msg/plugins/features.py @@ -0,0 +1,51 @@ +"""Installed feature ownership; defaults remain owned by bootstrap.json. + +These are diagnostic result keys, never import paths or executable resource data. +One plugin can own several independently reported features. Dependencies remain +owned by PluginManifest and are checked by Registry.add before registration. +""" +from msg.core.errors import require + + +FEATURE_SOURCES = { + 'money': ('money', 'market_clearing', 'market_e2e'), + 'bounty': ('bounty', 'market_clearing', 'market_e2e'), + 'orders': ('orders', 'market_clearing', 'market_e2e'), + 'bootstrap': ('system', 'bootstrap', 'isolated_namespace'), + 'market': ('store', 'market', 'market_lifecycle'), + 'postgres': ('system', 'storage', 'stable_id_read'), + 'root_trust': ('identity', 'root_trust', 'root_network_rejected'), + 'credential_delivery': ('identity', 'credential_delivery', 'credential_delivery_recovery'), + 'custodial_history': ('identity', 'custodial_history', 'custodial_history_recovery'), + 'identity_upgrade': ('identity', 'authority_snapshot', 'identity_upgrade_recovery'), + 'online_ca': ('identity', 'online_ca', 'online_registration'), + 'authorization': ('identity', 'authority_snapshot', 'certgate'), + 'audit': ('system', 'audit', 'online_delegation'), + 'content': ('content', 'content_layout', 'idempotency'), + 'git_content': ('content', 'git', None), + 'valkey_signal': ('system', 'valkey', None), + 'recovery': ('recovery', 'recovery_checkpoint', 'recovery_checkpoint_replay'), + 'search': ('discovery', 'lexical_search', 'lexical_search_access'), + 'hosting': ('extensions', 'hosting', 'hosting_deploy'), +} + + +def feature_ids(plugin_name): + return tuple(key for key, source in FEATURE_SOURCES.items() if source[0] == plugin_name) + + +def validate_feature_sources(rows): + require({row['feature_id'] for row in rows} == set(FEATURE_SOURCES), + 'feature_source_mismatch') + for row in rows: + require((row['doctor_check'], row['selftest_case']) == + FEATURE_SOURCES[row['feature_id']][1:], 'feature_source_mismatch') + + +def validate_feature_claims(manifest): + claims = manifest.feature_ids + require(isinstance(claims, tuple) and all(isinstance(key, str) for key in claims), + 'feature_source_mismatch') + require(len(set(claims)) == len(claims) and + all(key in FEATURE_SOURCES and FEATURE_SOURCES[key][0] == manifest.name + for key in claims), 'feature_source_mismatch') diff --git a/tests/test_manifest_feature_linkage.py b/tests/test_manifest_feature_linkage.py new file mode 100644 index 00000000..56211461 --- /dev/null +++ b/tests/test_manifest_feature_linkage.py @@ -0,0 +1,78 @@ +from copy import deepcopy +from dataclasses import replace + +import pytest + +from msg.bootstrap import feature_manifest, manifest +from msg.core.errors import Failure +from msg.core.models import PluginManifest +from msg.core.registry import Registry + + +def plugin(name='identity', **changes): + return PluginManifest(name=name, version='1', dependencies=(), resource_types=(), + capabilities=(), operations=(), migrations=(), **changes) + + +def test_legacy_manifest_keeps_empty_feature_claims(): + assert plugin().feature_ids == () + + +def test_bootstrap_rejects_unknown_feature_and_wrong_check_owner(): + for changes in ({'feature_id': 'unowned'}, {'doctor_check': 'import:evil'}, + {'selftest_case': 'root_network_rejected'}): + definition = deepcopy(manifest()) + definition['features'][0].update(changes) + with pytest.raises(Failure, match='feature_source_mismatch'): + feature_manifest(definition) + + +def test_feature_claims_reject_other_owner_and_duplicates_atomically(): + registry = Registry() + for claims in (('money',), ('root_trust', 'root_trust'), ('absent',)): + with pytest.raises(Failure, match='feature_source_mismatch'): + registry.add(plugin(feature_ids=claims)) + assert not registry._plugins + registry.add(plugin(feature_ids=('root_trust',))) + registry.freeze() + + +def test_plugin_defaults_resolve_from_bootstrap_and_are_isolated(): + registry = Registry() + registry.add(plugin()) + registry.add(replace(plugin('money', feature_ids=('money',)), dependencies=('identity',))) + registry.freeze() + row, = registry.features('money') + assert row == next(row for row in feature_manifest() if row['feature_id'] == 'money') + row['default_config']['banks'].append('mutated') + assert registry.features('money')[0]['default_config']['banks'] == [] + with pytest.raises(Failure, match='unknown_plugin'): + registry.features('absent') + + +def test_missing_feature_source_is_rejected_at_freeze(monkeypatch): + definition = deepcopy(manifest()) + definition['features'].pop() + monkeypatch.setattr('msg.bootstrap.manifest', lambda: definition) + registry = Registry() + registry.add(plugin()) + with pytest.raises(Failure, match='feature_source_mismatch'): + registry.freeze() + assert not registry.frozen + + +def test_features_do_not_bypass_plugin_dependencies(): + registry = Registry() + dependent = replace(plugin('money', feature_ids=('money',)), dependencies=('identity',)) + with pytest.raises(Failure, match='missing_plugin_dependency'): + registry.add(dependent) + assert not registry._plugins + + +@pytest.mark.asyncio +async def test_installed_manifests_link_entire_inventory(installed): + app, _ = installed + from msg.plugins.features import FEATURE_SOURCES + rows = [row for name in app.registry._plugins for row in app.registry.features(name)] + assert len(rows) == len(FEATURE_SOURCES) + assert {row['feature_id'] for row in rows} == set(FEATURE_SOURCES) From b15d879812538b040195518df50953933b89d1a1 Mon Sep 17 00:00:00 2001 From: LIghtJUNction Date: Tue, 29 Sep 2026 18:21:25 +0800 Subject: [PATCH 034/136] feat: filter search by current revision and source version --- src/msg/plugins/discovery.py | 14 ++- src/msg/plugins/transfer.py | 2 + src/msg/transports/http_routes.py | 32 ++++--- tests/test_search_source_relation_filters.py | 92 +++++++++++++++++++- 4 files changed, 126 insertions(+), 14 deletions(-) diff --git a/src/msg/plugins/discovery.py b/src/msg/plugins/discovery.py index 19212ad1..eecb00dd 100644 --- a/src/msg/plugins/discovery.py +++ b/src/msg/plugins/discovery.py @@ -782,6 +782,12 @@ async def collection_page(arguments,depth, *, last=None,boundary=None): 'properties':{**lexical_schema_v3['properties'], 'suggest':BOOLEAN}} + lexical_schema_v5={**lexical_schema_v4, + 'properties':{**lexical_schema_v4['properties'], + 'revision':IDENTIFIER, + 'source_version':{'type':'integer','minimum':1,'maximum':2147483647}}} + + @op('discovery.lexical_search',lexical_schema_v5,effect='read',version=5) @op('discovery.lexical_search',lexical_schema_v1,effect='read') @op('discovery.lexical_search',lexical_schema,effect='read',version=2) @op('discovery.lexical_search',lexical_schema_v3,effect='read',version=3) @@ -801,7 +807,8 @@ async def lexical_search(ctx,request,tx): a={**saved['arguments'],'cursor':a['cursor']} # A cursor or sealed QueryRef carries arguments from an earlier call. # Keep those arguments inside the version selected for this call too. - require(not (request.contract_version<4 and 'suggest' in a) and + require(not (request.contract_version<5 and {'revision','source_version'}&a.keys()) and + not (request.contract_version<4 and 'suggest' in a) and not (request.contract_version<3 and {'source_kind','relation_type'}&a.keys()) and not (request.contract_version<2 and 'facets' in a), @@ -891,6 +898,11 @@ async def lexical_search(ctx,request,tx): # These predicates inspect only the current revision of an already # readable resource. Historical relations and source metadata must # not affect rank, facets or page positions. + if a.get('revision') and (revision is None or revision.id!=a['revision']): + continue + if 'source_version' in a and (revision is None or + revision.source_version!=a['source_version']): + continue if a.get('source_kind') and (revision is None or revision.source_kind!=a['source_kind']): continue diff --git a/src/msg/plugins/transfer.py b/src/msg/plugins/transfer.py index f1c0dd8c..0c53f488 100644 --- a/src/msg/plugins/transfer.py +++ b/src/msg/plugins/transfer.py @@ -44,6 +44,7 @@ async def sealed_read_query(app,ctx,request,tx,transfer): 'cursor' not in args,'invalid_query_ref') operation='discovery.read_query' if kind=='read' else 'discovery.lexical_search' contract_version=(read_query_version(args) if kind=='read' else + 5 if kind=='search' and {'revision','source_version'}&args.keys() else 4 if kind=='search' and 'suggest' in args else 3 if kind=='search' and any(name in args for name in ('source_kind','relation_type')) else 2 if kind=='search' and 'facets' in args else 1) @@ -195,6 +196,7 @@ async def query_get(ctx,request,tx): require(query.get('query_kind')==kind,'query_ref_digest_mismatch') operation='discovery.read_query' if kind=='read' else 'discovery.lexical_search' contract_version=(read_query_version(args) if kind=='read' else + 5 if kind=='search' and {'revision','source_version'}&args.keys() else 4 if kind=='search' and 'suggest' in args else 3 if kind=='search' and any(name in args for name in ('source_kind','relation_type')) else 2 if kind=='search' and 'facets' in args else 1) diff --git a/src/msg/transports/http_routes.py b/src/msg/transports/http_routes.py index a2c2258f..7c4633e8 100644 --- a/src/msg/transports/http_routes.py +++ b/src/msg/transports/http_routes.py @@ -74,6 +74,7 @@ 'facets':'fc'} SEARCH_V3_SEGMENTS={**SEARCH_V2_SEGMENTS,'source_kind':'sk','relation_type':'rt'} SEARCH_V4_SEGMENTS={**SEARCH_V3_SEGMENTS,'suggest':'sg'} +SEARCH_V5_SEGMENTS={**SEARCH_V4_SEGMENTS,'revision':'rv','source_version':'sv'} GREP_V1_SEGMENTS={'scope':'s','pattern':'t','regex':'r','glob':'g', 'exclude_glob':'x','case_sensitive':'i','before':'b','after':'a', 'max_matches':'m','max_files':'f','files_with_matches':'w', @@ -302,13 +303,18 @@ def compile_lexical_search(query): 'owner','author','tag','state','created_after','created_before', 'updated_after','updated_before','has_attachment','order','limit', 'cursor','snippet','explain','fields','facets','source_kind', - 'relation_type','suggest','depth','recursive'}, + 'relation_type','suggest','revision','source_version','depth','recursive'}, 'unknown_query_parameter') if 'cursor' in query: require(set(query)=={'cursor'},'cursor_query_mismatch') return {'cursor':query['cursor']} require(bool(query.get('scope')),'search_scope_required') args=dict(query) + if 'source_version' in args: + require(args['source_version'].isascii() and args['source_version'].isdecimal() and + len(args['source_version'])<=10 and 1<=int(args['source_version'])<=2147483647, + 'invalid_search_source_version') + args['source_version']=int(args['source_version']) for name in ('limit','depth'): if name in args: require(args[name].isdecimal() and @@ -331,7 +337,8 @@ def compile_lexical_search(query): def decode_search_v2_path(raw_path,version=b'2'): prefix=raw_path.split(b'/',3)[1] - segments=(SEARCH_V4_SEGMENTS if version==b'4' else + segments=(SEARCH_V5_SEGMENTS if version==b'5' else + SEARCH_V4_SEGMENTS if version==b'4' else SEARCH_V3_SEGMENTS if version==b'3' else SEARCH_V2_SEGMENTS) values,proof=decode_query_path(raw_path,prefix,segments,version) modes={'a':'all','n':'any'} @@ -704,7 +711,8 @@ async def dispatch(request:Request): 'cursor_kind_mismatch') if operation=='discovery.lexical_search': saved_args=query.get('arguments',{}) - contract_version=(4 if 'suggest' in saved_args else + contract_version=(5 if {'revision','source_version'}&saved_args.keys() else + 4 if 'suggest' in saved_args else 3 if any(name in saved_args for name in ('source_kind','relation_type')) else 2 if 'facets' in saved_args else 1) @@ -812,14 +820,15 @@ async def dispatch(request:Request): lexical_path_v2=raw_path.startswith((b'/_search/q/2/',b'/_s/q/2/')) lexical_path_v3=raw_path.startswith((b'/_search/q/3/',b'/_s/q/3/')) lexical_path_v4=raw_path.startswith((b'/_search/q/4/',b'/_s/q/4/')) - lexical_path=lexical_path_v2 or lexical_path_v3 or lexical_path_v4 + lexical_path_v5=raw_path.startswith((b'/_search/q/5/',b'/_s/q/5/')) + lexical_path=lexical_path_v2 or lexical_path_v3 or lexical_path_v4 or lexical_path_v5 if path in {'/_search','/_s'} or search_path or lexical_path or raw_path.startswith(( b'/_index/by-tag/',b'/_i/by-tag/')): require(request.method in {'GET','HEAD'},'method_not_allowed') if search_path or lexical_path: require(not request.url.query,'unknown_query_parameter') query,path_proof=(decode_search_v2_path(raw_path, - b'4' if lexical_path_v4 else b'3' if lexical_path_v3 else b'2') + b'5' if lexical_path_v5 else b'4' if lexical_path_v4 else b'3' if lexical_path_v3 else b'2') if lexical_path else decode_search_query_path(raw_path)) else: @@ -832,7 +841,7 @@ async def dispatch(request:Request): lexical=lexical_path or (not is_index and bool(set(query)&{'terms','exact','not_terms','scope','mode','field','order', 'snippet','explain','has_attachment','facets', - 'source_kind','relation_type','suggest'})) + 'source_kind','relation_type','suggest','revision','source_version'})) if lexical: args=compile_lexical_search(query) operation='discovery.lexical_search' @@ -875,7 +884,9 @@ async def dispatch(request:Request): else: packet=request_for(operation,args,service.settings.service_url, source='manual', - contract_version=(4 if lexical and (lexical_path_v4 or + contract_version=(5 if lexical and (lexical_path_v5 or + {'revision','source_version'}&args.keys()) else + 4 if lexical and (lexical_path_v4 or 'suggest' in args) else 3 if lexical and (lexical_path_v3 or 'source_kind' in args or 'relation_type' in args) else @@ -969,18 +980,19 @@ async def dispatch(request:Request): require(request.method in {'GET','HEAD'},'method_not_allowed') require(service.registry.operation('discovery.lexical_search').effect=='read', 'effect_mismatch') - document={'version':4,'operation':'discovery.lexical_search', + document={'version':5,'operation':'discovery.lexical_search', 'segments':SEARCH_V2_SEGMENTS,'segments_v3':SEARCH_V3_SEGMENTS, - 'segments_v4':SEARCH_V4_SEGMENTS, + 'segments_v4':SEARCH_V4_SEGMENTS,'segments_v5':SEARCH_V5_SEGMENTS, 'mode':{'all':'a','any':'n'}, 'field':{'all':'a','body':'b','name':'n','metadata':'m'}, 'order':{'relevance':'r','updated':'u','created':'c','name':'n'}, 'facets':['type','tag'], 'contract_version':{'default':1,'with_facets':2, - 'with_source_or_relation':3,'with_suggest':4}, + 'with_source_or_relation':3,'with_suggest':4,'with_revision_or_source_version':5}, 'template':'/_search/q/2/s/{percent-encoded-scope}/t/{terms}/m/{mode}/f/{field}/n/{limit}', 'template_v3':'/_search/q/3/s/{percent-encoded-scope}/t/{terms}/sk/{source-kind}/rt/{relation-type}', 'template_v4':'/_search/q/4/s/{percent-encoded-scope}/t/{terms}/sg/1', + 'template_v5':'/_search/q/5/s/{percent-encoded-scope}/t/{terms}/rv/{revision}/sv/{source-version}', 'proof_suffix':'/p/{short-lived-signed-OperationRequest}'} etag='"'+digest(document)[7:]+'"' headers={**BASE_HEADERS,'ETag':etag,'Cache-Control':'private, no-cache'} diff --git a/tests/test_search_source_relation_filters.py b/tests/test_search_source_relation_filters.py index 7e895812..f9842f5f 100644 --- a/tests/test_search_source_relation_filters.py +++ b/tests/test_search_source_relation_filters.py @@ -11,7 +11,8 @@ @pytest.mark.asyncio -async def test_v3_query_string_path_and_sealed_ref_share_filters(installed): +@pytest.mark.parametrize('version', [3, 5]) +async def test_v3_query_string_path_and_sealed_ref_share_filters(installed, version): app,_=installed key,user,_=await register(app,'search-v3-http') root=await call(app,'content.post_create',{'parent':'/main','body':'v3needle root'}, @@ -26,8 +27,10 @@ async def test_v3_query_string_path_and_sealed_ref_share_filters(installed): assert query.status_code==path.status_code==200,(query.text,path.text) assert query.content==path.content assert [row['id'] for row in query.json()['items']]==[reply.resources[0].id] - descriptor=canonical({'version':1,'kind':'search','arguments':{ - 'scope':'/main','terms':'v3needle','relation_type':'reply_to'}}) + descriptor_args={'scope':'/main','terms':'v3needle','relation_type':'reply_to'} + if version==5: + descriptor_args['revision']=reply.resources[0].revision + descriptor=canonical({'version':1,'kind':'search','arguments':descriptor_args}) opened=await call(app,'transfer.open',{'direction':'upload','size':len(descriptor), 'digest':digest(descriptor),'media_type':'application/vnd.msg.read-query+json'}, key=key,subject=user) @@ -122,3 +125,86 @@ async def test_v3_old_cursor_rechecks_grants_before_items_facets_and_rank(instal assert wire(continued)['data']['facets'] == {'type': [{'value': 'post', 'count': 1}]} fresh = await call(app, 'discovery.lexical_search', args, contract_version=3) assert [row['id'] for row in fresh.data['items']] == [visible_first] + + +@pytest.mark.asyncio +async def test_v5_revision_and_source_version_filters_use_current_readable_revision(installed): + app, _ = installed + key, subject, _ = await register(app, 'search-v5-owner') + created = await call(app, 'content.post_create', {'parent': '/main', 'body': 'v5revisionneedle'}, + key=key, subject=subject) + assert created.status == 'ok', wire(created) + rid, revision = created.resources[0].id, created.resources[0].revision + args = {'scope': '/main', 'terms': 'v5revisionneedle', 'revision': revision} + found = await call(app, 'discovery.lexical_search', args, contract_version=5) + assert found.status == 'ok', wire(found) + assert [item['id'] for item in found.data['items']] == [rid] + old = await call(app, 'discovery.lexical_search', args, contract_version=4) + assert old.status == 'error' + async with httpx.AsyncClient(transport=httpx.ASGITransport(app=create_app(app)), + base_url=app.settings.service_url) as http: + query = await http.get('/_search', params=args) + path = await http.get('/_s/q/5/s/%2Fmain/t/v5revisionneedle/rv/' + revision) + assert query.status_code == path.status_code == 200, (query.text, path.text) + assert query.content == path.content + edited = await call(app, 'content.post_edit', {'id': rid, 'body': 'v5revisionneedle updated', + 'expected_revision': revision}, key=key, subject=subject, + expected=((rid, created.data['generation']),)) + assert edited.status == 'ok', wire(edited) + gone = await call(app, 'discovery.lexical_search', args, contract_version=5) + assert gone.status == 'ok' and not gone.data['items'] + release = await call(app, 'discovery.lexical_search', + {'scope': '/_rules', 'terms': 'identity', 'source_kind': 'release'}, + contract_version=3) + ref = release.data['items'][0]['id'] + from msg.core.models import ResourceRef + async with app.metadata.transaction(write=False) as tx: + source = await tx.revision(ResourceRef(id=ref)) + filtered = await call(app, 'discovery.lexical_search', { + 'scope': '/_rules', 'terms': 'identity', 'revision': source.id, + 'source_kind': 'release', 'source_version': source.source_version}, contract_version=5) + assert filtered.status == 'ok', wire(filtered) + assert [item['id'] for item in filtered.data['items']] == [ref] + missed = await call(app, 'discovery.lexical_search', { + 'scope': '/_rules', 'terms': 'identity', 'revision': source.id, + 'source_version': source.source_version + 1}, contract_version=5) + assert missed.status == 'ok' and not missed.data['items'] + + +@pytest.mark.asyncio +async def test_v5_source_version_cursor_preserves_contract_and_rejects_downgrade(installed): + app, _ = installed + first = await call(app, 'discovery.lexical_search', { + 'scope': '/_rules', 'terms': 'identity', 'source_kind': 'release', + 'source_version': 1, 'limit': 1}, contract_version=5) + assert first.status == 'ok' and first.data['cursor'], wire(first) + downgraded = await call(app, 'discovery.lexical_search', + {'cursor': first.data['cursor']}, contract_version=4) + assert downgraded.status == 'error' and downgraded.error.code == 'cursor_query_mismatch' + async with httpx.AsyncClient(transport=httpx.ASGITransport(app=create_app(app)), + base_url=app.settings.service_url) as http: + next_page = await http.get(first.data['next']) + assert next_page.status_code == 200, next_page.text + query = await http.get('/_search?scope=%2F_rules&terms=identity&source_version=1') + path = await http.get('/_s/q/5/s/%2F_rules/t/identity/sv/1') + assert query.status_code == path.status_code == 200, (query.text, path.text) + assert query.content == path.content + + +@pytest.mark.asyncio +async def test_v5_revision_filter_does_not_reveal_private_resource(installed): + app, _ = installed + key, subject, _ = await register(app, 'private-revision-owner') + created = await call(app, 'content.post_create', + {'parent': '/main', 'body': 'privatev5needle'}, key=key, subject=subject) + rid = created.resources[0].id + changed = await call(app, 'content.chmod', {'id': rid, 'mode': '0600'}, + key=key, subject=subject, expected=((rid, created.data['generation']),)) + assert changed.status == 'ok', wire(changed) + args = {'scope': '/main', 'terms': 'privatev5needle', + 'revision': created.resources[0].revision, 'facets': ['type'], 'snippet': True} + denied = await call(app, 'discovery.lexical_search', args, contract_version=5) + assert denied.status == 'ok' and not denied.data['items'] + assert wire(denied.data['facets']) == {'type': []} + allowed = await call(app, 'discovery.lexical_search', args, key=key, subject=subject, contract_version=5) + assert allowed.status == 'ok' and [item['id'] for item in allowed.data['items']] == [rid] From b65d32994f98dd2b35c45ffca378182781272ad4 Mon Sep 17 00:00:00 2001 From: LIghtJUNction Date: Tue, 29 Sep 2026 18:21:32 +0800 Subject: [PATCH 035/136] fix(tools): mount managed Python runtime for real bubblewrap execution --- src/msg/workers/sandbox.py | 16 +++++-- tests/test_tool_sandbox_acceptance.py | 62 +++++++++++++++++++++++++++ 2 files changed, 75 insertions(+), 3 deletions(-) create mode 100644 tests/test_tool_sandbox_acceptance.py diff --git a/src/msg/workers/sandbox.py b/src/msg/workers/sandbox.py index ea3cf00e..09e8f2ee 100644 --- a/src/msg/workers/sandbox.py +++ b/src/msg/workers/sandbox.py @@ -26,9 +26,19 @@ async def __call__(self, tool: ToolSpec, arguments: JsonMap, for path in ('/usr', '/bin', '/lib', '/lib64'): if Path(path).exists(): command += ['--ro-bind', path, path] - prefix = str(Path(sys.prefix).resolve()) - if not prefix.startswith('/usr') and prefix not in {'/','/etc','/home','/mnt'}: - command += ['--ro-bind', prefix, prefix] + # A virtualenv may symlink its executable into a separate managed + # Python installation (for example uv). It needs that interpreter and + # stdlib tree as well as the virtualenv's installed dependencies. + for prefix in dict.fromkeys(str(Path(p).resolve()) for p in (sys.prefix, sys.base_prefix)): + if not Path(prefix).is_relative_to('/usr') and prefix not in {'/','/etc','/home','/mnt'}: + command += ['--ro-bind', prefix, prefix] + # Preserve a managed interpreter's alias path used by the venv's + # symlink, without exposing the rest of the user's installation tree. + base_executable = Path(getattr(sys, '_base_executable', sys.executable)) + interpreter_alias = base_executable.parent.parent + if (interpreter_alias != interpreter_alias.resolve() and + interpreter_alias.resolve() == Path(sys.base_prefix).resolve()): + command += ['--ro-bind', str(interpreter_alias.resolve()), str(interpreter_alias)] command += ['--dir','/etc'] for path in ('/etc/resolv.conf', '/etc/hosts', '/etc/nsswitch.conf', '/etc/ssl/certs'): if Path(path).exists(): diff --git a/tests/test_tool_sandbox_acceptance.py b/tests/test_tool_sandbox_acceptance.py new file mode 100644 index 00000000..adf4e622 --- /dev/null +++ b/tests/test_tool_sandbox_acceptance.py @@ -0,0 +1,62 @@ +"""Real bubblewrap child enforces network policy without contacting a peer.""" +from types import SimpleNamespace + +import pytest + +from msg.core.errors import Failure +from msg.core.models import NetworkPolicy +from msg.workers.sandbox import BubblewrapRunner + + +@pytest.mark.asyncio +@pytest.mark.parametrize('url', ['http://127.0.0.1:9/', 'http://169.254.169.254/', + 'file:///etc/passwd', 'http://[::1]:9/']) +async def test_real_sandbox_denies_private_or_non_http_target(tmp_path, url): + app = SimpleNamespace(settings=SimpleNamespace(server=SimpleNamespace( + limits=SimpleNamespace(max_request_bytes=1024)))) + tool = SimpleNamespace(executor_key='curl') + with pytest.raises(Failure) as caught: + await BubblewrapRunner(app)(tool, {'url': url}, (NetworkPolicy(schemes=frozenset({'http', 'https'}), hosts=(), + ports=frozenset({9, 80, 443}), methods=frozenset({'GET'}), allow_private=False, + timeout_ms=1000, max_response_bytes=1024, max_redirects=0),), tmp_path) + assert caught.value.code == 'network_policy_denied' + assert not (tmp_path / 'output.bin').exists() + + +@pytest.mark.asyncio +@pytest.mark.parametrize('body', [b'test', b'oversized']) +async def test_real_sandbox_explicit_local_grant_returns_bounded_bytes(tmp_path, body): + import asyncio + + requests = [] + + async def receive(reader, writer): + requests.append(await reader.readuntil(b'\r\n\r\n')) + writer.write(f'HTTP/1.1 200 OK\r\nContent-Length: {len(body)}\r\n'.encode() + + b'Content-Type: text/plain\r\nConnection: close\r\n\r\n' + body) + await writer.drain() + writer.close() + await writer.wait_closed() + + server = await asyncio.start_server(receive, '127.0.0.1', 0) + async with server: + port = server.sockets[0].getsockname()[1] + app = SimpleNamespace(settings=SimpleNamespace(server=SimpleNamespace( + limits=SimpleNamespace(max_request_bytes=1024)))) + policy = NetworkPolicy(schemes=frozenset({'http'}), hosts=('127.0.0.1',), + ports=frozenset({port}), methods=frozenset({'GET'}), allow_private=True, + timeout_ms=3000, max_response_bytes=4, max_redirects=0) + invocation = BubblewrapRunner(app)(SimpleNamespace(executor_key='curl'), + {'url': f'http://127.0.0.1:{port}/fixture'}, (policy,), tmp_path) + if len(body) > policy.max_response_bytes: + with pytest.raises(Failure, match='tool_response_too_large'): + await invocation + assert not (tmp_path / 'output.bin').exists() + assert len(requests) == 1 + return + result = await invocation + assert result.path == tmp_path / 'output.bin' + assert result.path.read_bytes() == b'test' + assert result.metadata['status'] == 200 + assert len(requests) == 1 + assert requests[0].startswith(b'GET /fixture HTTP/1.1\r\n') From 26c2d3c2ba1c6c55c51e3b77f644753262df9f41 Mon Sep 17 00:00:00 2001 From: LIghtJUNction Date: Tue, 29 Sep 2026 18:21:37 +0800 Subject: [PATCH 036/136] fix: reject ambiguous base anchors during unified rebase --- src/msg/core/text_patch.py | 8 +++++- tests/test_patch_rebase_identity.py | 41 +++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+), 1 deletion(-) create mode 100644 tests/test_patch_rebase_identity.py diff --git a/src/msg/core/text_patch.py b/src/msg/core/text_patch.py index 431384ef..73dcaed7 100644 --- a/src/msg/core/text_patch.py +++ b/src/msg/core/text_patch.py @@ -276,7 +276,13 @@ def apply_patch(source, patch, *, base_source=None): result = apply_text_patch(source,patch['exact'],patch['replacement'], patch.get('before',''),patch.get('after','')) elif kind=='unified': - result = _unified(source,_hunks(patch['diff']),rebase=base_source is not None) + hunks = _hunks(patch['diff']) + if base_source is not None: + # Coordinates identify a target only in the original revision. + # A duplicate old block surviving elsewhere is not that target; + # relocation requires a unique anchor in both revisions. + _unified(base_source,hunks,rebase=True) + result = _unified(source,hunks,rebase=base_source is not None) else: result = _structured(source,patch) require(len(result.encode('utf-8'))<=PATCH_LIMIT, 'patch_too_large') diff --git a/tests/test_patch_rebase_identity.py b/tests/test_patch_rebase_identity.py new file mode 100644 index 00000000..cd39b7c0 --- /dev/null +++ b/tests/test_patch_rebase_identity.py @@ -0,0 +1,41 @@ +"""A stale unified patch cannot relocate to a different identical old block.""" +import pytest + +from msg.core.codec import wire +from msg.core.errors import Failure +from msg.core.text_patch import apply_patch +from test_service import call, register +from test_text_patch import state, content + +PATCH = {'kind': 'unified', 'diff': '@@ -1 +1 @@\n-old\n+new\n'} + + +def test_unified_rebase_requires_unique_anchor_in_base_too(): + base = 'old\nseparator\nold\n' + current = 'changed\nseparator\nold\n' + assert apply_patch(base, PATCH) == 'new\nseparator\nold\n' + with pytest.raises(Failure) as exc: + apply_patch(current, PATCH, base_source=base) + assert exc.value.code == 'patch_ambiguous' + + +@pytest.mark.asyncio +async def test_signed_rebase_cannot_edit_surviving_duplicate(installed): + app, _ = installed + key, owner, _ = await register(app, 'rebase-identity') + created = await call(app, 'content.post_create', + {'parent': '/main', 'body': 'old\nseparator\nold\n'}, key=key, subject=owner) + rid = created.resources[0].id + edited = await call(app, 'content.text_patch', + {'id': rid, 'base_revision': created.resources[0].revision, + 'exact': 'old\nseparator', 'replacement': 'changed\nseparator'}, + key=key, subject=owner, expected=((rid, created.data['generation']),)) + assert edited.status == 'ok', wire(edited) + before = await state(app, rid) + result = await call(app, 'content.text_patch', + {'id': rid, 'base_revision': created.resources[0].revision, + 'base_generation': created.data['generation'], 'rebase': True, 'patch': PATCH}, + contract_version=3, key=key, subject=owner, expected=((rid, edited.data['generation']),)) + assert result.status == 'error' and result.error.code == 'patch_ambiguous', wire(result) + assert await state(app, rid) == before + assert (await content(app, rid))[1] == 'changed\nseparator\nold\n' From d7519db76227b9f07299549ac8c383d46cb7926f Mon Sep 17 00:00:00 2001 From: LIghtJUNction Date: Tue, 29 Sep 2026 18:22:15 +0800 Subject: [PATCH 037/136] Register immutable template and tool contracts alongside runtime resources --- docs/registry-template-tool-contracts.md | 27 ++++ src/msg/core/registry.py | 50 ++++++- src/msg/extensions/tools.py | 17 ++- src/msg/plugins/content.py | 20 ++- tests/test_registry_template_tools.py | 169 +++++++++++++++++++++++ 5 files changed, 276 insertions(+), 7 deletions(-) create mode 100644 docs/registry-template-tool-contracts.md create mode 100644 tests/test_registry_template_tools.py diff --git a/docs/registry-template-tool-contracts.md b/docs/registry-template-tool-contracts.md new file mode 100644 index 00000000..606ca771 --- /dev/null +++ b/docs/registry-template-tool-contracts.md @@ -0,0 +1,27 @@ +# Template and tool registration boundary + +The installed Registry indexes `TemplateSpec` and `ToolSpec` by resource ID and +explicit contract version. These maps reject unknown lookups and duplicates, +snapshot nested data, and stop accepting additions after freeze. They do not +change the serialized model fields, published operation versions, or short codes. +Tool operation and input/output schema references are checked at freeze. + +The content plugin registers the shipped template declarations from the existing +bootstrap source. The template version is the DSL header version; renderer version +1 identifies the finite, non-executable renderer. Fields and defaults are frozen +metadata. Creating or updating a user's template still writes Resource/Revision +facts. Runtime parsing and version resolution remain authoritative for those +resources. Registered fields are reused only when the selected revision's digest +exactly matches the installed declaration; later or user-created revisions keep +their own fields, defaults, digest, and template relation. + +The tools plugin registers the existing DNS and HTTP descriptors alongside their +schemas. A runtime tool revision must match the installed executor and schemas. +Its pinned revision and network limits remain resource facts; authorization, +certificate ceilings, deployment limits, and worker rechecks still apply. Registry +registration supplies no permission and does not execute resource content. + +Focused coverage is in `tests/test_registry_template_tools.py`, alongside the +existing tool invocation/worker, schema ownership, and Registry reference tests. +This boundary does not assert completion of the broader chapter 15 acceptance +matrix or production verification. diff --git a/src/msg/core/registry.py b/src/msg/core/registry.py index 0a05cb8f..1109987f 100644 --- a/src/msg/core/registry.py +++ b/src/msg/core/registry.py @@ -5,7 +5,7 @@ from referencing.exceptions import Unresolvable from msg.core.schema_policy import local_validator -from msg.core.codec import digest,wire +from msg.core.codec import decode,digest,wire from msg.core.errors import Failure,require @@ -27,6 +27,8 @@ def __init__(self): self._types={} self._capabilities={} self._operations={} + self._templates={} + self._tools={} self._schemas={} self._validators={} self._plugins={} @@ -50,6 +52,35 @@ def add_operation(self,spec): require(spec.entries and spec.entries<=frozenset({'local_admin','network','worker'}),'invalid_entries') self._insert(self._operations,spec,'operation') + def _insert_resource_spec(self, collection, spec, version, kind): + # Registration versions describe installed contracts; resource revisions + # remain immutable runtime facts and are never added to this registry. + require(not self._frozen, 'registry_frozen') + require(type(version) is int and version >= 1 and spec.resource.id + and '*' not in spec.resource.id, 'invalid_registry_name') + key=(spec.resource.id,version) + require(key not in collection, 'duplicate_'+kind) + collection[key]=decode(type(spec),wire(spec)) + + def add_template(self,spec,version=1): + from msg.core.template_dsl import _check + from msg.core.codec import loads + require(spec.renderer_version == 1, 'unknown_template_renderer') + names=[field.name for field in spec.fields] + require(len(names)==len(set(names)), 'duplicate_field') + for field in spec.fields: + require(field.type in {'str','text','int','bool','enum','ref','file'}, + 'unknown_field_type') + require(not field.required or field.default_json is None, + 'required_field_has_default') + if field.default_json is not None: + _check(field,loads(field.default_json)) + self._insert_resource_spec(self._templates,spec,version,'template') + + def add_tool(self,spec,version=1): + require(bool(spec.executor_key), 'untrusted_tool_executor') + self._insert_resource_spec(self._tools,spec,version,'tool') + def add_schema(self,ref,schema): require(not self._frozen and ref.id not in self._schemas,'schema_conflict') # Keep one owned snapshot for both validation and publication. Caller @@ -112,6 +143,11 @@ def freeze(self): cap.constraints_schema.id in self._schemas,'missing_schema') require(cap.scope_types<=set(n for n,v in self._types),'unknown_scope_type') require(cap.operations<=operation_ids,'unknown_capability_operation') + for tool in self._tools.values(): + operation=tool.operation if '@' in tool.operation else tool.operation+'@1' + require(operation in operation_ids,'unknown_tool_operation') + require(tool.input_schema.id in self._schemas and + tool.output_schema.id in self._schemas,'missing_schema') self._frozen=True @property @@ -133,6 +169,18 @@ def capability(self,name,version=1): def resource_type(self,name,version=1): return self._get(self._types,name,version,'resource_type') + def template(self,resource_id,version=1): + return self._get(self._templates,resource_id,version,'template') + + def tool(self,resource_id,version=1): + return self._get(self._tools,resource_id,version,'tool') + + def templates(self): + return tuple(self._templates[k] for k in sorted(self._templates)) + + def tools(self): + return tuple(self._tools[k] for k in sorted(self._tools)) + def capabilities(self): return tuple(self._capabilities[k] for k in sorted(self._capabilities)) diff --git a/src/msg/extensions/tools.py b/src/msg/extensions/tools.py index 56c27715..0d1d9256 100644 --- a/src/msg/extensions/tools.py +++ b/src/msg/extensions/tools.py @@ -54,10 +54,14 @@ async def read_tool(app,tx,rid,revision=None): require(resource.type=='tool' and resource.parent==TOOLS_SPACE,'not_a_tool') rev=await tx.revision(ResourceRef(id=rid,revision=revision)) value=loads(await app.contents.read_bytes(rev.content,limit=65536)) - require(value['executor_key'] in {'dns','curl'} and value['tool_id']==rid,'untrusted_tool_executor') - return ToolSpec(resource=ResourceRef(id=rid,revision=rev.id),operation='tool.run', - input_schema=decode(ResourceRef,value['input_schema']),output_schema=decode(ResourceRef,value['output_schema']), - executor_key=value['executor_key'],network=decode(NetworkPolicy,value['network'])) + registered=app.registry.tool(rid,value['version']) + require(value['tool_id']==rid and value['executor_key']==registered.executor_key, + 'untrusted_tool_executor') + require(decode(ResourceRef,value['input_schema'])==registered.input_schema and + decode(ResourceRef,value['output_schema'])==registered.output_schema, + 'tool_schema_mismatch') + return replace(registered,resource=ResourceRef(id=rid,revision=rev.id), + network=decode(NetworkPolicy,value['network'])) async def tool_policies(app,principal,tool,args,tx): @@ -109,6 +113,11 @@ def register(app,op): for name,schema in (('dns',DNS_INPUT),('curl',CURL_INPUT)): app.registry.add_schema(ResourceRef(id='schema:tool.'+name+':input'),schema) app.registry.add_schema(ResourceRef(id='schema:tool.'+name+':output'),TOOL_OUTPUT) + value=descriptor(name) + app.registry.add_tool(ToolSpec(resource=ResourceRef(id=value['tool_id']), + operation=value['operation'],input_schema=decode(ResourceRef,value['input_schema']), + output_schema=decode(ResourceRef,value['output_schema']),executor_key=value['executor_key'], + network=decode(NetworkPolicy,value['network'])),version=value['version']) @op('tool.run',obj({'id':IDENTIFIER,'revision':IDENTIFIER,'arguments':{'type':'object'}},('id','arguments')),effect='external') async def invoke(ctx,request,tx): diff --git a/src/msg/plugins/content.py b/src/msg/plugins/content.py index 0d72d725..052ea572 100644 --- a/src/msg/plugins/content.py +++ b/src/msg/plugins/content.py @@ -6,7 +6,7 @@ from msg.constants import ROOT_SPACE,ROOT_SUBJECT from msg.core.codec import canonical,decode,loads,wire,unb64,digest,parse_time from msg.core.errors import Failure,require -from msg.core.models import HandlerOutput,ResourceRef,Relation,EffectJob,BlobRef,Event +from msg.core.models import HandlerOutput,ResourceRef,Relation,EffectJob,BlobRef,Event,TemplateSpec from msg.core.tags import normalize_tags from msg.core.text_patch import (PATCH_LIMIT,PATCH_CONTEXT_LIMIT,PATCH_CANDIDATE_LIMIT, PATCH_SCHEMA,apply_text_patch,apply_patch,validate_patch) @@ -85,7 +85,16 @@ async def template_content(app,ctx,request,tx,parent,template,values): rev=await tx.revision(ref) source=(await app.contents.read_bytes(rev.content)).decode('utf-8') parsed=parse_template(source) - normalized=normalize_values(parsed,values or {}) + # Installed defaults have a frozen field contract. User-owned templates and + # later revisions remain resource data, parsed under the same finite DSL. + try: + registered=app.registry.template(rid,parsed.version) + except Failure as exc: + if exc.code!='unknown_template': + raise + registered=None + fields=registered if registered is not None and registered.digest==rev.content.digest else parsed + normalized=normalize_values(fields,values or {}) content={'template_id':rid,'template_version':parsed.version,'template_digest':rev.content.digest,'values':normalized} return canonical(content),'application/json',content,Relation(type='template',target=ref) @@ -237,6 +246,13 @@ async def prepare_text_patch(app,ctx,request,tx,args, *, post_only=False): def install(app): + from msg.bootstrap import manifest + for name,source in manifest()['templates'].items(): + parsed=parse_template(source) + require(name==parsed.name,'template_name_mismatch') + app.registry.add_template(TemplateSpec(resource=ResourceRef(id='tpl_'+name), + digest='sha256:'+sha256(source.encode()).hexdigest(),fields=parsed.fields, + renderer_version=1),version=parsed.version) op,finish=registration(app,'content',('identity',)) post_fields={'parent':IDENTIFIER,'name':STRING,'body':STRING,'template':{'anyOf':[STRING,obj({'id':IDENTIFIER,'version':INTEGER},('id',))]}, 'values':{'type':'object'},'source':REF,'content_created_at':STRING,'resource_id':IDENTIFIER,'revision_id':IDENTIFIER,'content_signature':SIGNATURE} diff --git a/tests/test_registry_template_tools.py b/tests/test_registry_template_tools.py new file mode 100644 index 00000000..29b11606 --- /dev/null +++ b/tests/test_registry_template_tools.py @@ -0,0 +1,169 @@ +"""Installed contract metadata stays separate from resource revisions and values.""" +from dataclasses import replace +from hashlib import sha256 + +import pytest + +from msg.bootstrap import manifest +from msg.core.codec import decode, wire +from msg.core.errors import Failure +from msg.core.models import FieldSpec, PluginManifest, ResourceRef, TemplateSpec +from msg.core.registry import Registry +from msg.core.template_dsl import normalize_values, parse_template +from msg.extensions.tools import read_tool + + +def template(): + return TemplateSpec(resource=ResourceRef(id='tpl_example'),digest='sha256:'+'0'*64, + fields=(FieldSpec(name='title',type='str',required=False,default_json=b'"hello"'),), + renderer_version=1) + + +def test_templates_are_versioned_owned_specs_with_defaults_and_strict_lookup(): + registry=Registry() + first=template() + registry.add_template(first) + second=replace(first,fields=(replace(first.fields[0],default_json=b'"new"'),)) + registry.add_template(second,version=2) + assert registry.template('tpl_example') == first + assert registry.template('tpl_example',2) == second + assert registry.templates() == (first,second) + assert normalize_values(registry.template('tpl_example'),{}) == {'title':'hello'} + with pytest.raises(Failure,match='^unknown_template$'): + registry.template('unregistered') + with pytest.raises(Failure,match='^duplicate_template$'): + registry.add_template(second) + with pytest.raises(Failure,match='^invalid_registry_name$'): + registry.add_template(first,version=0) + + +def test_template_registration_snapshots_nested_fields_and_validates_defaults(): + registry=Registry() + constraints={'label':['one']} + spec=replace(template(),fields=(replace(template().fields[0],constraints=constraints),)) + registry.add_template(spec) + constraints['label'].append('two') + assert registry.template('tpl_example').fields[0].constraints['label']==('one',) + with pytest.raises(Failure,match='^template_type_error'): + registry.add_template(replace(spec,fields=(replace(spec.fields[0],default_json=b'42'),)),2) + with pytest.raises(Failure,match='^unknown_template_renderer$'): + registry.add_template(replace(spec,renderer_version=2),2) + + +async def test_installed_template_and_tool_contracts_match_published_resources(installed): + app,_=installed + for name,source in manifest()['templates'].items(): + parsed=parse_template(source) + registered=app.registry.template('tpl_'+name,parsed.version) + assert registered.fields==parsed.fields + assert registered.digest=='sha256:'+sha256(source.encode()).hexdigest() + assert registered.resource.revision is None + assert {spec.resource.id for spec in app.registry.tools()}=={'tool_dns','tool_curl'} + async with app.metadata.transaction(write=False) as tx: + for registered in app.registry.tools(): + actual=await read_tool(app,tx,registered.resource.id) + assert actual.resource.revision is not None + assert replace(actual,resource=registered.resource)==registered + with pytest.raises(Failure,match='^registry_frozen$'): + app.registry.add_template(template()) + with pytest.raises(Failure,match='^registry_frozen$'): + app.registry.add_tool(app.registry.tool('tool_dns'),2) + with pytest.raises(Failure,match='^unknown_tool$'): + app.registry.tool('tool_dns',2) + + +async def test_tool_registry_rejects_conflicts_missing_schemas_and_operations(installed): + app,_=installed + original=app.registry.tool('tool_dns') + # Build a fresh registration graph from the installed public contracts. + registry=Registry() + registry.add(PluginManifest(name='identity',version='1',dependencies=(), + resource_types=(),capabilities=(),operations=(),migrations=())) + registry.add_tool(original) + with pytest.raises(Failure,match='^duplicate_tool$'): + registry.add_tool(original) + with pytest.raises(Failure,match='^unknown_tool_operation$'): + registry.freeze() + operation=app.registry.operation('tool.run') + registry.add_operation(operation) + for ref in (operation.input_schema,operation.output_schema): + registry.add_schema(ref,app.registry.schema(ref)) + with pytest.raises(Failure,match='^missing_schema$'): + registry.freeze() + for ref in (original.input_schema,original.output_schema): + registry.add_schema(ref,app.registry.schema(ref)) + registry.freeze() + assert registry.tool('tool_dns')==decode(type(original),wire(original)) + + +async def test_shipped_template_post_keeps_signed_content_contract(installed): + from test_service import register,call + app,_=installed + key,uid,_=await register(app,'registry-template-user') + result=await call(app,'content.post_create', + {'parent':'/tmp','template':'message','values':{'body':'hello'}},key=key,subject=uid) + assert result.status=='ok',wire(result) + async with app.metadata.transaction(write=False) as tx: + revision=await tx.revision(result.resources[0]) + from msg.core.codec import loads + content=loads(await app.contents.read_bytes(revision.content)) + assert content=={'template_id':'tpl_message','template_version':1, + 'template_digest':app.registry.template('tpl_message').digest,'values':{'body':'hello'}} + + +@pytest.mark.parametrize('mutation,error',[ + ({'executor_key':'uninstalled'},'untrusted_tool_executor'), + ({'input_schema':{'id':'schema:tool.curl:input'}},'tool_schema_mismatch'), + ({'version':99},'unknown_tool'), + ({},None), +]) +async def test_tool_resource_policy_is_preserved_but_cannot_replace_registered_code(installed,mutation,error): + from types import SimpleNamespace + from msg.constants import TOOLS_SPACE + from msg.core.codec import canonical + from msg.extensions.tools import descriptor + app,_=installed + value=descriptor('dns') + value['network']['timeout_ms']=1234 + value.update(mutation) + async def resource(rid): + return SimpleNamespace(type='tool',parent=TOOLS_SPACE) + async def revision(ref): + assert ref.revision=='pinned' + return SimpleNamespace(id='pinned',content=None) + async def read_bytes(content,limit): + return canonical(value) + context=SimpleNamespace(registry=app.registry,contents=SimpleNamespace(read_bytes=read_bytes)) + tx=SimpleNamespace(resource=resource,revision=revision) + if error: + with pytest.raises(Failure,match='^'+error+'$'): + await read_tool(context,tx,'tool_dns','pinned') + else: + actual=await read_tool(context,tx,'tool_dns','pinned') + assert actual.resource.revision=='pinned' + assert actual.network.timeout_ms==1234 + assert app.registry.tool('tool_dns').network.timeout_ms==30000 + + +async def test_user_template_versions_remain_resources_outside_frozen_registry(installed): + from test_service import register,call + app,_=installed + key,uid,_=await register(app,'registry-user-template') + before=app.registry.templates() + created=await call(app,'content.template_put', + {'parent':'/templates','source':'user-notice@7\nbody:text!\npriority:int=2\n'}, + key=key,subject=uid) + assert created.status=='ok',wire(created) + rid=created.resources[0].id + result=await call(app,'content.post_create',{'parent':'/tmp', + 'template':{'id':rid,'version':7},'values':{'body':'user data'}},key=key,subject=uid) + assert result.status=='ok',wire(result) + async with app.metadata.transaction(write=False) as tx: + revision=await tx.revision(result.resources[0]) + from msg.core.codec import loads + content=loads(await app.contents.read_bytes(revision.content)) + assert content['template_id']==rid and content['template_version']==7 + assert content['values']=={'body':'user data','priority':2} + assert app.registry.templates()==before + with pytest.raises(Failure,match='^unknown_template$'): + app.registry.template(rid,7) From 60f0a551865bf948e67e593750c927575f4224e0 Mon Sep 17 00:00:00 2001 From: LIghtJUNction Date: Tue, 29 Sep 2026 18:23:09 +0800 Subject: [PATCH 038/136] ci: install bubblewrap for real tool isolation acceptance --- .github/workflows/ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 22b3ed01..4b322861 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -60,10 +60,11 @@ jobs: - name: Install required recovery and ingress test tools run: | sudo apt-get update -qq - sudo apt-get install -y -qq age nginx openssl git-lfs + sudo apt-get install -y -qq age nginx openssl git-lfs bubblewrap age --version nginx -v openssl version + bwrap --version - name: Install the complete dependency set run: python -m pip install -e '.[dev]' - name: Compile all new modules From 9178d802b7f4cd98c8e703a1d0384950c9594add Mon Sep 17 00:00:00 2001 From: LIghtJUNction Date: Tue, 29 Sep 2026 18:23:43 +0800 Subject: [PATCH 039/136] fix: verify reused blob bytes before publishing revisions --- src/msg/plugins/common.py | 13 ++++++++++++ tests/test_file_copy_integrity.py | 34 +++++++++++++++++++++++++++++++ 2 files changed, 47 insertions(+) create mode 100644 tests/test_file_copy_integrity.py diff --git a/src/msg/plugins/common.py b/src/msg/plugins/common.py index 37f99924..b5c186ed 100644 --- a/src/msg/plugins/common.py +++ b/src/msg/plugins/common.py @@ -1,6 +1,7 @@ """Explicit resource use cases shared by content and extensions.""" from __future__ import annotations import re +import hashlib from dataclasses import replace from uuid import uuid4 from msg.constants import * @@ -121,6 +122,18 @@ async def revise_resource(app,ctx,request,tx,resource,body,media_type='text/mark require(signature is None or revision_id is not None,'revision_id_required') if isinstance(body,BlobRef): blob=body + # A retained index/reference is not evidence the payload survived. + # Verify copies and reused blobs before publishing another Revision. + hasher=hashlib.sha256() + size=0 + try: + async for chunk in app.contents.read(blob): + size+=len(chunk) + hasher.update(chunk) + except FileNotFoundError as exc: + raise Failure('content_missing') from exc + require(size==blob.size and 'sha256:'+hasher.hexdigest()==blob.digest, + 'content_digest_mismatch') else: blob=await app.contents.put_bytes(body.encode('utf-8') if isinstance(body,str) else body,media_type) for relation in relations: diff --git a/tests/test_file_copy_integrity.py b/tests/test_file_copy_integrity.py new file mode 100644 index 00000000..2ed2aed2 --- /dev/null +++ b/tests/test_file_copy_integrity.py @@ -0,0 +1,34 @@ +"""Copying a retained reference must verify its payload before publication.""" +import pytest +from msg.core.codec import b64, wire +from msg.core.models import ResourceRef +from test_service import call, register + + +@pytest.mark.asyncio +@pytest.mark.parametrize('damage', ['missing', 'corrupt']) +async def test_binary_copy_rejects_unavailable_source_without_publication(installed, damage): + app, _ = installed + key, owner, _ = await register(app, 'copy-integrity') + parent = '/@copy-integrity/files' + created = await call(app, 'content.file_put', + {'parent': parent, 'name': 'source.bin', 'data': b64(b'original')}, + key=key, subject=owner) + assert created.status == 'ok', wire(created) + async with app.metadata.transaction(write=False) as tx: + revision = await tx.revision(ResourceRef(id=created.resources[0].id)) + before = tuple(tx.one('SELECT COUNT(*) FROM '+table)[0] + for table in ('resources','revisions','events')) + path = app.contents.binary / revision.content.digest.removeprefix('sha256:') + if damage == 'missing': + path.unlink() + else: + path.write_bytes(b'tampered') + copied = await call(app, 'content.file_put', + {'parent': parent, 'name': 'copy.bin', 'source': {'id': created.resources[0].id}}, + key=key, subject=owner) + assert copied.status == 'error', wire(copied) + assert copied.error.code == ('content_missing' if damage == 'missing' else 'content_digest_mismatch') + async with app.metadata.transaction(write=False) as tx: + assert tuple(tx.one('SELECT COUNT(*) FROM '+table)[0] + for table in ('resources','revisions','events')) == before From d271103f4660fe85fb0ef127b3dd42e5d9573074 Mon Sep 17 00:00:00 2001 From: LIghtJUNction Date: Tue, 29 Sep 2026 18:23:49 +0800 Subject: [PATCH 040/136] feat(bootstrap): support pinned release rule retirement preserving history --- src/msg/application.py | 3 +- src/msg/bootstrap.py | 69 +++++++++- src/msg/core/registry.py | 6 +- tests/test_system_rule_retirement.py | 190 +++++++++++++++++++++++++++ 4 files changed, 260 insertions(+), 8 deletions(-) create mode 100644 tests/test_system_rule_retirement.py diff --git a/src/msg/application.py b/src/msg/application.py index 90a8dec9..f9545101 100644 --- a/src/msg/application.py +++ b/src/msg/application.py @@ -105,7 +105,8 @@ async def load(self): # Recheck under the writer lock; loading a restore must not # mutate release resources before recovery has been accepted. if not quarantine_active(tx): - await sync_system_sources(tx,self.contents,self.clock(),namespace_root=self.namespace_root) + await sync_system_sources(tx,self.contents,self.clock(),namespace_root=self.namespace_root, + registry=self.registry) else: quarantined=True self.authenticator=AuthenticationService(self.registry,self.certificates,self.settings.service_url,self.clock, diff --git a/src/msg/bootstrap.py b/src/msg/bootstrap.py index 5e120956..9417582e 100644 --- a/src/msg/bootstrap.py +++ b/src/msg/bootstrap.py @@ -33,6 +33,12 @@ # Release maintainers declare a relocation in both maps in the same change. SOURCE_PATH_OVERRIDES={} SOURCE_MIGRATIONS={} +# Release-only pins: rule_id -> {source_path: relative .md path, version: int, +# digest: sha256:}. Remove active links/operation dependencies in the same +# release; keep RULE_SPECS identity and this pin so upgrades preserve history. +# Retirement archives existing resources; fresh installs invent no old revision. +# No current rule is retired. A pin mismatch requires an explicit staged upgrade. +SOURCE_RETIREMENTS={} def system_source_root(): @@ -42,15 +48,36 @@ def system_source_root(): async def sync_system_sources(tx,contents,now, *, source_root=None,namespace_root=ROOT_SPACE, - source_paths=None,migrations=None): + source_paths=None,migrations=None,retirements=None,registry=None): """Sync each release-owned document independently; never delete a source implicitly.""" root=Path(source_root or system_source_root()) source_paths=dict(SOURCE_PATH_OVERRIDES if source_paths is None else source_paths) migrations=dict(SOURCE_MIGRATIONS if migrations is None else migrations) require(set(source_paths)<=RULE_PATHS.keys() and set(migrations)<=RULE_PATHS.keys(), 'unknown_rule_id') + retirements=dict(SOURCE_RETIREMENTS if retirements is None else retirements) + require(set(retirements)<=RULE_PATHS.keys()-{'msg.bootstrap','msg.rules.index'}, + 'system_source_invalid_retirement') + require(not set(retirements)&(source_paths.keys()|migrations.keys()), + 'system_source_invalid_retirement') + for rule_id,pin in retirements.items(): + require(isinstance(pin,dict) and set(pin)=={'source_path','version','digest'} and + isinstance(pin['source_path'],str) and + pin['source_path']==Path(pin['source_path']).as_posix() and + not Path(pin['source_path']).is_absolute() and + '..' not in Path(pin['source_path']).parts and + pin['source_path'].endswith('.md') and + type(pin['version']) is int and pin['version']>0 and + isinstance(pin['digest'],str) and + re.fullmatch(r'sha256:[0-9a-f]{64}',pin['digest']) is not None, + 'system_source_invalid_retirement',rule_id) + if retirements: + require(registry is not None and all(not set(spec.requires_rules)&retirements.keys() + for spec in registry.operations()),'system_source_retirement_referenced') specs=[] for rule_id,rid,parent,name,default_path,max_bytes in RULE_SPECS: + if rule_id in retirements: + continue relative=source_paths.get(rule_id,default_path) require(isinstance(relative,str) and relative.endswith('.md') and relative==Path(relative).as_posix() and @@ -72,6 +99,7 @@ async def sync_system_sources(tx,contents,now, *, source_root=None,namespace_roo rule_id=match.group(1) require(rule_id in RULE_PATHS,'unknown_rule_id',relative) require(rule_id not in observed,'duplicate_rule_id',rule_id) + require(rule_id not in retirements,'system_source_retirement_source_present',rule_id) observed[rule_id]=relative require(not expected-actual,'system_source_deleted_requires_migration', details={'paths':sorted(expected-actual)}) @@ -101,10 +129,35 @@ async def sync_system_sources(tx,contents,now, *, source_root=None,namespace_roo 'dangling_requires_rules',rule_id) require(len(required)<=1,'duplicate_requires_rules',rule_id) dependencies=tuple(part.strip() for part in required[0].split(',')) if required else () + require(not set(dependencies)&retirements.keys() and not any( + re.search(re.escape(RULE_PATHS[retired])+r'(?=[/#?\s)\]<>]|$)',source_text) + for retired in retirements),'system_source_retirement_referenced') require(all(dep in RULE_PATHS and dep!=rule_id for dep in dependencies) and len(set(dependencies))==len(dependencies),'dangling_requires_rules',rule_id) prepared.append((relative,rid,parent,name,rule_id,version,raw,change_note)) - require(rule_ids==RULE_PATHS.keys(),'system_source_inventory_mismatch') + require(rule_ids==RULE_PATHS.keys()-retirements.keys(),'system_source_inventory_mismatch') + retired=[] + for rule_id,pin in retirements.items(): + spec=next(spec for spec in RULE_SPECS if spec[0]==rule_id) + rid=spec[1] + old=tx.one('''SELECT source_path,source_version,source_digest,revision_id,source_kind,rule_id + FROM system_sources WHERE resource_id=?''',(rid,)) + if old is None: + require(tx.one('SELECT id FROM resources WHERE id=?',(rid,)) is None, + 'system_source_retirement_mismatch',rule_id) + continue # A fresh install has no historical revision to archive. + require(tuple(old[:3])==('docs/system/'+pin['source_path'],pin['version'],pin['digest']) + and old[4] in {'release','release_retired'} and old[5]==rule_id, + 'system_source_retirement_mismatch',rule_id) + resource=await tx.resource(rid) + require((resource.type,resource.parent,resource.name,resource.owner,resource.group, + resource.mode)==('file',spec[2],spec[3],ROOT_SUBJECT,PUBLIC_GROUP,0o444), + 'bootstrap_drift',rid) + require(resource.revision==old[3],'system_source_pointer_drift') + require(resource.state in {'active','archived'} and + (old[4]!='release_retired' or resource.state=='archived'), + 'system_source_retirement_mismatch',rule_id) + retired.append((resource,old[0])) for rule_id,move in migrations.items(): require(isinstance(move,(list,tuple)) and len(move)==2 and all(isinstance(p,str) for p in move) and @@ -128,9 +181,10 @@ async def sync_system_sources(tx,contents,now, *, source_root=None,namespace_roo dict(id=rid,type='file',name=name,parent=parent, owner=ROOT_SUBJECT,group=PUBLIC_GROUP,mode='0444'),now) source_digest=digest(raw) - old=tx.one('''SELECT source_path,rule_id,source_version,source_digest,revision_id + old=tx.one('''SELECT source_path,rule_id,source_version,source_digest,revision_id,source_kind FROM system_sources WHERE resource_id=?''',(rid,)) if old is not None: + require(old[5]!='release_retired','system_source_retirement_required',rule_id) require(old[1]==rule_id,'system_source_identity_drift') if old[0]!=source_path: require(migrations.get(rule_id)==(old[0].removeprefix('docs/system/'),relative), @@ -167,6 +221,13 @@ async def sync_system_sources(tx,contents,now, *, source_root=None,namespace_roo tx.execute('''UPDATE system_sources SET source_path=?,source_kind='release',source_version=?, source_digest=?,revision_id=? WHERE resource_id=?''', (source_path,version,source_digest,revision_id,rid),write=True) + for resource,source_path in retired: + desired.add(source_path) + if resource.state!='archived': + await tx.replace(replace(resource,state='archived',generation=resource.generation+1, + modified_at=now,modified_by=ROOT_SUBJECT),resource.generation) + tx.execute("UPDATE system_sources SET source_kind='release_retired' WHERE resource_id=?", + (resource.id,),write=True) missing={path for (path,) in tx.rows('SELECT source_path FROM system_sources')} - desired require(not missing,'system_source_deleted_requires_migration',details={'paths':sorted(missing)}) @@ -294,7 +355,7 @@ async def bootstrap(store,contents,registry,now, *, selftest_run_id=None): for key,schema in registry._schemas.items(): await seed_resource(tx,contents,dict(id=key,type='file',name=key.removeprefix('schema:'),parent='t_schema', owner=ROOT_SUBJECT,group=PUBLIC_GROUP,mode='0444'),now,canonical(schema),'application/json') - await sync_system_sources(tx,contents,now,namespace_root=namespace_root) + await sync_system_sources(tx,contents,now,namespace_root=namespace_root,registry=registry) tx.set_setting('bootstrap',{'version':definition['version'],'digest':digest(definition)}) diff --git a/src/msg/core/registry.py b/src/msg/core/registry.py index 1109987f..2c649b0f 100644 --- a/src/msg/core/registry.py +++ b/src/msg/core/registry.py @@ -124,13 +124,13 @@ def features(self,plugin_name): if row['feature_id'] in claims) def freeze(self): - from msg.bootstrap import feature_manifest + from msg.bootstrap import RULE_PATHS,SOURCE_RETIREMENTS,feature_manifest feature_manifest() - from msg.bootstrap import RULE_PATHS require('identity' in self._plugins,'identity_plugin_required') for spec in self._operations.values(): require(spec.requires_rules and len(set(spec.requires_rules))==len(spec.requires_rules) - and all(rule_id in RULE_PATHS for rule_id in spec.requires_rules), + and all(rule_id in RULE_PATHS and rule_id not in SOURCE_RETIREMENTS + for rule_id in spec.requires_rules), 'dangling_requires_rules') require(spec.input_schema.id in self._schemas and spec.output_schema.id in self._schemas,'missing_schema') operation_ids={f'{s.name}@{s.version}' for s in self._operations.values()} diff --git a/tests/test_system_rule_retirement.py b/tests/test_system_rule_retirement.py new file mode 100644 index 00000000..b067fb44 --- /dev/null +++ b/tests/test_system_rule_retirement.py @@ -0,0 +1,190 @@ +"""Release-only retirement preserves history and refuses dangling dependencies.""" +import shutil +import re +from dataclasses import replace + +import pytest + +from msg.bootstrap import sync_system_sources, system_source_root +from msg.core.codec import wire +from msg.core.errors import Failure +from msg.core.models import ResourceRef +from test_service import NOW, call, register + + +async def retirement(app, tmp_path): + source = tmp_path / 'retirement-source' + shutil.copytree(system_source_root(), source) + (source / 'rules/recovery.md').unlink() + for path in source.rglob('*.md'): + text = path.read_text() + if '/_rules/recovery' in text: + text = re.sub(r'version: (\d+)', lambda match: 'version: ' + str(int(match[1])+1), + text, count=1) + path.write_text('\n'.join(line for line in text.splitlines() + if '/_rules/recovery' not in line) + '\n') + async with app.metadata.transaction(write=False) as tx: + row = tx.one('SELECT source_path,source_version,source_digest FROM system_sources ' + 'WHERE rule_id=?', ('msg.recovery',)) + return source, {'msg.recovery': {'source_path': row[0].removeprefix('docs/system/'), + 'version': row[1], 'digest': row[2]}} + + +@pytest.mark.asyncio +async def test_declared_retirement_preserves_history_is_idempotent_and_read_only(installed, tmp_path, monkeypatch): + app, _ = installed + source, declaration = await retirement(app, tmp_path) + async with app.metadata.transaction(write=False) as tx: + before = await tx.resource('r_rule_recovery') + revision = await tx.revision(ResourceRef(id=before.id, revision=before.revision)) + original = await app.contents.read_bytes(revision.content) + for _ in range(2): + async with app.metadata.transaction(write=True) as tx: + await sync_system_sources(tx, app.contents, NOW, source_root=source, + retirements=declaration, registry=app.registry) + async with app.metadata.transaction(write=False) as tx: + after = await tx.resource(before.id) + assert after.state == 'archived' + assert (after.revision, after.generation) == (before.revision, before.generation + 1) + assert wire(await tx.revision(ResourceRef(id=before.id, revision=before.revision))) == wire(revision) + assert await app.contents.read_bytes(revision.content) == original + import msg.bootstrap as bootstrap + from msg.application import Application + monkeypatch.setattr(bootstrap, 'SOURCE_RETIREMENTS', declaration) + monkeypatch.setattr(bootstrap, 'system_source_root', lambda: source) + reloaded = Application(app.settings, clock=lambda: NOW) + try: + await reloaded.load() + async with reloaded.metadata.transaction(write=False) as tx: + assert (await tx.resource(before.id)).state == 'archived' + finally: + await reloaded.close() + read = await call(app, 'discovery.get', {'id': before.id, 'revision': before.revision}) + assert read.status == 'ok', wire(read) + assert read.data['content'].encode() == original + key, subject, _ = await register(app, 'retirement-writer') + for operation, args in ( + ('content.chmod', {'id': before.id, 'mode': '0777'}), + ('content.post_edit', {'id': before.id, 'expected_revision': before.revision, 'body': 'replace'}), + ): + result = await call(app, operation, args, key=key, subject=subject) + assert result.error.code == 'system_managed_resource', wire(result) + + +@pytest.mark.asyncio +@pytest.mark.parametrize('mismatch', ['version', 'digest', 'source_path']) +async def test_retirement_must_pin_current_release_metadata(installed, tmp_path, mismatch): + app, _ = installed + source, declaration = await retirement(app, tmp_path) + declaration['msg.recovery'][mismatch] = {'version': 999, 'digest': 'sha256:' + '0' * 64, + 'source_path': 'rules/wrong.md'}[mismatch] + with pytest.raises(Failure, match='^system_source_retirement_mismatch$'): + async with app.metadata.transaction(write=True) as tx: + await sync_system_sources(tx, app.contents, NOW, source_root=source, + retirements=declaration, registry=app.registry) + async with app.metadata.transaction(write=False) as tx: + assert (await tx.resource('r_rule_recovery')).state == 'active' + + +@pytest.mark.asyncio +@pytest.mark.parametrize('dependency', ['operation', 'index', 'source', 'registry_missing']) +async def test_retirement_rejects_active_dependencies(installed, tmp_path, dependency): + app, _ = installed + source, declaration = await retirement(app, tmp_path) + registry = app.registry + if dependency == 'operation': + key = ('discovery.get', 1) + registry._operations[key] = replace(registry.operation(*key), requires_rules=('msg.recovery',)) + elif dependency == 'index': + with (source / 'rules/_index.md').open('a') as handle: + handle.write('\n[Recovery](/_rules/recovery)\n') + elif dependency == 'source': + with (source / 'rules/security.md').open('a') as handle: + handle.write('\n\n') + else: + registry = None + with pytest.raises(Failure, match='^system_source_retirement_referenced$'): + async with app.metadata.transaction(write=True) as tx: + await sync_system_sources(tx, app.contents, NOW, source_root=source, + retirements=declaration, registry=registry) + async with app.metadata.transaction(write=False) as tx: + assert (await tx.resource('r_rule_recovery')).state == 'active' + + +@pytest.mark.asyncio +async def test_fresh_install_does_not_invent_retired_history(installed, tmp_path, pg_dsn, monkeypatch): + import msg.bootstrap as bootstrap + from msg.admin.root import _provision + from msg.application import Application + from msg.config import write_example + + source, declaration = await retirement(installed[0], tmp_path) + monkeypatch.setattr(bootstrap, 'SOURCE_RETIREMENTS', declaration) + monkeypatch.setattr(bootstrap, 'system_source_root', lambda: source) + app = Application(write_example(tmp_path / 'fresh-config', tmp_path / 'fresh-data', + postgres_dsn=pg_dsn), clock=lambda: NOW) + try: + await _provision(app, 'test-retirement-passphrase') + async with app.metadata.transaction(write=False) as tx: + assert tx.one('SELECT id FROM resources WHERE id=?', ('r_rule_recovery',)) is None + assert tx.one('SELECT resource_id FROM system_sources WHERE rule_id=?', ('msg.recovery',)) is None + assert tx.one('SELECT id FROM resources WHERE id=?', ('r_rule_security',)) is not None + finally: + await app.close() + + +def test_registry_freeze_rejects_dependency_on_release_retired_rule(monkeypatch): + import msg.bootstrap as bootstrap + from test_registry_operation_rules import assemble + + monkeypatch.setattr(bootstrap, 'SOURCE_RETIREMENTS', {'msg.recovery': {}}) + registry = assemble('recovery.custom', requires_rules=('msg.recovery',)) + with pytest.raises(Failure, match='^dangling_requires_rules$'): + registry.freeze() + assert not registry.frozen + + +@pytest.mark.asyncio +async def test_retirement_cannot_implicitly_reactivate_archived_source(installed, tmp_path): + app, _ = installed + source, declaration = await retirement(app, tmp_path) + async with app.metadata.transaction(write=True) as tx: + await sync_system_sources(tx, app.contents, NOW, source_root=source, + retirements=declaration, registry=app.registry) + shutil.copyfile(system_source_root() / 'rules/recovery.md', source / 'rules/recovery.md') + with pytest.raises(Failure, match='^system_source_retirement_required$'): + async with app.metadata.transaction(write=True) as tx: + await sync_system_sources(tx, app.contents, NOW, source_root=source, + registry=app.registry) + async with app.metadata.transaction(write=False) as tx: + assert (await tx.resource('r_rule_recovery')).state == 'archived' + + +@pytest.mark.asyncio +@pytest.mark.parametrize(('change', 'code'), [ + ('unknown', 'system_source_invalid_retirement'), + ('mandatory', 'system_source_invalid_retirement'), + ('malformed', 'system_source_invalid_retirement'), + ('source_present', 'system_source_retirement_source_present'), + ('pointer_drift', 'system_source_pointer_drift'), +]) +async def test_retirement_refuses_invalid_declarations_or_source_drift(installed, tmp_path, change, code): + app, _ = installed + source, declaration = await retirement(app, tmp_path) + if change == 'unknown': + declaration['msg.unknown'] = declaration.pop('msg.recovery') + elif change == 'mandatory': + declaration['msg.rules.index'] = declaration.pop('msg.recovery') + elif change == 'malformed': + declaration['msg.recovery']['version'] = True + elif change == 'source_present': + shutil.copyfile(system_source_root() / 'rules/recovery.md', source / 'rules/recovery.md') + elif change == 'pointer_drift': + async with app.metadata.transaction(write=True) as tx: + resource = await tx.resource('r_rule_recovery') + await tx.replace(replace(resource, revision=None, generation=resource.generation+1), + resource.generation) + with pytest.raises(Failure, match='^' + code + '$'): + async with app.metadata.transaction(write=True) as tx: + await sync_system_sources(tx, app.contents, NOW, source_root=source, + retirements=declaration, registry=app.registry) From 2c16fe31489fe1cc150a763de3f8c1668f8015a4 Mon Sep 17 00:00:00 2001 From: LIghtJUNction Date: Tue, 29 Sep 2026 18:23:53 +0800 Subject: [PATCH 041/136] feat: replay pinned typed policy restrictions without promoting recovery --- .github/workflows/issue-recovery.yml | 2 +- docs/TASK_A_RECOVERY_20260928.md | 33 ++ src/msg/admin/recovery_policy.py | 65 ++++ src/msg/admin/recovery_replay.py | 64 +++- .../recovery-policy-checkpoint.schema.json | 319 ++++++++++++++++++ tests/test_recovery_policy_v2.py | 183 ++++++++++ 6 files changed, 648 insertions(+), 18 deletions(-) create mode 100644 src/msg/admin/recovery_policy.py create mode 100644 src/msg/data/recovery-policy-checkpoint.schema.json create mode 100644 tests/test_recovery_policy_v2.py diff --git a/.github/workflows/issue-recovery.yml b/.github/workflows/issue-recovery.yml index 9b0b0f83..36dad705 100644 --- a/.github/workflows/issue-recovery.yml +++ b/.github/workflows/issue-recovery.yml @@ -72,7 +72,7 @@ jobs: run: >- python -m pytest tests/test_recovery_current_authority.py tests/test_task_a_recovery_checkpoint.py tests/test_backup_retirement_attestation.py - tests/test_backup_retirement_cli.py + tests/test_backup_retirement_cli.py tests/test_recovery_policy_v2.py --junitxml=artifacts/recovery-current.xml - uses: actions/upload-artifact@v4 if: always() diff --git a/docs/TASK_A_RECOVERY_20260928.md b/docs/TASK_A_RECOVERY_20260928.md index c564e2fc..605a6f66 100644 --- a/docs/TASK_A_RECOVERY_20260928.md +++ b/docs/TASK_A_RECOVERY_20260928.md @@ -130,3 +130,36 @@ Application/MetadataStore，不做 DDL、迁移、计数器推进、目录修复 最终远端 head 必须另外完成现有完整四分片、精确 JUnit node-ID gate、conformance 及 wheel/sdist。 最终 source SHA、CI run 与实际数量在 PR/issue 的验收评论中记录；未得到最终结果前不声明成功。 没有部署、生产迁移/恢复、生产 Root/PIN/私钥访问、真实资金或真实邮件/Webhook 外发。 + +## Typed policy ceiling checkpoint v2 (partial, not promotion) + +`msg-revocation-checkpoint-v2` preserves the v1 contract and uses a separate +`recovery-checkpoint-v2` signature purpose. Its exact independently supplied pin +is still required. It adds typed `value` payloads: + +- `resource.acl.restrict`: `owner`, `group`, `mode`. Owner and group must match + the restored resource; the mode becomes its intersection with the supplied mask. + Repeated or broader masks never restore removed permission bits. Current resource + generation advances when changed; historical Revision bodies remain untouched. +- `topic.policy.restrict`: `membership_policy`. Existing and supplied policies + intersect: open accepts the supplied restriction, closed remains closed, and + incompatible approval/invite ceilings become closed. It never reopens a topic. +- `topic_ban.set`: explicit `expires_at` (UTC or null), bound by the signed fact. + The target must be a topic; the existing ban/member-removal replay path is used. + Active bans combine conservatively: permanent dominates finite; otherwise the + later expiry wins. A shorter expiry never lifts a previously active ban; only + an explicit `topic_ban.lift` fact lifts it. + +The packet must state `coverage.complete=false` and an exact sorted list of the +fact domains it contains (`resource_acl`, `topic_policy`, `topic_ban`, +`revocations`). Missing/mismatched coverage or a claim of complete recovery is +rejected. The packaged schema is `recovery-policy-checkpoint.schema.json`. +All facts, receipt, authorization epoch and audit commit together. Existing v1 +prefixes can continue into v2; rollback to v1 after a v2 receipt is refused. + +This is conservative restriction, not reconstruction of all current authority. +Owner/group/parent changes, certificate grant/constraint reconciliation, complete +external log provenance/currentness, exhaustive inventory and controlled promotion +remain unfinished. Certificates are still revoked through existing facts; their +signed body is never rewritten to manufacture current grants. Neither coverage +metadata nor an isolated test establishes external freshness or backup destruction. diff --git a/src/msg/admin/recovery_policy.py b/src/msg/admin/recovery_policy.py new file mode 100644 index 00000000..c08d09cb --- /dev/null +++ b/src/msg/admin/recovery_policy.py @@ -0,0 +1,65 @@ +"""Typed policy restrictions for pinned recovery v2; never a promotion contract.""" +from dataclasses import replace + +from msg.constants import ROOT_SUBJECT +from msg.core.codec import parse_time +from msg.core.errors import require + +FACTS = frozenset({'resource.acl.restrict', 'topic.policy.restrict', 'topic_ban.set'}) +DOMAINS = {'resource.acl.restrict': 'resource_acl', 'topic.policy.restrict': 'topic_policy', + 'topic_ban.set': 'topic_ban'} + + +def validate(fact): + kind, value = fact['kind'], fact['value'] + require(type(value) is dict, 'recovery_checkpoint_invalid') + if kind == 'resource.acl.restrict': + require(set(value) == {'owner', 'group', 'mode'} and + all(type(value[k]) is str and 0 < len(value[k]) <= 256 for k in ('owner', 'group')) and + type(value['mode']) is int and 0 <= value['mode'] <= 0o777, + 'recovery_checkpoint_invalid') + elif kind == 'topic.policy.restrict': + require(set(value) == {'membership_policy'} and + value['membership_policy'] in ('open', 'approval', 'invite', 'closed'), + 'recovery_checkpoint_invalid') + else: + require(kind == 'topic_ban.set' and set(value) == {'expires_at'} and + (value['expires_at'] is None or parse_time(value['expires_at']) > parse_time(fact['at'])), + 'recovery_checkpoint_invalid') + + +async def apply(tx, fact): + """Refuse owner/group changes and permission expansion instead of inventing authority.""" + kind, value, target, subject = (fact[k] for k in ('kind', 'value', 'target', 'subject')) + resource = await tx.resource(target) + require(resource.owner == subject, 'recovery_fact_subject_mismatch') + if kind == 'resource.acl.restrict': + require(resource.owner == value['owner'] and resource.group == value['group'], + 'recovery_policy_authority_change_unsupported') + mode = resource.mode & value['mode'] + if resource.mode == mode: + return False + await tx.replace(replace(resource, mode=mode, generation=resource.generation + 1, + modified_at=max(resource.modified_at, parse_time(fact['at'])), + modified_by=ROOT_SUBJECT), + resource.generation) + return True + require(resource.type == 'topic', 'not_a_topic') + if kind == 'topic.policy.restrict': + row = tx.one('SELECT membership_policy FROM topic_settings WHERE topic=?', (target,)) + previous = row[0] if row is not None else 'open' + desired = value['membership_policy'] + require(previous in {'open', 'approval', 'invite', 'closed'}, 'recovery_policy_invalid') + # Approval and invite are incomparable; their conservative intersection + # is closed. Replaying an earlier broader ceiling never reopens a topic. + if desired == 'open' or previous == desired: + return False + if previous != 'open': + desired = 'closed' + if previous == desired: + return False + tx.execute('INSERT INTO topic_settings (topic,membership_policy) VALUES (?,?) ' + 'ON CONFLICT(topic) DO UPDATE SET membership_policy=excluded.membership_policy', + (target, desired), write=True) + return True + raise AssertionError('topic_ban.set uses the existing replay membership-removal path') diff --git a/src/msg/admin/recovery_replay.py b/src/msg/admin/recovery_replay.py index 09403b35..7823b73f 100644 --- a/src/msg/admin/recovery_replay.py +++ b/src/msg/admin/recovery_replay.py @@ -13,6 +13,7 @@ from datetime import UTC, datetime from uuid import uuid4 +from msg.admin import recovery_policy from msg.constants import ROOT_SUBJECT from msg.core.codec import canonical, decode, digest, loads, parse_time, wire from msg.core.errors import Failure, require @@ -21,6 +22,7 @@ from msg.security.quarantine import SETTING, active FORMAT = 'msg-revocation-checkpoint-v1' +POLICY_FORMAT = 'msg-revocation-checkpoint-v2' SUPPORTED_FACTS = frozenset({'credential.revoke', 'certificate.revoke', 'share_grant.revoke', 'share_grant_v2.revoke', 'share_link.revoke', 'membership.remove', 'topic_membership.remove', 'topic_ban.apply', 'topic_ban.lift', 'identity_key.retire', @@ -55,13 +57,16 @@ def verify_checkpoint(packet, *, pin): require(type(packet) is dict and set(packet) == {'checkpoint', 'signature'}, 'recovery_checkpoint_invalid') body = packet['checkpoint'] - require(type(body) is dict and set(body) == { - 'format', 'service', 'source_backup_sha256', 'sequence', 'entries'}, + policy = type(body) is dict and body.get('format') == POLICY_FORMAT + fields = {'format', 'service', 'source_backup_sha256', 'sequence', 'entries'} + if policy: + fields.add('coverage') + require(type(body) is dict and set(body) == fields, 'recovery_checkpoint_invalid') require(type(pin.public_key) is bytes and len(pin.public_key) == 32 and type(pin.sequence) is int and 0 <= pin.sequence <= _MAX_ENTRIES and type(body['sequence']) is int and body['sequence'] == pin.sequence and - body['format'] == FORMAT and body['service'] == pin.service and + body['format'] in {FORMAT, POLICY_FORMAT} and body['service'] == pin.service and isinstance(pin.service, str) and bool(pin.service) and isinstance(body['source_backup_sha256'], str) and re.fullmatch('[0-9a-f]{64}', body['source_backup_sha256']) is not None and @@ -69,16 +74,26 @@ def verify_checkpoint(packet, *, pin): entries = body['entries'] require(type(entries) is list and len(entries) == pin.sequence, 'recovery_checkpoint_invalid') + domains = set() for sequence, fact in enumerate(entries, 1): - require(type(fact) is dict and set(fact) == { - 'sequence', 'kind', 'subject', 'target', 'at'} and + typed = policy and type(fact) is dict and fact.get('kind') in recovery_policy.FACTS + fact_fields = {'sequence', 'kind', 'subject', 'target', 'at'} + if typed: + fact_fields.add('value') + require(type(fact) is dict and set(fact) == fact_fields and type(fact['sequence']) is int and fact['sequence'] == sequence and - fact['kind'] in SUPPORTED_FACTS and + fact['kind'] in (SUPPORTED_FACTS | recovery_policy.FACTS if policy else SUPPORTED_FACTS) and all(isinstance(fact[name], str) and 0 < len(fact[name]) <= 256 for name in ('subject', 'target')), 'recovery_checkpoint_invalid') require(parse_time(fact['at']) <= datetime.now(UTC), 'recovery_checkpoint_invalid') + if typed: + recovery_policy.validate(fact) + domains.add(recovery_policy.DOMAINS[fact['kind']] if typed else 'revocations') + if policy: + require(body['coverage'] == {'complete': False, 'domains': sorted(domains)} and + body['coverage'].get('complete') is False, 'recovery_checkpoint_incomplete_contract') verify(pin.public_key, canonical(body), decode(Signature, packet['signature']), - purpose='recovery-checkpoint-v1') + purpose='recovery-checkpoint-v2' if policy else 'recovery-checkpoint-v1') # Own the verified bytes across the subsequent await/transaction boundary. return loads(canonical(body)) except (Failure, KeyError, TypeError, ValueError, OverflowError) as exc: @@ -121,6 +136,8 @@ def _revoke_credential(tx, subject, target, at): async def _apply(tx, fact): subject, target, at, kind = (fact[name] for name in ('subject', 'target', 'at', 'kind')) + if kind in recovery_policy.FACTS - {'topic_ban.set'}: + return await recovery_policy.apply(tx, fact) if kind == 'credential.revoke': return _revoke_credential(tx, subject, target, at) if kind == 'certificate.revoke': @@ -162,7 +179,11 @@ async def _apply(tx, fact): (target, subject), write=True) return True return False - if kind == 'topic_ban.apply': + if kind in {'topic_ban.apply', 'topic_ban.set'}: + expires = fact['value']['expires_at'] if kind == 'topic_ban.set' else None + if kind == 'topic_ban.set': + await tx.subject(subject) + require((await tx.resource(target)).type == 'topic', 'not_a_topic') require(tx.one('SELECT id FROM resources WHERE id=?', (target,)) is not None, 'recovery_fact_missing') # A live ban removes membership as well as denying topic participation. @@ -178,14 +199,20 @@ async def _apply(tx, fact): if row is None: tx.execute('''INSERT INTO topic_bans (topic,subject,actor,created_at,expires_at,reason,status) - VALUES (?,?,?,?,NULL,?,?)''', - (target, subject, ROOT_SUBJECT, at, 'recovery-replay', 'active'), write=True) + VALUES (?,?,?,?,?,?,?)''', + (target, subject, ROOT_SUBJECT, at, expires, 'recovery-replay', 'active'), write=True) return True - # This checkpoint fact has no expiry. Do not inherit one from an older - # snapshot and silently lift the restored deny, now or in the future. - if row[0] != 'active' or row[1] is not None: - tx.execute("UPDATE topic_bans SET status='active',expires_at=NULL WHERE topic=? AND subject=?", - (target, subject), write=True) + # Restrictions never shorten an existing active ban. A permanent ban + # dominates finite expiries; only a separate explicit lift can remove it. + if kind == 'topic_ban.set' and row[0] == 'active': + if row[1] is None or expires is None: + expires = None + elif parse_time(row[1]) > parse_time(expires): + expires = row[1] + # V1 has no expiry and must not inherit a stale snapshot expiry. + if row[0] != 'active' or row[1] != expires: + tx.execute("UPDATE topic_bans SET status='active',expires_at=? WHERE topic=? AND subject=?", + (expires, target, subject), write=True) return True return member_changed if kind == 'topic_ban.lift': @@ -256,18 +283,21 @@ async def _replay(store, packet, *, pin, operator): 'recovery_checkpoint_backup_mismatch') previous = tx.setting('recovery_replay') if previous is not None: - require(isinstance(previous, dict) and previous.get('format') == FORMAT and + require(isinstance(previous, dict) and previous.get('format') in {FORMAT, body['format']} and type(previous.get('sequence')) is int and 0 <= previous['sequence'] <= pin.sequence and previous.get('prefix_digest') == digest(body['entries'][:previous['sequence']]), 'recovery_checkpoint_regression') changed = 0 for fact in body['entries']: changed += bool(await _apply(tx, fact)) - receipt = {'format': FORMAT, 'checkpoint_digest': pin.digest, 'sequence': pin.sequence, + receipt = {'format': body['format'], 'checkpoint_digest': pin.digest, 'sequence': pin.sequence, 'prefix_digest': digest(body['entries']), 'scope': 'supported_deny_only_facts', 'promotion': 'blocked', 'promotion_blocked_reasons': list(PROMOTION_BLOCKED_REASONS), 'backup_retired': False} + if body['format'] == POLICY_FORMAT: + receipt['scope'] = 'supported_policy_restrictions_only' + receipt['coverage'] = body['coverage'] if changed: tx.set_setting('authorization_epoch', tx.setting('authorization_epoch', 0) + 1) if changed or previous != receipt: diff --git a/src/msg/data/recovery-policy-checkpoint.schema.json b/src/msg/data/recovery-policy-checkpoint.schema.json new file mode 100644 index 00000000..f254862c --- /dev/null +++ b/src/msg/data/recovery-policy-checkpoint.schema.json @@ -0,0 +1,319 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "urn:msg:recovery-checkpoint:v2", + "title": "Independently pinned partial recovery policy ceilings", + "description": "Internal local replay only. Explicitly incomplete; cannot authorize promotion. V1 remains unchanged. ACL masks only remove bits; topic policy ceilings never reopen membership.", + "type": "object", + "additionalProperties": false, + "required": [ + "checkpoint", + "signature" + ], + "properties": { + "checkpoint": { + "type": "object", + "additionalProperties": false, + "required": [ + "format", + "service", + "source_backup_sha256", + "sequence", + "entries", + "coverage" + ], + "properties": { + "format": { + "const": "msg-revocation-checkpoint-v2" + }, + "service": { + "type": "string", + "minLength": 1 + }, + "source_backup_sha256": { + "type": "string", + "pattern": "^[0-9a-f]{64}$" + }, + "sequence": { + "type": "integer", + "minimum": 0, + "maximum": 10000, + "default": 0 + }, + "entries": { + "type": "array", + "maxItems": 10000, + "default": [], + "items": { + "oneOf": [ + { + "type": "object", + "additionalProperties": false, + "required": [ + "sequence", + "kind", + "subject", + "target", + "at" + ], + "properties": { + "sequence": { + "type": "integer", + "minimum": 1, + "maximum": 10000 + }, + "kind": { + "enum": [ + "credential.revoke", + "certificate.revoke", + "share_grant.revoke", + "share_grant_v2.revoke", + "share_link.revoke", + "membership.remove", + "topic_membership.remove", + "topic_ban.apply", + "topic_ban.lift", + "identity_key.retire", + "encryption_key.retire", + "vault.destroy" + ] + }, + "subject": { + "type": "string", + "minLength": 1, + "maxLength": 256 + }, + "target": { + "type": "string", + "minLength": 1, + "maxLength": 256 + }, + "at": { + "type": "string", + "format": "date-time", + "pattern": "Z$" + } + } + }, + { + "type": "object", + "additionalProperties": false, + "required": [ + "sequence", + "kind", + "subject", + "target", + "at", + "value" + ], + "properties": { + "sequence": { + "type": "integer", + "minimum": 1, + "maximum": 10000 + }, + "kind": { + "const": "resource.acl.restrict" + }, + "subject": { + "type": "string", + "minLength": 1, + "maxLength": 256 + }, + "target": { + "type": "string", + "minLength": 1, + "maxLength": 256 + }, + "at": { + "type": "string", + "format": "date-time", + "pattern": "Z$" + }, + "value": { + "type": "object", + "additionalProperties": false, + "required": [ + "owner", + "group", + "mode" + ], + "properties": { + "owner": { + "type": "string", + "minLength": 1, + "maxLength": 256 + }, + "group": { + "type": "string", + "minLength": 1, + "maxLength": 256 + }, + "mode": { + "type": "integer", + "minimum": 0, + "maximum": 511 + } + } + } + } + }, + { + "type": "object", + "additionalProperties": false, + "required": [ + "sequence", + "kind", + "subject", + "target", + "at", + "value" + ], + "properties": { + "sequence": { + "type": "integer", + "minimum": 1, + "maximum": 10000 + }, + "kind": { + "const": "topic.policy.restrict" + }, + "subject": { + "type": "string", + "minLength": 1, + "maxLength": 256 + }, + "target": { + "type": "string", + "minLength": 1, + "maxLength": 256 + }, + "at": { + "type": "string", + "format": "date-time", + "pattern": "Z$" + }, + "value": { + "type": "object", + "additionalProperties": false, + "required": [ + "membership_policy" + ], + "properties": { + "membership_policy": { + "enum": [ + "open", + "approval", + "invite", + "closed" + ] + } + } + } + } + }, + { + "type": "object", + "additionalProperties": false, + "required": [ + "sequence", + "kind", + "subject", + "target", + "at", + "value" + ], + "properties": { + "sequence": { + "type": "integer", + "minimum": 1, + "maximum": 10000 + }, + "kind": { + "const": "topic_ban.set" + }, + "subject": { + "type": "string", + "minLength": 1, + "maxLength": 256 + }, + "target": { + "type": "string", + "minLength": 1, + "maxLength": 256 + }, + "at": { + "type": "string", + "format": "date-time", + "pattern": "Z$" + }, + "value": { + "type": "object", + "additionalProperties": false, + "required": [ + "expires_at" + ], + "properties": { + "expires_at": { + "type": [ + "string", + "null" + ], + "format": "date-time" + } + } + } + } + } + ] + } + }, + "coverage": { + "type": "object", + "additionalProperties": false, + "required": [ + "complete", + "domains" + ], + "properties": { + "complete": { + "const": false + }, + "domains": { + "type": "array", + "uniqueItems": true, + "items": { + "enum": [ + "resource_acl", + "topic_ban", + "topic_policy", + "revocations" + ] + } + } + } + } + } + }, + "signature": { + "type": "object", + "additionalProperties": false, + "required": [ + "key_id", + "algorithm", + "value" + ], + "properties": { + "key_id": { + "type": "string" + }, + "algorithm": { + "const": "ed25519" + }, + "value": { + "type": "string", + "pattern": "^[A-Za-z0-9_-]{86}$" + } + } + } + } +} diff --git a/tests/test_recovery_policy_v2.py b/tests/test_recovery_policy_v2.py new file mode 100644 index 00000000..a79adc19 --- /dev/null +++ b/tests/test_recovery_policy_v2.py @@ -0,0 +1,183 @@ +"""Pinned typed ceilings shrink old authorization without claiming complete recovery.""" +from copy import deepcopy +from datetime import timedelta + +import pytest + +from msg.admin.recovery_replay import TrustedCheckpointPin, _replay, verify_checkpoint, POLICY_FORMAT +from msg.core.codec import canonical, digest, wire +from msg.core.errors import Failure +from msg.security.crypto import Ed25519Signer +from msg.security.quarantine import SETTING, active +from test_service import NOW + + +@pytest.fixture +async def policy_target(installed): + app, _ = installed + async with app.metadata.transaction(write=True) as tx: + resource = await tx.resource('u_root') + tx.set_setting(SETTING, {'format': 'msg-recovery-quarantine-v1', + 'source_backup_sha256': 'a' * 64, 'outbound_enabled': False, 'authority': 'health_only'}) + return app, resource + + +def packet_for(resource, *, mode=0, complete=False): + signer = Ed25519Signer.generate() + body = {'format': POLICY_FORMAT, 'service': 'http://testserver', + 'source_backup_sha256': 'a' * 64, 'sequence': 1, + 'coverage': {'complete': complete, 'domains': ['resource_acl']}, + 'entries': [{'sequence': 1, 'kind': 'resource.acl.restrict', 'subject': resource.owner, + 'target': resource.id, 'at': wire(NOW), + 'value': {'owner': resource.owner, 'group': resource.group, 'mode': mode}}]} + def signed(body, purpose='recovery-checkpoint-v2'): + return {'checkpoint': body, 'signature': wire(signer.sign(canonical(body), purpose=purpose))} + def pin(body): + return TrustedCheckpointPin(service=body['service'], public_key=signer.public_key, + digest=digest(body), sequence=body['sequence']) + return body, signed, pin + + +async def replay(app, body, signed, pin): + return await _replay(app.metadata, signed(body), pin=pin(body), operator='isolated-test') + + +async def test_v2_acl_is_monotonic_idempotent_and_preserves_history(policy_target): + app, resource = policy_target + body, signed, pin = packet_for(resource) + async with app.metadata.transaction(write=False) as tx: + history = tx.rows('SELECT body FROM revisions ORDER BY id') + result = await replay(app, body, signed, pin) + assert result['changed'] == 1 and result['promotion'] == 'blocked' + assert result['coverage'] == {'complete': False, 'domains': ['resource_acl']} + assert (await replay(app, body, signed, pin))['changed'] == 0 + async with app.metadata.transaction(write=False) as tx: + updated = await tx.resource(resource.id) + assert updated.mode == 0 and updated.generation == resource.generation + 1 + assert updated.revision == resource.revision and active(tx) + assert tx.rows('SELECT body FROM revisions ORDER BY id') == history + # A newly pinned broader ceiling cannot restore rights already removed. + broader, signed2, pin2 = packet_for(resource, mode=0o777) + broader['entries'] = body['entries'] + [dict(broader['entries'][0], sequence=2)] + broader['sequence'] = 2 + assert (await replay(app, broader, signed2, pin2))['changed'] == 0 + + +@pytest.mark.parametrize('change', ['complete', 'coverage', 'owner', 'bool', 'extra', 'purpose', 'v1']) +async def test_v2_rejects_unsupported_authority_and_contracts(policy_target, change): + app, resource = policy_target + body, signed, pin = packet_for(resource) + if change == 'complete': + body['coverage']['complete'] = True + elif change == 'coverage': + body['coverage']['domains'] = [] + elif change == 'owner': + body['entries'][0]['value']['owner'] = 'u_other' + elif change == 'bool': + body['entries'][0]['value']['mode'] = True + elif change == 'extra': + body['entries'][0]['value']['grant'] = 'root' + elif change == 'v1': + body['format'] = 'msg-revocation-checkpoint-v1' + body.pop('coverage') + with pytest.raises(Failure): + if change == 'purpose': + verify_checkpoint(signed(body, 'recovery-checkpoint-v1'), pin=pin(body)) + else: + await replay(app, body, signed, pin) + async with app.metadata.transaction(write=False) as tx: + assert await tx.resource(resource.id) == resource + assert tx.setting('recovery_replay') is None and active(tx) + + +async def test_v2_policy_ban_expiry_and_rollback(policy_target): + from msg.core.models import Resource + app, owner = policy_target + topic = Resource(id='t_recovery_policy', type='topic', type_version=1, name='policy', + parent=owner.id, owner=owner.id, group='g_public', mode=0o777, generation=1, + revision=None, state='active', created_at=NOW, created_by=owner.id, + modified_at=NOW, modified_by=owner.id) + async with app.metadata.transaction(write=True) as tx: + await tx.insert(topic) + tx.execute('INSERT INTO topic_memberships VALUES (?,?,?,?,?,NULL)', + (topic.id, owner.id, 'admin', 'active', wire(NOW)), write=True) + body, signed, pin = packet_for(topic) + fact = {'sequence': 2, 'kind': 'topic.policy.restrict', 'subject': owner.id, + 'target': topic.id, 'at': wire(NOW), 'value': {'membership_policy': 'invite'}} + ban = dict(fact, sequence=3, kind='topic_ban.set', + value={'expires_at': wire(NOW + timedelta(days=1))}) + body.update(sequence=3, entries=body['entries'] + [fact, ban], + coverage={'complete': False, 'domains': ['resource_acl', 'topic_ban', 'topic_policy']}) + bad = deepcopy(body) + bad['entries'][2]['target'] = 'missing' + with pytest.raises(Failure): + await replay(app, bad, signed, pin) + async with app.metadata.transaction(write=False) as tx: + assert (await tx.resource(topic.id)).mode == 0o777 + assert tx.one('SELECT membership_policy FROM topic_settings WHERE topic=?', (topic.id,)) is None + assert (await replay(app, body, signed, pin))['changed'] == 3 + assert (await replay(app, body, signed, pin))['changed'] == 0 + async with app.metadata.transaction(write=False) as tx: + assert tx.one('SELECT membership_policy FROM topic_settings WHERE topic=?', (topic.id,)) == ('invite',) + assert tx.one('SELECT expires_at FROM topic_bans WHERE topic=?', (topic.id,)) == (ban['value']['expires_at'],) + assert tx.one('SELECT role,status FROM topic_memberships WHERE topic=?', (topic.id,)) == ('member', 'removed') + assert active(tx) + + # Approval and invite cannot be ordered; their safe intersection stays closed. + body['entries'].append(dict(fact, sequence=4, value={'membership_policy': 'approval'})) + body['sequence'] = 4 + assert (await replay(app, body, signed, pin))['changed'] == 1 + assert (await replay(app, body, signed, pin))['changed'] == 0 + async with app.metadata.transaction(write=False) as tx: + assert tx.one('SELECT membership_policy FROM topic_settings WHERE topic=?', (topic.id,)) == ('closed',) + + +def test_packaged_policy_schema_matches_typed_contract(): + import json + from importlib.resources import files + import jsonschema + from types import SimpleNamespace + resource = SimpleNamespace(id='t_test', owner='u_owner', group='g_public') + body, signed, pin = packet_for(resource) + schema = json.loads(files('msg.data').joinpath('recovery-policy-checkpoint.schema.json').read_text()) + jsonschema.validate(signed(body), schema) + assert verify_checkpoint(signed(body), pin=pin(body)) == body + body['coverage']['complete'] = True + with pytest.raises(jsonschema.ValidationError): + jsonschema.validate(signed(body), schema) + + +async def test_v1_prefix_can_continue_in_v2_but_not_downgrade(policy_target): + app, resource = policy_target + body, signed, pin = packet_for(resource) + old = dict(body, format='msg-revocation-checkpoint-v1', sequence=0, entries=[]) + old.pop('coverage') + await _replay(app.metadata, signed(old, 'recovery-checkpoint-v1'), pin=pin(old), operator='isolated-test') + await replay(app, body, signed, pin) + with pytest.raises(Failure, match='recovery_checkpoint_regression'): + await _replay(app.metadata, signed(old, 'recovery-checkpoint-v1'), pin=pin(old), operator='isolated-test') + async with app.metadata.transaction(write=False) as tx: + assert tx.setting('recovery_replay')['format'] == POLICY_FORMAT and active(tx) + + +@pytest.mark.parametrize('existing_days', [None, 1, 3]) +async def test_v2_ban_never_shortens_an_active_restriction(policy_target, existing_days): + from dataclasses import replace + app, owner = policy_target + topic = replace(owner, id='t_existing_ban', type='topic', name='existing-ban', + parent=owner.id, revision=None, generation=1) + previous = wire(NOW + timedelta(days=existing_days)) if existing_days else None + desired = wire(NOW + timedelta(days=2)) + async with app.metadata.transaction(write=True) as tx: + await tx.insert(topic) + tx.execute('INSERT INTO topic_bans VALUES (?,?,?,?,?,?,?)', + (topic.id, owner.id, owner.id, wire(NOW), previous, 'prior', 'active'), write=True) + body, signed, pin = packet_for(topic) + body['entries'][0].update(kind='topic_ban.set', value={'expires_at': desired}) + body['coverage']['domains'] = ['topic_ban'] + await replay(app, body, signed, pin) + assert (await replay(app, body, signed, pin))['changed'] == 0 + async with app.metadata.transaction(write=False) as tx: + expected = None if existing_days is None else max(previous, desired) + assert tx.one('SELECT expires_at FROM topic_bans WHERE topic=?', (topic.id,)) == (expected,) + assert active(tx) From c3cf58b01f0725fb7fbf7621224f88dc8717abdb Mon Sep 17 00:00:00 2001 From: LIghtJUNction Date: Tue, 29 Sep 2026 18:22:47 +0800 Subject: [PATCH 042/136] fix: recheck current visibility in collaboration reads --- src/msg/plugins/collaboration.py | 5 +- src/msg/plugins/communication.py | 1 + .../test_collaboration_current_visibility.py | 55 +++++++++++++++++++ 3 files changed, 59 insertions(+), 2 deletions(-) create mode 100644 tests/test_collaboration_current_visibility.py diff --git a/src/msg/plugins/collaboration.py b/src/msg/plugins/collaboration.py index 95d31c42..90c20cc8 100644 --- a/src/msg/plugins/collaboration.py +++ b/src/msg/plugins/collaboration.py @@ -46,10 +46,11 @@ async def _visible_refs(app, ctx, request, tx, refs): output = [] for rid in refs: try: - await check_access(app, ctx, request, tx, rid, 'read') + await _safe_ref(app, ctx, request, tx, rid) except Failure as exc: if exc.code in {'permission_denied', 'credential_ceiling', 'certificate_gate', - 'delegation_scope', 'ancestor_inactive', 'not_found'}: + 'delegation_scope', 'ancestor_inactive', 'not_found', + 'collaboration_ref_forbidden'}: continue raise output.append(rid) diff --git a/src/msg/plugins/communication.py b/src/msg/plugins/communication.py index a97ce67a..095ad0e7 100644 --- a/src/msg/plugins/communication.py +++ b/src/msg/plugins/communication.py @@ -243,6 +243,7 @@ async def webhook_subscription(ctx,request,tx): async def presence_get(ctx,request,tx): subject=await resolve(tx,request.arguments['subject_id']) await tx.subject(subject) + await check_access(app,ctx,request,tx,subject,'read') row=tx.one('SELECT expires_at,body FROM presence WHERE subject=?',(subject,)) if row is None or parse_time(row[0])<=ctx.now: return HandlerOutput(data={'subject_id':subject,'state':'unknown'}) diff --git a/tests/test_collaboration_current_visibility.py b/tests/test_collaboration_current_visibility.py new file mode 100644 index 00000000..ed94a299 --- /dev/null +++ b/tests/test_collaboration_current_visibility.py @@ -0,0 +1,55 @@ +"""Collaboration views obey current visibility, including after target moves.""" +import pytest + +from msg.core.codec import wire +from test_service import call, register + + +@pytest.mark.asyncio +async def test_presence_rechecks_current_subject_permissions(installed): + app, _ = installed + key, subject, _ = await register(app, 'private-presence') + created = await call(app, 'communication.presence_set', + {'state': 'busy', 'message': 'private activity'}, + key=key, subject=subject) + assert created.status == 'ok', wire(created) + assert (await call(app, 'communication.presence_get', + {'subject_id': subject})).data['message'] == 'private activity' + async with app.metadata.transaction(write=False) as tx: + generation = (await tx.resource(subject)).generation + changed = await call(app, 'content.chmod', {'id': subject, 'mode': '0700'}, + key=key, subject=subject, expected=((subject, generation),)) + assert changed.status == 'ok', changed.error + hidden = await call(app, 'communication.presence_get', {'subject_id': subject}) + assert hidden.status == 'error', wire(hidden) + assert 'private activity' not in str(wire(hidden)) + own = await call(app, 'communication.presence_get', {'subject_id': subject}, + key=key, subject=subject) + assert own.data['message'] == 'private activity' + + +@pytest.mark.asyncio +async def test_handoff_filters_references_moved_to_forbidden_namespace(installed): + app, _ = installed + key, subject, _ = await register(app, 'handoff-move-owner') + other_key, other, _ = await register(app, 'handoff-move-other') + post = await call(app, 'content.post_create', {'parent': '/main', 'body': 'Work'}, + key=key, subject=subject) + rid = post.resources[0].id + created = await call(app, 'communication.handoff_create', + {'to_subject': other, 'resource_refs': [rid]}, + key=key, subject=subject) + hid = created.data['handoff']['id'] + moved = await call(app, 'content.move', + {'id': rid, 'parent': subject, 'name': 'todos'}, + key=key, subject=subject, + expected=((rid, post.data['generation']),)) + assert moved.status == 'ok', moved.error + for actor_key, actor in ((key, subject), (other_key, other)): + got = await call(app, 'communication.handoff_get', {'id': hid}, + key=actor_key, subject=actor) + assert got.status == 'ok', wire(got) + assert not got.data['handoff']['resource_refs'] + listed = await call(app, 'communication.handoff_list', {}, + key=actor_key, subject=actor) + assert not listed.data['items'][0]['resource_refs'] From acc3e0a5f4c93f7e046faf1abd65ed07751fa806 Mon Sep 17 00:00:00 2001 From: LIghtJUNction Date: Tue, 29 Sep 2026 18:26:27 +0800 Subject: [PATCH 043/136] feat: filter search relations with current endpoint visibility --- src/msg/plugins/discovery.py | 51 ++++++++++++++++++-- src/msg/plugins/transfer.py | 4 +- src/msg/transports/http_routes.py | 15 +++--- tests/test_search_source_relation_filters.py | 46 ++++++++++++++++++ 4 files changed, 104 insertions(+), 12 deletions(-) diff --git a/src/msg/plugins/discovery.py b/src/msg/plugins/discovery.py index eecb00dd..c1aafe71 100644 --- a/src/msg/plugins/discovery.py +++ b/src/msg/plugins/discovery.py @@ -784,7 +784,8 @@ async def collection_page(arguments,depth, *, last=None,boundary=None): lexical_schema_v5={**lexical_schema_v4, 'properties':{**lexical_schema_v4['properties'], - 'revision':IDENTIFIER, + 'revision':IDENTIFIER,'relation_to':IDENTIFIER,'relation_from':IDENTIFIER, + 'has_replies':BOOLEAN,'has_references':BOOLEAN, 'source_version':{'type':'integer','minimum':1,'maximum':2147483647}}} @op('discovery.lexical_search',lexical_schema_v5,effect='read',version=5) @@ -807,7 +808,7 @@ async def lexical_search(ctx,request,tx): a={**saved['arguments'],'cursor':a['cursor']} # A cursor or sealed QueryRef carries arguments from an earlier call. # Keep those arguments inside the version selected for this call too. - require(not (request.contract_version<5 and {'revision','source_version'}&a.keys()) and + require(not (request.contract_version<5 and {'revision','source_version','relation_to','relation_from','has_replies','has_references'}&a.keys()) and not (request.contract_version<4 and 'suggest' in a) and not (request.contract_version<3 and {'source_kind','relation_type'}&a.keys()) and @@ -854,6 +855,12 @@ async def lexical_search(ctx,request,tx): if a.get('suggest'): require(2<=len(suggest_prefix)<=32,'query_cost_exceeded') scanned=0 + relation_scanned=0 + endpoint_visibility={} + async def endpoint_visible(rid): + if rid not in endpoint_visibility: + endpoint_visibility[rid]=await visible(app,ctx,request,tx,rid) + return endpoint_visibility[rid] # Restrict the SQL candidate set before applying the work budget. A # global LIMIT lets unrelated (or unreadable) rows starve a small scope. owner_scope=scope_resource.type in {'user','organization'} @@ -906,12 +913,48 @@ async def lexical_search(ctx,request,tx): if a.get('source_kind') and (revision is None or revision.source_kind!=a['source_kind']): continue - if a.get('relation_type') and (revision is None or not any( + if request.contract_version>=5: + outgoing=[] + if {'relation_to','relation_type','has_attachment'}&a.keys(): + for relation in revision.relations if revision else (): + relation_scanned+=1 + require(relation_scanned<=4096 and time.monotonic()