From f87ea8c0f72a774d98d002cfdb5aa95e29979b22 Mon Sep 17 00:00:00 2001 From: LIghtJUNction Date: Mon, 28 Sep 2026 23:02:46 +0800 Subject: [PATCH 1/7] fix: replay an active topic ban without promoting recovery A restored database can omit a ban that the pinned log still requires. Record that ban locally, keep lift deny-only, and state why promotion stays blocked. Co-authored-by: Cursor --- src/msg/admin/recovery_replay.py | 31 +++++++++++++++++--- src/msg/data/recovery-checkpoint.schema.json | 1 + tests/test_task_a_recovery_checkpoint.py | 14 +++++++-- 3 files changed, 40 insertions(+), 6 deletions(-) diff --git a/src/msg/admin/recovery_replay.py b/src/msg/admin/recovery_replay.py index 0df26631..f0a61b1e 100644 --- a/src/msg/admin/recovery_replay.py +++ b/src/msg/admin/recovery_replay.py @@ -23,8 +23,13 @@ FORMAT = 'msg-revocation-checkpoint-v1' SUPPORTED_FACTS = frozenset({'credential.revoke', 'certificate.revoke', 'share_grant.revoke', 'share_grant_v2.revoke', 'share_link.revoke', 'membership.remove', - 'topic_membership.remove', 'topic_ban.lift', 'identity_key.retire', + 'topic_membership.remove', 'topic_ban.apply', 'topic_ban.lift', 'identity_key.retire', 'encryption_key.retire', 'vault.destroy'}) +PROMOTION_BLOCKED_REASONS = ( + 'quarantine_remains', + 'supported_facts_are_deny_only', + 'backup_retirement_not_attested', +) _SHARE_TABLES = {'share_grant.revoke': 'share_grants', 'share_grant_v2.revoke': 'share_grants_v2', 'share_link.revoke': 'share_links'} _MAX_ENTRIES = 10000 @@ -157,6 +162,21 @@ async def _apply(tx, fact): (target, subject), write=True) return True return False + if kind == 'topic_ban.apply': + require(tx.one('SELECT id FROM resources WHERE id=?', (target,)) is not None, + 'recovery_fact_missing') + row = tx.one('SELECT status FROM topic_bans WHERE topic=? AND subject=?', (target, subject)) + if row is None: + tx.execute('''INSERT INTO topic_bans + (topic,subject,actor,created_at,expires_at,reason,status) + VALUES (?,?,?,?,NULL,?,?)''', + (target, subject, ROOT_SUBJECT, at, 'recovery-replay', 'active'), write=True) + return True + if row[0] != 'active': + tx.execute("UPDATE topic_bans SET status='active' WHERE topic=? AND subject=?", + (target, subject), write=True) + return True + return False if kind == 'topic_ban.lift': row = tx.one('SELECT status FROM topic_bans WHERE topic=? AND subject=?', (target, subject)) require(row is not None, 'recovery_fact_missing') @@ -209,8 +229,9 @@ async def _replay(store, packet, *, pin, operator): Always re-apply facts, including after a repeated invocation/restart: cached watermarks are not evidence that an authorization row has not been restored. - This neither imports keys, emits effects, nor clears quarantine. topic_ban.lift - is the only topic-ban fact; it does not reconcile ACLs or promote production. + This neither imports keys, emits effects, nor clears quarantine. topic_ban.apply + only records an active ban, and topic_ban.lift only marks an existing ban lifted. + Neither reconciles ACLs nor promotes production. """ require(isinstance(operator, str) and bool(operator), 'recovery_operator_required') body = verify_checkpoint(packet, pin=pin) @@ -233,7 +254,9 @@ async def _replay(store, packet, *, pin, operator): changed += bool(await _apply(tx, fact)) receipt = {'format': FORMAT, 'checkpoint_digest': pin.digest, 'sequence': pin.sequence, 'prefix_digest': digest(body['entries']), 'scope': 'supported_deny_only_facts', - 'promotion': 'blocked', 'backup_retired': False} + 'promotion': 'blocked', + 'promotion_blocked_reasons': list(PROMOTION_BLOCKED_REASONS), + 'backup_retired': False} if changed: tx.set_setting('authorization_epoch', tx.setting('authorization_epoch', 0) + 1) if changed or previous != receipt: diff --git a/src/msg/data/recovery-checkpoint.schema.json b/src/msg/data/recovery-checkpoint.schema.json index a0b06413..a2726df4 100644 --- a/src/msg/data/recovery-checkpoint.schema.json +++ b/src/msg/data/recovery-checkpoint.schema.json @@ -67,6 +67,7 @@ "share_link.revoke", "membership.remove", "topic_membership.remove", + "topic_ban.apply", "topic_ban.lift", "identity_key.retire", "encryption_key.retire", diff --git a/tests/test_task_a_recovery_checkpoint.py b/tests/test_task_a_recovery_checkpoint.py index 4fe1c0c4..c5cba9ca 100644 --- a/tests/test_task_a_recovery_checkpoint.py +++ b/tests/test_task_a_recovery_checkpoint.py @@ -89,7 +89,10 @@ async def test_replay_is_atomic_monotonic_idempotent_and_never_promotes(restored body, pin, signed = checkpoint results = await asyncio.gather(*(replay(restored, signed(body), pin=pin) for _ in range(3))) assert sum(result['changed'] for result in results) == 1 - assert all(result['promotion'] == 'blocked' and result['backup_retired'] is False for result in results) + assert all(result['promotion'] == 'blocked' and result['backup_retired'] is False and + result['promotion_blocked_reasons'] == [ + 'quarantine_remains', 'supported_facts_are_deny_only', + 'backup_retirement_not_attested'] for result in results) async with restored.transaction(write=False) as tx: assert active(tx) credential = await tx.credential('token_one') @@ -286,10 +289,13 @@ async def test_all_supported_owned_revocations_preserve_signed_bytes_and_never_i tx.execute('''INSERT INTO topic_bans (topic,subject,actor,created_at,expires_at,reason,status) VALUES (?,?,?,?,NULL,?,?)''', (resource.id, owner, owner, wire(NOW), 'restored-ban', 'active'), write=True) + banned = replace(resource, id='t_ban', name='ban') + await tx.insert(banned) targets = [('credential.revoke', 'token_one'), ('certificate.revoke', 'cert_one'), ('share_grant.revoke', 'sg_one'), ('share_grant_v2.revoke', 'sg_two'), ('share_link.revoke', 'link_one'), ('membership.remove', 'g_test'), - ('topic_membership.remove', 't_test'), ('topic_ban.lift', resource.id), + ('topic_membership.remove', 't_test'), ('topic_ban.apply', 't_ban'), + ('topic_ban.lift', resource.id), ('identity_key.retire', signer.key_id), ('encryption_key.retire', age_id), ('vault.destroy', age_id)] assert {kind for kind, _ in targets} == SUPPORTED_FACTS @@ -299,6 +305,8 @@ async def test_all_supported_owned_revocations_preserve_signed_bytes_and_never_i pin = replace(pin, sequence=len(facts), digest=digest(body)) result = await replay(restored, signed(body), pin=pin) assert result['changed'] == len(facts) and result['backup_retired'] is False + assert result['promotion_blocked_reasons'] == [ + 'quarantine_remains', 'supported_facts_are_deny_only', 'backup_retirement_not_attested'] async with restored.transaction(write=False) as tx: assert tx.one('SELECT body,revoked FROM certificates WHERE id=?', ('cert_one',)) == (original, 1) assert (await tx.credential(signer.key_id)).revoked_at == NOW @@ -314,6 +322,8 @@ async def test_all_supported_owned_revocations_preserve_signed_bytes_and_never_i ('t_test', owner)) == ('removed',) assert tx.one('SELECT status FROM topic_bans WHERE topic=? AND subject=?', ('t_test', owner)) == ('lifted',) + assert tx.one('SELECT status,actor,reason FROM topic_bans WHERE topic=? AND subject=?', + ('t_ban', owner)) == ('active', 'u_root', 'recovery-replay') assert tx.one('SELECT status,signing_nonce,signing_ciphertext,age_nonce,age_ciphertext ' 'FROM custodial_vault WHERE subject=?', (owner,)) == ('destroyed', None, None, None, None) assert tx.one('SELECT count(*) FROM jobs')[0] == 0 and active(tx) From ed12fe897ac451bb9b021856ccffbf33cab0cf8c Mon Sep 17 00:00:00 2001 From: LIghtJUNction Date: Mon, 28 Sep 2026 23:33:09 +0800 Subject: [PATCH 2/7] test: nest the banned topic so the fixture keeps a single root resource The one_root index allows only one parentless resource, so a second root topic made the topic_ban.apply fixture fail with constraint_conflict on PostgreSQL. Co-authored-by: Cursor --- tests/test_task_a_recovery_checkpoint.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_task_a_recovery_checkpoint.py b/tests/test_task_a_recovery_checkpoint.py index c5cba9ca..602159ea 100644 --- a/tests/test_task_a_recovery_checkpoint.py +++ b/tests/test_task_a_recovery_checkpoint.py @@ -289,7 +289,7 @@ async def test_all_supported_owned_revocations_preserve_signed_bytes_and_never_i tx.execute('''INSERT INTO topic_bans (topic,subject,actor,created_at,expires_at,reason,status) VALUES (?,?,?,?,NULL,?,?)''', (resource.id, owner, owner, wire(NOW), 'restored-ban', 'active'), write=True) - banned = replace(resource, id='t_ban', name='ban') + banned = replace(resource, id='t_ban', name='ban', parent=resource.id) await tx.insert(banned) targets = [('credential.revoke', 'token_one'), ('certificate.revoke', 'cert_one'), ('share_grant.revoke', 'sg_one'), ('share_grant_v2.revoke', 'sg_two'), From f2ebc093fb8333c2a0f196b6ebd86cfdf717affa Mon Sep 17 00:00:00 2001 From: LIghtJUNction Date: Tue, 29 Sep 2026 02:21:38 +0800 Subject: [PATCH 3/7] chore: mark the rewrite preview as 1.0.0a1 The published 0.x line is a different series. This source version does not claim a PyPI release or a finished production migration. Co-authored-by: Cursor --- CHANGELOG.md | 6 ++++++ pyproject.toml | 2 +- src/msg/__init__.py | 2 +- 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9f08c7c1..b2e93c1e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,11 @@ # 变更记录 +## 1.0.0a1 — 重构预览 + +以 1.x 预发布版本标识当前重构，与 PyPI 上的 0.x 系列区分。新版本使用 Python 3.15、PostgreSQL 权威状态、版本化操作与签名授权，并通过 `uv_build` 构建。 + +这只是源码版本调整，尚未发布到 PyPI，也不表示生产部署、旧数据迁移或恢复验收已经完成。 + ## 0.1.0a1 — 本地独立重写交付 从项目设计契约重新实现模块化单体。删除旧实现，不包含旧数据库迁移器或线上自动升级动作。 diff --git a/pyproject.toml b/pyproject.toml index dac7c944..48f96b6f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "uv_build" [project] name = "msg-lmm-best" -version = "0.1.0a1" +version = "1.0.0a1" description = "Atomic communication infrastructure for sandboxed agents" readme = "README.md" requires-python = ">=3.15" diff --git a/src/msg/__init__.py b/src/msg/__init__.py index 966a4f77..d72073ba 100644 --- a/src/msg/__init__.py +++ b/src/msg/__init__.py @@ -1,2 +1,2 @@ """Atomic communication for sandboxed agents.""" -__version__ = "0.1.0a1" +__version__ = "1.0.0a1" From 3490750dd8fd59c8070e9816a104292db0937cea Mon Sep 17 00:00:00 2001 From: LIghtJUNction Date: Tue, 29 Sep 2026 02:43:40 +0800 Subject: [PATCH 4/7] feat(security): accept only console-imported root-signed backup retirement records backup_retired now needs one exact root-signed record bound to the subject and both old key IDs, imported behind require_local_console and re-verified on every read; server_key_retired additionally needs online retirement. Missing, altered, expired, misbound or pre-rotation records fail closed. Slice of #68. Co-authored-by: Cursor --- docs/CUSTODIAL_HISTORY.md | 24 +- docs/ISSUES_68_70_PROGRESS.md | 6 +- src/msg/admin/backup_retirement.py | 56 ++++ src/msg/admin/root.py | 37 +++ src/msg/plugins/custodial_lifecycle.py | 7 +- src/msg/plugins/recovery.py | 3 +- src/msg/security/backup_retirement.py | 135 +++++++++ src/msg/security/custodial_migration.py | 32 +- src/msg/security/custody_history.py | 20 +- tests/test_backup_retirement_attestation.py | 318 ++++++++++++++++++++ 10 files changed, 612 insertions(+), 26 deletions(-) create mode 100644 src/msg/admin/backup_retirement.py create mode 100644 src/msg/security/backup_retirement.py create mode 100644 tests/test_backup_retirement_attestation.py diff --git a/docs/CUSTODIAL_HISTORY.md b/docs/CUSTODIAL_HISTORY.md index db25bc9e..c051f56c 100644 --- a/docs/CUSTODIAL_HISTORY.md +++ b/docs/CUSTODIAL_HISTORY.md @@ -90,11 +90,25 @@ Test Root, rewraps, locally decrypts, ACKs and finalizes it. It refuses a produc application without the isolated selftest namespace. `history_recoverable` covers **enumerated and client-verified revisions only**. -`online_key_retired` describes online vault material only. `backup_retired` and -`server_key_retired` remain false until a separate independently verified offline -retirement procedure exists. Neither a boolean supplied by a network caller nor a -green test run can establish that old backups lost a key. Restore/revocation replay -and real operator acceptance are tracked separately in issues #69 and #70. +`online_key_retired` describes online vault material only. `backup_retired` stays +false unless a root-signed backup retirement record was imported at the physical +console (`RootAdmin.sign_backup_retirement` / `RootAdmin.import_backup_retirement`, +both behind `require_local_console`). The record binds the subject, both old key +IDs, a non-empty list of named backup sets and a validity window of at most 400 +days; it is signed with purpose `custodial-backup-retirement-v1`. Every read +re-verifies it against the active, unrevoked root credential, so an altered, +expired, misbound or pre-rotation record reports `backup_retired=false` +(`backup_status=local_attestation_invalid|expired`). `server_key_retired` is true +only when that record and observable online retirement both hold. + +The record is an operator attestation for the **listed backup sets only**. It +does not prove that no other copy exists and it does not claim that production +backups were destroyed. No network operation writes, accepts or forwards it; a +`backup_retirement` value inside the network-writable migration body is ignored. +Neither a boolean supplied by a network caller nor a green test run can establish +that old backups lost a key. There is no `msg-admin` CLI wiring yet. Restore/ +revocation replay and real operator acceptance are tracked separately in issues +#69 and #70. ## Published contract compatibility diff --git a/docs/ISSUES_68_70_PROGRESS.md b/docs/ISSUES_68_70_PROGRESS.md index 94c5e9e7..1245a4f0 100644 --- a/docs/ISSUES_68_70_PROGRESS.md +++ b/docs/ISSUES_68_70_PROGRESS.md @@ -21,8 +21,10 @@ only the challenge's new signing key may inspect its completed result. The read reports `identity_switched`, `history_recoverable`, `online_retired`, `backup_retired`, `server_key_retired` and a separate `completion_status`. Online retirement requires the new primary keys, retired old keys, revoked old -signing credential and empty destroyed vault fields. No backup-destruction -attestation exists yet, so `backup_retired` and `server_key_retired` remain false. +signing credential and empty destroyed vault fields. `backup_retired` becomes true +only for one exact root-signed record imported at the physical console and +re-verified on each read (see `CUSTODIAL_HISTORY.md`); `server_key_retired` +additionally needs online retirement. Without that record both remain false. The legacy `status=completed` from an empty-inventory identity switch is not a claim that every server-held backup has been destroyed; use the independent retirement fields. diff --git a/src/msg/admin/backup_retirement.py b/src/msg/admin/backup_retirement.py new file mode 100644 index 00000000..adb6b164 --- /dev/null +++ b/src/msg/admin/backup_retirement.py @@ -0,0 +1,56 @@ +"""Local import of a root-signed custodial backup retirement record. + +Never imported by a network adapter, plugin or worker. RootAdmin calls these use +cases only after require_local_console(); tests call them against disposable +databases. The record is verified again on every read, so this import is not a +trusted cache of a success flag. +""" +from __future__ import annotations + +from msg.constants import ROOT_SUBJECT +from msg.core.codec import digest, loads +from msg.core.errors import require +from msg.core.models import AuditEvent, Event, ResourceRef +from msg.plugins.common import new_id +from msg.security.backup_retirement import FIELDS, DOMAIN, check, root_verifier, setting_key + + +def unsigned_statement(value): + require(isinstance(value, dict) and set(value) == FIELDS and value.get('domain') == DOMAIN, + 'backup_retirement_invalid') + return value + + +def completed_migration(tx, subject_id, old_identity_key_id): + """The one completed migration of this subject that retired exactly this old key.""" + matches = [loads(body) for (body,) in tx.rows( + "SELECT body FROM custodial_upgrades WHERE subject=? AND status='completed' ORDER BY id", + (subject_id,)) if loads(body).get('old_identity_key_id') == old_identity_key_id] + require(len(matches) == 1, 'backup_retirement_migration_not_found') + return matches[0] + + +async def import_record(tx, record, *, now, operator): + body = record.get('statement') if isinstance(record, dict) else None + require(isinstance(body, dict) and isinstance(body.get('subject_id'), str) and + isinstance(body.get('old_identity_key_id'), str), 'backup_retirement_invalid') + subject = body['subject_id'] + details = completed_migration(tx, subject, body['old_identity_key_id']) + attestation = check(record, await root_verifier(tx), subject, details, now=now) + key = setting_key(subject, attestation.old_identity_key_id) + previous = tx.setting(key) + tx.set_setting(key, record) + data = {'subject_id': subject, 'old_identity_key_id': attestation.old_identity_key_id, + 'old_encryption_key_id': attestation.old_encryption_key_id, + 'backup_sets': list(attestation.backup_sets), 'record_digest': attestation.record_digest, + 'signer_key_id': attestation.signer_key_id, 'operator': operator, + 'scope': 'listed_backup_sets_only'} + event = Event(id=new_id('audit'), type='root.custodial.backup_retirement', time=now, + request_id=new_id('local'), actor=ROOT_SUBJECT, subject=subject, + resources=(ResourceRef(id=subject),), data=data) + await tx.append_audit(AuditEvent(event=event, + authority=(ResourceRef(id=tx.setting('active_root_certificate')),), + before_digest=digest(previous) if previous is not None else None, + after_digest=attestation.record_digest, previous_digest=None, entry_digest='', + result='imported')) + return attestation diff --git a/src/msg/admin/root.py b/src/msg/admin/root.py index bf31d7d8..c77107fa 100644 --- a/src/msg/admin/root.py +++ b/src/msg/admin/root.py @@ -262,6 +262,43 @@ async def read(): return asyncio.run(_revoke(app,certificate_id,Ed25519Signer.from_bytes(private),reason=reason,operator=operator)) + def sign_backup_retirement(self,source,destination): + """Root-sign an operator's statement that listed backup sets lost one old key.""" + require_local_console(self.config_dir) + from msg.admin.backup_retirement import unsigned_statement + from msg.security.backup_retirement import sign_record + target=Path(destination) + require(not target.exists() and not target.is_symlink(),'backup_retirement_destination_exists') + body=unsigned_statement(loads(Path(source).read_bytes())) + print(canonical({'statement':body,'digest':digest(body), + 'claim':'listed_backup_sets_only'}).decode()) + require(input('Type ATTEST BACKUP RETIREMENT '+digest(body)+': ')== + 'ATTEST BACKUP RETIREMENT '+digest(body),'approval_cancelled') + private=open_private_key(loads(root_envelope(self.config_dir).read_bytes()),getpass.getpass('Root PIN/passphrase: ')) + record=sign_record(body,Ed25519Signer.from_bytes(private)) + durable_write(target,canonical(record),mode=0o600) + return {'status':'backup_retirement_signed','path':str(target),'digest':digest(record)} + + def import_backup_retirement(self,source): + """Only this console path persists a record; reads re-verify it every time.""" + operator=require_local_console(self.config_dir) + from msg.admin.backup_retirement import import_record + app=self._app() + record=loads(Path(source).read_bytes()) + print(canonical({'record':record,'digest':digest(record)}).decode()) + require(input('Type IMPORT BACKUP RETIREMENT '+digest(record)+': ')== + 'IMPORT BACKUP RETIREMENT '+digest(record),'approval_cancelled') + async def run(): + await app.load() + try: + async with app.metadata.transaction(write=True) as tx: + return await import_record(tx,record,now=app.clock(),operator=operator) + finally: + await app.close() + attestation=asyncio.run(run()) + return {'status':'backup_retirement_imported','scope':'listed_backup_sets_only', + **attestation.view()} + def backup(self,destination): require_local_console(self.config_dir) target=Path(destination) diff --git a/src/msg/plugins/custodial_lifecycle.py b/src/msg/plugins/custodial_lifecycle.py index 20e76017..c933cd88 100644 --- a/src/msg/plugins/custodial_lifecycle.py +++ b/src/msg/plugins/custodial_lifecycle.py @@ -66,7 +66,7 @@ async def transition(app, ctx, request, tx, subject): else: require(hmac.compare_digest(details['age_possession_digest'], digest(args['age_proof'])), 'age_possession_failed') - state = await snapshot(tx, subject.resource_id, row[1], challenge, details) + state = await snapshot(tx, subject.resource_id, row[1], challenge, details, now=ctx.now) state.require_observation(args) before = digest(details) if args['action'] == 'refresh': @@ -142,7 +142,8 @@ async def transition(app, ctx, request, tx, subject): await audit(tx, ctx, request, subject.resource_id, 'identity.custodial_' + args['action'], {'challenge_id': args['challenge_id'], 'before_digest': before, 'after_digest': digest(details), 'decision_signature': wire(args['migration_ack'])}) - return HandlerOutput(data=(await snapshot(tx, subject.resource_id, row[1], challenge, details)).data) + return HandlerOutput(data=(await snapshot(tx, subject.resource_id, row[1], challenge, details, + now=ctx.now)).data) async def switch_identity(app, ctx, request, tx, subject, challenge, details, *, retain_decrypt=False): @@ -204,7 +205,7 @@ async def switch_identity(app, ctx, request, tx, subject, challenge, details, *, tx.execute('''UPDATE custodial_vault SET status='destroyed',signing_nonce=NULL, signing_ciphertext=NULL,age_nonce=NULL,age_ciphertext=NULL,destroyed_at=? WHERE subject=?''', (wire(ctx.now), uid), write=True) - current = await snapshot(tx, uid, 'completed', challenge, details) + current = await snapshot(tx, uid, 'completed', challenge, details, now=ctx.now) completed.update(completion_scope='identity_switch_only', phase=current.data['phase'], history_recoverable=current.data['history_recoverable'], verification_scope=current.data['verification_scope'], diff --git a/src/msg/plugins/recovery.py b/src/msg/plugins/recovery.py index e1359f5d..1b76240a 100644 --- a/src/msg/plugins/recovery.py +++ b/src/msg/plugins/recovery.py @@ -40,7 +40,8 @@ async def migration(ctx, request, tx, *, pending=False): challenge, details = loads(row[2]), loads(row[3]) authorize_migration(ctx, subject, row[0], row[1], details) require(not pending or row[1] != 'pending', 'custodial_upgrade_not_pending_rewrap') - state = await migration_snapshot(tx, subject.resource_id, row[1], challenge, details) + state = await migration_snapshot(tx, subject.resource_id, row[1], challenge, details, + now=ctx.now) return subject, row[1], challenge, details, state @op('identity.custodial_upgrade_inventory',obj({'challenge_id':IDENTIFIER}, diff --git a/src/msg/security/backup_retirement.py b/src/msg/security/backup_retirement.py new file mode 100644 index 00000000..f0001e80 --- /dev/null +++ b/src/msg/security/backup_retirement.py @@ -0,0 +1,135 @@ +"""Root-signed attestation that listed backup sets no longer hold one old key. + +Only the physical-console import in msg.admin.backup_retirement persists a +record, under a settings key that no network operation writes. Every read +re-verifies the signature against the current, unrevoked root credential and the +exact subject and old key IDs; an absent, altered, expired or superseded-root +record is not evidence. The statement is an operator attestation for the listed +backup sets only, not a proof that no other copy of the key exists anywhere. +""" +from __future__ import annotations + +from dataclasses import dataclass +from datetime import datetime, timedelta + +from msg.constants import ROOT_SUBJECT +from msg.core.codec import canonical, decode, digest, parse_time, wire +from msg.core.errors import Failure, require +from msg.core.models import Signature +from msg.security.crypto import key_id, verify + +DOMAIN = 'msg-custodial-backup-retirement-v1' +PURPOSE = 'custodial-backup-retirement-v1' +PREFIX = 'custodial_backup_retirement:' +MAX_VALIDITY = timedelta(days=400) +FIELDS = frozenset({'domain', 'subject_id', 'old_identity_key_id', 'old_encryption_key_id', + 'backup_sets', 'attested_at', 'not_after', 'operator'}) + + +@dataclass(frozen=True) +class BackupRetirement: + """Constructed only by check(); a bare boolean or dict is never accepted.""" + subject_id: str + old_identity_key_id: str + old_encryption_key_id: str + backup_sets: tuple + attested_at: datetime + not_after: datetime + signer_key_id: str + record_digest: str + + def binds(self, subject_id, details): + return (self.subject_id == subject_id and + self.old_identity_key_id == details.get('old_identity_key_id') and + self.old_encryption_key_id == details.get('old_encryption_key_id')) + + def view(self): + return {'scope': 'listed_backup_sets_only', 'backup_sets': list(self.backup_sets), + 'attested_at': wire(self.attested_at), 'not_after': wire(self.not_after), + 'signer_key_id': self.signer_key_id, 'record_digest': self.record_digest} + + +def setting_key(subject_id, old_identity_key_id): + return PREFIX + subject_id + ':' + old_identity_key_id + + +def statement(subject_id, details, backup_sets, *, attested_at, not_after, operator): + return {'domain': DOMAIN, 'subject_id': subject_id, + 'old_identity_key_id': details['old_identity_key_id'], + 'old_encryption_key_id': details['old_encryption_key_id'], + 'backup_sets': sorted(set(backup_sets)), 'attested_at': wire(attested_at), + 'not_after': wire(not_after), 'operator': operator} + + +def sign_record(body, signer): + return {'statement': body, 'signature': wire(signer.sign(canonical(body), purpose=PURPOSE))} + + +async def root_verifier(tx): + """The key of the active, unrevoked root certificate; rotation retires old records.""" + try: + certificate_id = tx.setting('active_root_certificate') + certificate = await tx.certificate(certificate_id) + credential = await tx.credential(certificate.key_id) + require(certificate.subject_id == ROOT_SUBJECT and + not await tx.certificate_revoked(certificate_id) and + credential.subject_id == ROOT_SUBJECT and credential.kind == 'signing_key' and + credential.revoked_at is None and + key_id(credential.verifier) == credential.id == certificate.key_id, + 'backup_retirement_root_unavailable') + except (Failure, KeyError, TypeError, ValueError) as exc: + raise Failure('backup_retirement_root_unavailable') from exc + return credential.verifier + + +def _sets(value): + return (isinstance(value, list) and bool(value) and value == sorted(set(value)) and + all(isinstance(item, str) and 0 < len(item) <= 256 for item in value)) + + +def check(record, verifier, subject_id, details, *, now): + """Return the attestation only if this exact record is valid now for these old keys.""" + try: + require(isinstance(record, dict) and set(record) == {'statement', 'signature'}, + 'backup_retirement_invalid') + body = record['statement'] + require(isinstance(body, dict) and set(body) == FIELDS and body['domain'] == DOMAIN and + all(isinstance(body[name], str) and body[name] for name in + ('subject_id', 'old_identity_key_id', 'old_encryption_key_id', 'operator')) and + _sets(body['backup_sets']), 'backup_retirement_invalid') + attested, not_after = parse_time(body['attested_at']), parse_time(body['not_after']) + signature = decode(Signature, record['signature']) + verify(verifier, canonical(body), signature, purpose=PURPOSE) + require(attested < not_after <= attested + MAX_VALIDITY, 'backup_retirement_invalid') + except (Failure, KeyError, TypeError, ValueError) as exc: + raise Failure('backup_retirement_invalid') from exc + require(isinstance(details, dict) and body['subject_id'] == subject_id and + body['old_identity_key_id'] == details.get('old_identity_key_id') and + body['old_encryption_key_id'] == details.get('old_encryption_key_id'), + 'backup_retirement_binding_mismatch') + try: + current = attested <= now < not_after + except TypeError: + current = False + require(current, 'backup_retirement_expired') + return BackupRetirement(subject_id=subject_id, + old_identity_key_id=body['old_identity_key_id'], + old_encryption_key_id=body['old_encryption_key_id'], + backup_sets=tuple(body['backup_sets']), attested_at=attested, not_after=not_after, + signer_key_id=signature.key_id, record_digest=digest(record)) + + +async def verified(tx, subject_id, details, *, now): + """(status, attestation-or-None); any failure is reported as not retired.""" + old = details.get('old_identity_key_id') if isinstance(details, dict) else None + if not isinstance(subject_id, str) or not isinstance(old, str) or not old: + return 'unavailable', None + record = tx.setting(setting_key(subject_id, old)) + if record is None: + return 'unavailable', None + if now is None: + return 'clock_unavailable', None + try: + return 'verified', check(record, await root_verifier(tx), subject_id, details, now=now) + except Failure as exc: + return ('expired' if exc.code == 'backup_retirement_expired' else 'invalid'), None diff --git a/src/msg/security/custodial_migration.py b/src/msg/security/custodial_migration.py index 09f6ce58..bcff8257 100644 --- a/src/msg/security/custodial_migration.py +++ b/src/msg/security/custodial_migration.py @@ -14,6 +14,7 @@ from msg.core.models import ResourceRef, Signature from msg.security.crypto import key_id, verify from msg.security.age_keys import encryption_key_id, public_from_recipient +from msg.security.backup_retirement import BackupRetirement, verified as verified_backup class MigrationPhase(StrEnum): @@ -94,21 +95,25 @@ def output_refs(details): return result -def retirement_view(details, vault, *, identity_switched): +def retirement_view(details, vault, *, identity_switched, subject_id=None, backup=None, + backup_status='requires_offline_verification'): """An absent online key cannot prove that historical backups lost that key.""" signing_deleted = bool(vault and vault[1] is None and vault[2] is None) encryption_deleted = bool(vault and vault[3] is None and vault[4] is None) - # Only a separately validated, local retirement record may set this field. - # The network migration API deliberately never writes or accepts one. - backup = details.get('backup_retirement') + # Only a locally imported, root-signed record verified by + # msg.security.backup_retirement may set this field. The network migration + # API never writes or accepts one; details['backup_retirement'] is ignored. + retired = isinstance(backup, BackupRetirement) and backup.binds(subject_id, details) + online = bool(identity_switched and vault and vault[0] == 'destroyed' and + signing_deleted and encryption_deleted) return {'identity_switched': identity_switched, 'token_revoked': identity_switched, 'online_signing_key_deleted': signing_deleted, 'online_encryption_key_deleted': encryption_deleted, - 'backup_retired': False, - 'backup_status': 'requires_offline_verification', - 'backup_retirement_record': backup, - 'server_key_retired': False} + 'backup_retired': retired, + 'backup_status': 'local_attestation_verified' if retired else backup_status, + 'backup_retirement_record': backup.view() if retired else None, + 'server_key_retired': retired and online} @dataclass(frozen=True) @@ -166,7 +171,8 @@ async def verify_challenge_binding(tx, subject, status, challenge, details): 'custodial_encryption_target_changed') -async def snapshot(tx, subject, status, challenge, details): +async def snapshot(tx, subject, status, challenge, details, *, now=None): + """Without a trusted clock a backup attestation cannot be current, so it is not used.""" await verify_challenge_binding(tx, subject, status, challenge, details) frozen = details.get('age_inventory') require(isinstance(frozen, list), 'custodial_inventory_required') @@ -245,13 +251,19 @@ async def snapshot(tx, subject, status, challenge, details): vault = tx.one('''SELECT status,signing_nonce,signing_ciphertext,age_nonce,age_ciphertext FROM custodial_vault WHERE subject=?''', (subject,)) switched = status == 'completed' - retirement = retirement_view(details, vault, identity_switched=switched) + backup_status, backup = await verified_backup(tx, subject, details, now=now) if switched \ + else ('unavailable', None) + retirement = retirement_view(details, vault, identity_switched=switched, subject_id=subject, + backup=backup, backup_status='requires_offline_verification' if backup_status == 'unavailable' + else 'local_attestation_' + backup_status) complete = not drift and verified == set(sources) phase = MigrationPhase.CHALLENGE if status == 'pending' else MigrationPhase.MIGRATING if switched: phase = MigrationPhase.HISTORY_RECOVERABLE if complete else MigrationPhase.IDENTITY_SWITCHED if retirement['online_signing_key_deleted'] and retirement['online_encryption_key_deleted']: phase = MigrationPhase.ONLINE_KEY_RETIRED + if retirement['server_key_retired']: + phase = MigrationPhase.SERVER_KEY_RETIRED results = {'mappings': mappings, 'acks': acks, 'recovery_envelope': envelope, 'recovery_acks': recovery_acks, 'resolution': details.get('resolution')} return MigrationSnapshot({'challenge_id': challenge['challenge_id'], 'status': status, diff --git a/src/msg/security/custody_history.py b/src/msg/security/custody_history.py index 39297716..661504ae 100644 --- a/src/msg/security/custody_history.py +++ b/src/msg/security/custody_history.py @@ -9,6 +9,7 @@ from msg.core.errors import Failure from msg.core.models import Signature from msg.security.age_keys import encryption_key_id, public_from_recipient +from msg.security.backup_retirement import verified as verified_backup from msg.security.crypto import key_id, verify @@ -101,8 +102,12 @@ def history_evidence(subject_id, challenge_id, challenge, details, current): return result -async def retirement_evidence(tx, subject_id, details, *, history_recoverable): - """Separate observable online retirement from unproven backup destruction.""" +async def retirement_evidence(tx, subject_id, details, *, history_recoverable, now=None): + """Separate observable online retirement from locally attested backup retirement. + + backup_retired needs one exact root-signed record imported at the physical + console; server_key_retired additionally needs observable online retirement. + """ new_signing = tx.one('SELECT key_id FROM identity_keys WHERE subject=? AND is_primary=1', (subject_id,)) new_encryption = tx.one('SELECT key_id FROM encryption_subkeys WHERE subject=? AND is_primary=1', @@ -129,8 +134,13 @@ async def retirement_evidence(tx, subject_id, details, *, history_recoverable): online = (switched and destroyed and signing_revoked and all(row is not None and row[0] is not None and not row[1] for row in (old_signing, old_encryption))) + status, attestation = await verified_backup(tx, subject_id, details, now=now) + backup = attestation is not None and attestation.binds(subject_id, details) + server = online and backup return {'identity_switched': switched, 'history_recoverable': history_recoverable, - 'online_retired': online, 'backup_retired': False, 'server_key_retired': False, - 'backup_retirement_evidence': 'unavailable', - 'completion_status': ('pending_backup_retirement' if online and history_recoverable + 'online_retired': online, 'backup_retired': backup, 'server_key_retired': server, + 'backup_retirement_evidence': status if not backup else 'local_attestation_verified', + 'backup_retirement_record': attestation.view() if backup else None, + 'completion_status': ('server_key_retired' if server and history_recoverable + else 'pending_backup_retirement' if online and history_recoverable else 'pending_history' if switched else 'pending_identity_switch')} diff --git a/tests/test_backup_retirement_attestation.py b/tests/test_backup_retirement_attestation.py new file mode 100644 index 00000000..12cb14b8 --- /dev/null +++ b/tests/test_backup_retirement_attestation.py @@ -0,0 +1,318 @@ +"""Issue #68 slice: a physical-console, root-signed backup retirement record. + +Without that exact record backup_retired stays false; server_key_retired also +needs observable online retirement. Altered, expired, misbound, wrong-signer and +superseded-root records fail closed. No network path can write or supply one. +""" +from dataclasses import replace +from datetime import timedelta +import os +from pathlib import Path + +import pytest + +from msg.admin.backup_retirement import import_record +from msg.admin.root import RootAdmin +from msg.constants import ROOT_SPACE, ROOT_SUBJECT +from msg.core.codec import b64, canonical, loads, unb64, wire +from msg.core.errors import Failure +from msg.core.models import (Certificate, CapabilityGrant, Credential, IssuancePolicy, Scope, + Signature) +from msg.security.age_keys import generate_age_key, public_from_recipient +from msg.security.backup_retirement import (BackupRetirement, check, setting_key, sign_record, + statement) +from msg.security.certificates import sign_certificate +from msg.security.crypto import Ed25519Signer +from msg.security.custodial_migration import retirement_view +from msg.security.custody_history import retirement_evidence +from msg.storage.sqlite import FakeMetadataStore +from test_custodial_upgrade import begin, finish +from test_service import NOW, call + +SUBJECT = 'u_exit' +SETS = ['offsite-2026-09-weekly', 'pg-basebackup-2026-09-20'] + + +def root_certificate(root): + grant = CapabilityGrant(capability='cert.issue', version=1, + scope=Scope(resource_id=ROOT_SPACE, descendants=True), + operations=frozenset({'cert.publish@1'}), constraints={}) + return sign_certificate(Certificate(resource_id='cert_root', serial='cert_root', + subject_id=ROOT_SUBJECT, key_id=root.key_id, issuer_id=ROOT_SUBJECT, + parent_certificate_id=None, authority_sources=(), kind='ca', grants=(grant,), + not_before=NOW - timedelta(days=1), expires_at=NOW + timedelta(days=365), + target_service='http://testserver', delegation_depth=8, + issuance=IssuancePolicy(issue_grants=(grant,), max_cert_ttl_seconds=86400, + max_child_ca_depth=3, max_delegation_depth=8), + signature=Signature(key_id=root.key_id, algorithm='ed25519', value=b'')), root) + + +def credential(signer, subject, *, revoked=None): + return Credential(id=signer.key_id, subject_id=subject, kind='signing_key', + verifier=signer.public_key, ceiling=(), not_before=NOW - timedelta(days=1), + expires_at=None, revoked_at=revoked) + + +@pytest.fixture +async def exited(): + """An online-retired custodial exit plus an active root, all in a disposable store.""" + store = FakeMetadataStore() + root, old, new = (Ed25519Signer.generate() for _ in range(3)) + details = {'old_identity_key_id': old.key_id, 'new_identity_key_id': new.key_id, + 'old_encryption_key_id': 'e_old', 'new_encryption_key_id': 'e_new'} + async with store.transaction(write=True) as tx: + for key, primary in ((old, 0), (new, 1)): + tx.execute('INSERT INTO identity_keys VALUES(?,?,?,?,?,?)', + (key.key_id, SUBJECT, b64(key.public_key), wire(NOW), + None if primary else wire(NOW), primary), write=True) + for kid, primary in (('e_old', 0), ('e_new', 1)): + _, recipient = generate_age_key() + tx.execute('INSERT INTO encryption_subkeys VALUES(?,?,?,?,?,?,?)', + (kid, SUBJECT, recipient, b64(public_from_recipient(recipient)), wire(NOW), + None if primary else wire(NOW), primary), write=True) + for signer, subject, revoked in ((old, SUBJECT, NOW), (root, ROOT_SUBJECT, None)): + tx.execute('INSERT INTO credentials VALUES(?,?,?)', (signer.key_id, subject, + canonical(credential(signer, subject, revoked=revoked)).decode()), write=True) + tx.execute('''INSERT INTO custodial_vault(subject,signing_key_id,encryption_key_id, + status,created_at,destroyed_at) VALUES(?,?,?,?,?,?)''', + (SUBJECT, old.key_id, 'e_old', 'destroyed', wire(NOW), wire(NOW)), write=True) + tx.execute('''INSERT INTO custodial_upgrades(id,subject,credential_id,status,expires_at, + challenge,body) VALUES(?,?,?,?,?,?,?)''', ('cupg_exit', SUBJECT, 't_old', 'completed', + wire(NOW), '{}', canonical(details).decode()), write=True) + await tx.register_certificate(root_certificate(root), None, 0) + tx.set_setting('active_root_certificate', 'cert_root') + yield store, root, details + await store.close() + + +def signed(details, signer, *, subject=SUBJECT, attested=NOW - timedelta(hours=1), + not_after=NOW + timedelta(days=30), sets=SETS, **changes): + body = statement(subject, dict(details, **changes), sets, attested_at=attested, + not_after=not_after, operator='console-operator') + return sign_record(body, signer) + + +async def evidence(store, details, *, now=NOW): + async with store.transaction(write=False) as tx: + return await retirement_evidence(tx, SUBJECT, details, history_recoverable=True, now=now) + + +async def stored(store, details): + async with store.transaction(write=False) as tx: + return tx.setting(setting_key(SUBJECT, details['old_identity_key_id'])) + + +@pytest.mark.asyncio +async def test_without_record_backup_and_server_key_remain_false(exited): + store, _, details = exited + async with store.transaction(write=True) as tx: + tx.set_setting('backup_retired', True) # An arbitrary flag is not evidence. + result = await evidence(store, details) + assert result['online_retired'] and not result['backup_retired'] + assert result['server_key_retired'] is False + assert result['backup_retirement_evidence'] == 'unavailable' + assert result['completion_status'] == 'pending_backup_retirement' + + +@pytest.mark.asyncio +async def test_imported_root_signed_record_retires_exactly_that_backup_scope(exited): + store, root, details = exited + async with store.transaction(write=True) as tx: + attestation = await import_record(tx, signed(details, root), now=NOW, operator='uid:0:/dev/tty1') + assert attestation.backup_sets == tuple(SETS) + result = await evidence(store, details) + assert result['backup_retired'] is True and result['server_key_retired'] is True + assert result['completion_status'] == 'server_key_retired' + assert result['backup_retirement_record']['scope'] == 'listed_backup_sets_only' + assert result['backup_retirement_record']['backup_sets'] == SETS + assert not (await evidence(store, details, now=None))['backup_retired'] + other = dict(details, old_identity_key_id=Ed25519Signer.generate().key_id) + assert not (await evidence(store, other))['backup_retired'] + async with store.transaction(write=False) as tx: + audits = [loads(body) for (body,) in tx.rows('SELECT body FROM audit')] + assert [a['event']['type'] for a in audits] == ['root.custodial.backup_retirement'] + assert audits[0]['event']['data']['operator'] == 'uid:0:/dev/tty1' + + +@pytest.mark.asyncio +async def test_backup_attestation_alone_is_not_server_key_retirement(exited): + store, root, details = exited + async with store.transaction(write=True) as tx: + await import_record(tx, signed(details, root), now=NOW, operator='console') + tx.execute('UPDATE custodial_vault SET age_ciphertext=? WHERE subject=?', + ('surviving encrypted key', SUBJECT), write=True) + result = await evidence(store, details) + assert result['backup_retired'] is True and result['online_retired'] is False + assert result['server_key_retired'] is False + assert result['completion_status'] == 'pending_history' + + +def tampered(record): + body = dict(record['statement'], backup_sets=['a-different-set']) + return dict(record, statement=body) + + +@pytest.mark.asyncio +@pytest.mark.parametrize('make, code', [ + (lambda d, root: signed(d, Ed25519Signer.generate()), 'backup_retirement_invalid'), + (lambda d, root: tampered(signed(d, root)), 'backup_retirement_invalid'), + (lambda d, root: signed(d, root, not_after=NOW), 'backup_retirement_expired'), + (lambda d, root: signed(d, root, attested=NOW + timedelta(minutes=1), + not_after=NOW + timedelta(days=1)), 'backup_retirement_expired'), + (lambda d, root: signed(d, root, not_after=NOW + timedelta(days=500)), 'backup_retirement_invalid'), + (lambda d, root: signed(d, root, sets=[]), 'backup_retirement_invalid'), + (lambda d, root: signed(d, root, old_encryption_key_id='e_new'), 'backup_retirement_binding_mismatch'), + (lambda d, root: signed(d, root, subject='u_other'), 'backup_retirement_migration_not_found'), + (lambda d, root: dict(signed(d, root), backup_retired=True), 'backup_retirement_invalid'), + (lambda d, root: dict(signed(d, root), statement=dict(signed(d, root)['statement'], extra=1)), + 'backup_retirement_invalid'), + (lambda d, root: True, 'backup_retirement_invalid'), +]) +async def test_bad_records_are_rejected_at_import_and_never_stored(exited, make, code): + store, root, details = exited + with pytest.raises(Failure, match='^' + code + '$'): + async with store.transaction(write=True) as tx: + await import_record(tx, make(details, root), now=NOW, operator='console') + assert await stored(store, details) is None + assert not (await evidence(store, details))['backup_retired'] + + +@pytest.mark.asyncio +async def test_persisted_record_is_reverified_and_fails_closed(exited): + store, root, details = exited + good = signed(details, root) + async with store.transaction(write=True) as tx: + await import_record(tx, good, now=NOW, operator='console') + later = await evidence(store, details, now=NOW + timedelta(days=30)) + assert not later['backup_retired'] and later['backup_retirement_evidence'] == 'expired' + for bad in (tampered(good), signed(details, Ed25519Signer.generate()), + signed(details, root, old_encryption_key_id='e_new')): + async with store.transaction(write=True) as tx: + tx.set_setting(setting_key(SUBJECT, details['old_identity_key_id']), bad) + result = await evidence(store, details) + assert not result['backup_retired'] and not result['server_key_retired'] + assert result['backup_retirement_evidence'] == 'invalid' + async with store.transaction(write=True) as tx: + tx.set_setting(setting_key(SUBJECT, details['old_identity_key_id']), good) + assert (await evidence(store, details))['backup_retired'] + async with store.transaction(write=True) as tx: + root_credential = await tx.credential(root.key_id) + tx.execute('UPDATE credentials SET body=? WHERE id=?', (canonical( + replace(root_credential, revoked_at=NOW)).decode(), root.key_id), write=True) + rotated = await evidence(store, details) + assert not rotated['backup_retired'] and rotated['backup_retirement_evidence'] == 'invalid' + + +def test_retirement_view_ignores_network_body_and_caller_flags(): + root = Ed25519Signer.generate() + details = {'old_identity_key_id': 'k_old', 'old_encryption_key_id': 'e_old'} + record = signed(details, root) + attestation = check(record, root.public_key, SUBJECT, details, now=NOW) + destroyed = ('destroyed', None, None, None, None) + network = dict(details, backup_retirement=record) + for backup in (None, True, 'verified', {'backup_retired': True}, record): + view = retirement_view(network, destroyed, identity_switched=True, subject_id=SUBJECT, + backup=backup) + assert view['backup_retired'] is False and view['server_key_retired'] is False + assert view['backup_retirement_record'] is None + assert not retirement_view(details, destroyed, identity_switched=True, subject_id='u_other', + backup=attestation)['backup_retired'] + forged = BackupRetirement(**{**attestation.__dict__, 'old_encryption_key_id': 'e_other'}) + assert not retirement_view(details, destroyed, identity_switched=True, subject_id=SUBJECT, + backup=forged)['backup_retired'] + view = retirement_view(details, destroyed, identity_switched=True, subject_id=SUBJECT, + backup=attestation) + assert view['backup_retired'] and view['server_key_retired'] + assert view['backup_status'] == 'local_attestation_verified' + retained = retirement_view(details, ('decrypt_only', None, None, 'n', 'c'), + identity_switched=True, subject_id=SUBJECT, backup=attestation) + assert retained['backup_retired'] and not retained['server_key_retired'] + + +@pytest.mark.parametrize('euid, env, code', [ + (1000, {}, 'local_os_administrator_required'), + (0, {'SSH_CONNECTION': '203.0.113.1 5 198.51.100.2 22'}, 'remote_admin_forbidden'), +]) +def test_import_and_signing_require_the_physical_console(tmp_path, monkeypatch, euid, env, code): + import msg.admin.root as root_module + monkeypatch.setattr(root_module.os, 'geteuid', lambda: euid, raising=False) + for name in ('SSH_CONNECTION', 'SSH_CLIENT', 'SSH_TTY'): + monkeypatch.delenv(name, raising=False) + for name, value in env.items(): + monkeypatch.setenv(name, value) + source = tmp_path/'record.json' + source.write_bytes(b'{}') + admin = RootAdmin(tmp_path) + with pytest.raises(Failure, match='^' + code + '$'): + admin.import_backup_retirement(source) + with pytest.raises(Failure, match='^' + code + '$'): + admin.sign_backup_retirement(source, tmp_path/'signed.json') + assert not (tmp_path/'signed.json').exists() + + +def test_no_network_adapter_can_write_or_import_the_record(): + src = Path(__file__).resolve().parents[1]/'src'/'msg' + allowed = {src/'admin'/'backup_retirement.py', src/'admin'/'root.py', + src/'security'/'backup_retirement.py'} + for path in src.rglob('*.py'): + if path in allowed: + continue + text = path.read_text() + assert 'admin.backup_retirement' not in text, path + assert 'import_record' not in text, path + assert 'custodial_backup_retirement:' not in text, path + assert 'setting_key' not in text or 'backup_retirement' not in text, path + + +@pytest.mark.asyncio +async def test_network_inventory_reports_backup_retired_only_after_local_import(installed): + app, root = installed + created = await call(app, 'identity.custodial_create', {'handle': 'backup-exit', + 'nonce': b64(os.urandom(32)), 'recovery_secret': b64(os.urandom(32))}, contract_version=2) + subject = created.data['subject_id'] + token = (created.data['credential_id'], unb64(created.data['token'])) + signer = Ed25519Signer.generate() + age_key, recipient = generate_age_key() + challenge = await begin(app, subject, token, signer, recipient) + done = await finish(app, subject, token, signer, age_key, challenge.data) + assert done.status == 'ok' and done.data['status'] == 'completed' + args = {'challenge_id': challenge.data['challenge_id']} + + async def inventory(extra=None): + return await call(app, 'identity.custodial_upgrade_inventory', dict(args, **(extra or {})), + key=signer, subject=subject) + + before = await inventory() + assert before.status == 'ok', wire(before) + assert before.data['phase'] == 'online_key_retired' + assert before.data['retirement']['backup_retired'] is False + assert before.data['retirement']['server_key_retired'] is False + async with app.metadata.transaction(write=False) as tx: + details = loads(tx.one('SELECT body FROM custodial_upgrades WHERE id=?', + (args['challenge_id'],))[0]) + supplied = await inventory({'backup_retirement': signed(details, root, subject=subject)}) + assert supplied.status == 'error' + with pytest.raises(Failure, match='^backup_retirement_invalid$'): + async with app.metadata.transaction(write=True) as tx: + await import_record(tx, signed(details, Ed25519Signer.generate(), subject=subject), + now=NOW, operator='console') + assert (await inventory()).data['retirement']['backup_retired'] is False + record = signed(details, root, subject=subject) + async with app.metadata.transaction(write=True) as tx: + await import_record(tx, record, now=NOW, operator='console') + after = await inventory() + assert after.status == 'ok', wire(after) + assert after.data['retirement']['backup_retired'] is True + assert after.data['retirement']['server_key_retired'] is True + assert list(after.data['retirement']['backup_retirement_record']['backup_sets']) == SETS + assert after.data['phase'] == 'server_key_retired' + async with app.metadata.transaction(write=False) as tx: + result = await retirement_evidence(tx, subject, details, history_recoverable=True, now=NOW) + assert result['online_retired'] and result['backup_retired'] and result['server_key_retired'] + async with app.metadata.transaction(write=True) as tx: + tx.set_setting(setting_key(subject, details['old_identity_key_id']), tampered(record)) + damaged = await inventory() + assert damaged.status == 'ok', wire(damaged) + assert damaged.data['retirement']['backup_retired'] is False + assert damaged.data['retirement']['backup_status'] == 'local_attestation_invalid' + assert damaged.data['phase'] == 'online_key_retired' From 1ca13fdf5ff71e0cbdca598a80f1cafb2561cbaa Mon Sep 17 00:00:00 2001 From: LIghtJUNction Date: Tue, 29 Sep 2026 03:30:29 +0800 Subject: [PATCH 5/7] fix: reject stale Root authority and replay bans without inherited expiry Fix the 15 failures reproduced by cloud run 36471956362 without changing any regression assertions. Keep current-authority reads mutation-free and recovery quarantined. Use time differences to avoid datetime overflow. Refs #68 #69 #70 #83 #85. --- docs/ISSUE_PROGRESS.md | 23 ++++++++++++++++++----- src/msg/admin/backup_retirement.py | 2 +- src/msg/admin/recovery_replay.py | 8 +++++--- src/msg/security/backup_retirement.py | 22 +++++++++++++++------- 4 files changed, 39 insertions(+), 16 deletions(-) diff --git a/docs/ISSUE_PROGRESS.md b/docs/ISSUE_PROGRESS.md index 66aca2eb..ee4980c2 100644 --- a/docs/ISSUE_PROGRESS.md +++ b/docs/ISSUE_PROGRESS.md @@ -1,14 +1,27 @@ # 当前 issue 推进笔记 -源请求：2026-09-28T19:16Z。此文件只记录本轮代码与验收证据；权威设计和已完成归档仍有效，入口见 issue #83。 +源请求：2026-09-28T19:16Z。此文件记录本轮代码与验收证据；权威设计和已完成归档仍有效，入口见 issue #83。只更新当前笔记，不重复堆叠状态文件。 ## 基线及集成 - 开始 main：`01e39345c755d6ec8f584534de20d9cd265ca922`，tree `f51cadf931d32c53c64501af8db33dbde43eacbb`。 -- 主线 CI `36465610565`、Recovery safety `36465610588` 已读为 success。 -- 保留 #145 `f2ebc093fb8333c2a0f196b6ebd86cfdf717affa` 与 #154 `3490750dd8fd59c8070e9816a104292db0937cea` 的提交历史，不修改原分支。 -- 本轮先加入失败回归：旧禁言过期时间、当前 Root 证书/凭据有效期与结构/签名、日期上溢。新工作流在云端真实 PostgreSQL 运行；原完整四分片、逐 node-ID gate、conformance、构建与恢复专项保持不变。 -- 当前为复现阶段，未声称测试通过或合并完成。运行的 source.json 绑定实际 checkout SHA/tree、原 PR head、run/attempt；PR 合并候选与原分支 head 分开记录。 +- 主线 Rewrite CI `36465610565`、Recovery safety `36465610588` 已读为 success。 +- 集成分支：`integrate/issue-recovery-cloud-20260928`。以 main 为第一父提交，完整保留 #145 `f2ebc093fb8333c2a0f196b6ebd86cfdf717affa` 与 #154 `3490750dd8fd59c8070e9816a104292db0937cea` 的历史，不修改原分支。 +- 完整读过权威设计及完成归档、两份 PR 的改动及 review/CI；旧 #103/#112 不再作为开放 PR 队列。 + +## 已复现与本次修复 + +失败基线：head `4bfeece690a26970b66f86b06cd3817dbb0b38cd`，tree `9f2b8db3861e8e299c6522d21a18d03d3d4b5dcd`。 + +云端专项 `36471956362` / job `109096152390` 使用真实 PostgreSQL 和 Python 3.15.0rc2，55 个用例中 **15 failed / 40 passed / 0 skipped / 0 errors**；原有两文件 36 个用例全部通过。已下载 artifact `10992276490`，实际核对 ZIP SHA256 `01d114f54d75777640e49df670f7a0dbb0c7f9404c726768065daeee201fe9d4`，source.json 与以上 head/tree 一致。 + +本次仅修实现，不修改失败断言： + +1. `topic_ban.apply` 不再继承旧快照中的 expires_at；无到期字段的检查点恢复为 active/NULL，保留原 actor/reason，重复与并发重放仍只改变一次。 +2. 退役证明读取和导入共同重验当前 Root 自签证书结构、签名、有效期及 signing credential 有效期；到期边界不再误报 verified，失败读取不写业务状态，失败导入不写审计。 +3. 有效期用时间差而非向不可信日期加 400 天，避免 year 9999 上溢；未来证明仍受当前时间检查拒绝。 + +本次修复后的 CI 尚待回读，不将预期通过写成通过。专项 source.json 绑定实际 checkout SHA/tree、PR head、run/attempt；完整四分片、逐 node-ID gate、conformance、构建及恢复专项不变。下一步检查本机命令接入及安全文件边界，再回读最终组合 head 的完整 CI。 ## 全部开放项（尚无整项关闭证据） diff --git a/src/msg/admin/backup_retirement.py b/src/msg/admin/backup_retirement.py index adb6b164..ef0ca393 100644 --- a/src/msg/admin/backup_retirement.py +++ b/src/msg/admin/backup_retirement.py @@ -36,7 +36,7 @@ async def import_record(tx, record, *, now, operator): isinstance(body.get('old_identity_key_id'), str), 'backup_retirement_invalid') subject = body['subject_id'] details = completed_migration(tx, subject, body['old_identity_key_id']) - attestation = check(record, await root_verifier(tx), subject, details, now=now) + attestation = check(record, await root_verifier(tx, now=now), subject, details, now=now) key = setting_key(subject, attestation.old_identity_key_id) previous = tx.setting(key) tx.set_setting(key, record) diff --git a/src/msg/admin/recovery_replay.py b/src/msg/admin/recovery_replay.py index f0a61b1e..573f8121 100644 --- a/src/msg/admin/recovery_replay.py +++ b/src/msg/admin/recovery_replay.py @@ -165,15 +165,17 @@ async def _apply(tx, fact): if kind == 'topic_ban.apply': require(tx.one('SELECT id FROM resources WHERE id=?', (target,)) is not None, 'recovery_fact_missing') - row = tx.one('SELECT status FROM topic_bans WHERE topic=? AND subject=?', (target, subject)) + row = tx.one('SELECT status,expires_at FROM topic_bans WHERE topic=? AND subject=?', (target, subject)) if row is None: tx.execute('''INSERT INTO topic_bans (topic,subject,actor,created_at,expires_at,reason,status) VALUES (?,?,?,?,NULL,?,?)''', (target, subject, ROOT_SUBJECT, at, 'recovery-replay', 'active'), write=True) return True - if row[0] != 'active': - tx.execute("UPDATE topic_bans SET status='active' WHERE topic=? AND subject=?", + # This checkpoint fact has no expiry. Do not inherit one from an older + # snapshot and silently lift the restored deny, now or in the future. + if row[0] != 'active' or row[1] is not None: + tx.execute("UPDATE topic_bans SET status='active',expires_at=NULL WHERE topic=? AND subject=?", (target, subject), write=True) return True return False diff --git a/src/msg/security/backup_retirement.py b/src/msg/security/backup_retirement.py index f0001e80..84c54ea9 100644 --- a/src/msg/security/backup_retirement.py +++ b/src/msg/security/backup_retirement.py @@ -16,6 +16,7 @@ from msg.core.codec import canonical, decode, digest, parse_time, wire from msg.core.errors import Failure, require from msg.core.models import Signature +from msg.security.certificates import verify_certificate from msg.security.crypto import key_id, verify DOMAIN = 'msg-custodial-backup-retirement-v1' @@ -65,18 +66,24 @@ def sign_record(body, signer): return {'statement': body, 'signature': wire(signer.sign(canonical(body), purpose=PURPOSE))} -async def root_verifier(tx): - """The key of the active, unrevoked root certificate; rotation retires old records.""" +async def root_verifier(tx, *, now): + """Recheck the current self-signed Root and its credential, not just historical validity.""" try: certificate_id = tx.setting('active_root_certificate') certificate = await tx.certificate(certificate_id) credential = await tx.credential(certificate.key_id) - require(certificate.subject_id == ROOT_SUBJECT and + require(certificate.resource_id == certificate_id and + certificate.subject_id == certificate.issuer_id == ROOT_SUBJECT and + certificate.parent_certificate_id is None and certificate.kind == 'ca' and + certificate.issuance is not None and + certificate.not_before <= now < certificate.expires_at and not await tx.certificate_revoked(certificate_id) and credential.subject_id == ROOT_SUBJECT and credential.kind == 'signing_key' and - credential.revoked_at is None and + credential.revoked_at is None and credential.not_before <= now and + (credential.expires_at is None or now < credential.expires_at) and key_id(credential.verifier) == credential.id == certificate.key_id, 'backup_retirement_root_unavailable') + verify_certificate(certificate, credential.verifier) except (Failure, KeyError, TypeError, ValueError) as exc: raise Failure('backup_retirement_root_unavailable') from exc return credential.verifier @@ -100,8 +107,9 @@ def check(record, verifier, subject_id, details, *, now): attested, not_after = parse_time(body['attested_at']), parse_time(body['not_after']) signature = decode(Signature, record['signature']) verify(verifier, canonical(body), signature, purpose=PURPOSE) - require(attested < not_after <= attested + MAX_VALIDITY, 'backup_retirement_invalid') - except (Failure, KeyError, TypeError, ValueError) as exc: + # Subtract instead of adding to an untrusted date near datetime.max. + require(timedelta(0) < not_after - attested <= MAX_VALIDITY, 'backup_retirement_invalid') + except (Failure, KeyError, TypeError, ValueError, OverflowError) as exc: raise Failure('backup_retirement_invalid') from exc require(isinstance(details, dict) and body['subject_id'] == subject_id and body['old_identity_key_id'] == details.get('old_identity_key_id') and @@ -130,6 +138,6 @@ async def verified(tx, subject_id, details, *, now): if now is None: return 'clock_unavailable', None try: - return 'verified', check(record, await root_verifier(tx), subject_id, details, now=now) + return 'verified', check(record, await root_verifier(tx, now=now), subject_id, details, now=now) except Failure as exc: return ('expired' if exc.code == 'backup_retirement_expired' else 'invalid'), None From f317920c06d4c2ecd9df9d72f4099604b5d4eeed Mon Sep 17 00:00:00 2001 From: LIghtJUNction Date: Tue, 29 Sep 2026 04:31:40 +0800 Subject: [PATCH 6/7] fix: verify active root certificate with existing primitives --- src/msg/security/backup_retirement.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/msg/security/backup_retirement.py b/src/msg/security/backup_retirement.py index 84c54ea9..17d0dad9 100644 --- a/src/msg/security/backup_retirement.py +++ b/src/msg/security/backup_retirement.py @@ -16,7 +16,7 @@ from msg.core.codec import canonical, decode, digest, parse_time, wire from msg.core.errors import Failure, require from msg.core.models import Signature -from msg.security.certificates import verify_certificate +from msg.security.certificates import certificate_body from msg.security.crypto import key_id, verify DOMAIN = 'msg-custodial-backup-retirement-v1' @@ -83,7 +83,7 @@ async def root_verifier(tx, *, now): (credential.expires_at is None or now < credential.expires_at) and key_id(credential.verifier) == credential.id == certificate.key_id, 'backup_retirement_root_unavailable') - verify_certificate(certificate, credential.verifier) + verify(credential.verifier, canonical(certificate_body(certificate)),\n certificate.signature, purpose='certificate') except (Failure, KeyError, TypeError, ValueError) as exc: raise Failure('backup_retirement_root_unavailable') from exc return credential.verifier From 0e4dbe777053e354bcf4845b851a0c9b86ab8310 Mon Sep 17 00:00:00 2001 From: LIghtJUNction Date: Tue, 29 Sep 2026 04:33:19 +0800 Subject: [PATCH 7/7] fix: correct root certificate verification call syntax --- src/msg/security/backup_retirement.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/msg/security/backup_retirement.py b/src/msg/security/backup_retirement.py index 17d0dad9..fcd10c90 100644 --- a/src/msg/security/backup_retirement.py +++ b/src/msg/security/backup_retirement.py @@ -83,7 +83,8 @@ async def root_verifier(tx, *, now): (credential.expires_at is None or now < credential.expires_at) and key_id(credential.verifier) == credential.id == certificate.key_id, 'backup_retirement_root_unavailable') - verify(credential.verifier, canonical(certificate_body(certificate)),\n certificate.signature, purpose='certificate') + verify(credential.verifier, canonical(certificate_body(certificate)), + certificate.signature, purpose='certificate') except (Failure, KeyError, TypeError, ValueError) as exc: raise Failure('backup_retirement_root_unavailable') from exc return credential.verifier