From 453d6f858e19136261278c7042c13598035aef1a Mon Sep 17 00:00:00 2001 From: LIghtJUNction Date: Tue, 29 Sep 2026 04:44:09 +0800 Subject: [PATCH 01/11] test(core): expose executor and shared wire dependency boundaries Refs #156 #83 #85. Run regression-first on real PostgreSQL in cloud without weakening existing gates. --- .github/workflows/architecture-ports.yml | 81 ++++++++++++++ docs/ISSUE_PROGRESS.md | 29 +++++ tests/test_architecture_ports.py | 129 +++++++++++++++++++++++ 3 files changed, 239 insertions(+) create mode 100644 .github/workflows/architecture-ports.yml create mode 100644 docs/ISSUE_PROGRESS.md create mode 100644 tests/test_architecture_ports.py diff --git a/.github/workflows/architecture-ports.yml b/.github/workflows/architecture-ports.yml new file mode 100644 index 00000000..2d63634c --- /dev/null +++ b/.github/workflows/architecture-ports.yml @@ -0,0 +1,81 @@ +name: Architecture boundary regressions +on: + push: + branches: [fix/executor-core-20260929-2045] + pull_request: + paths: + - 'src/msg/**' + - 'tests/test_architecture_ports.py' + - '.github/workflows/architecture-ports.yml' + workflow_dispatch: +permissions: + contents: read +concurrency: + group: architecture-ports-${{ github.ref }} + cancel-in-progress: true +jobs: + focused: + runs-on: ubuntu-latest + timeout-minutes: 12 + services: + postgres: + image: postgres:16 + env: + POSTGRES_USER: msgtest + POSTGRES_PASSWORD: msgtest + POSTGRES_DB: postgres + ports: ['5432:5432'] + options: >- + --health-cmd "pg_isready -U msgtest" + --health-interval 5s --health-timeout 5s --health-retries 10 + valkey: + image: valkey/valkey:9.0.6 + ports: ['6379:6379'] + options: >- + --health-cmd "valkey-cli ping" + --health-interval 5s --health-timeout 5s --health-retries 10 + env: + MSG_TEST_POSTGRES_URL_TEMPLATE: postgresql://msgtest:msgtest@127.0.0.1:5432/{database} + MSG_TEST_VALKEY_URL: redis://127.0.0.1:6379/0 + steps: + - uses: actions/checkout@v7 + with: + persist-credentials: false + - uses: actions/setup-python@v7 + with: + python-version: '3.15' + allow-prereleases: true + cache: pip + - name: Install isolated test dependencies + run: | + sudo apt-get update -qq + sudo apt-get install -y -qq age + python -m pip install -e '.[dev]' + - name: Record source identity + env: + EVENT_HEAD: ${{ github.event.pull_request.head.sha || github.sha }} + run: | + mkdir -p artifacts + python - <<'PY' + import json, os, platform, subprocess + from pathlib import Path + def git(ref): + return subprocess.check_output(['git', 'rev-parse', ref], text=True).strip() + Path('artifacts/source.json').write_text(json.dumps({ + 'head': os.environ['EVENT_HEAD'], 'checkout': git('HEAD'), 'tree': git('HEAD^{tree}'), + 'run': os.environ['GITHUB_RUN_ID'], 'attempt': os.environ['GITHUB_RUN_ATTEMPT'], + 'python': platform.python_version(), 'scope': 'isolated-regression-not-production' + }, indent=2) + '\n') + PY + - name: Exercise boundaries and unchanged security contracts + run: >- + python -m pytest tests/test_architecture_ports.py tests/test_batch.py + tests/test_batch_secret_release.py tests/test_tools.py tests/test_webhook_domain_events.py + tests/test_read_projection_cache_transports.py tests/test_read_projection_cache_unit.py + tests/issue_78_80 --junitxml=artifacts/architecture-ports.xml + - uses: actions/upload-artifact@v4 + if: always() + with: + name: architecture-ports-evidence + path: artifacts/ + if-no-files-found: error diff --git a/docs/ISSUE_PROGRESS.md b/docs/ISSUE_PROGRESS.md new file mode 100644 index 00000000..e73da20e --- /dev/null +++ b/docs/ISSUE_PROGRESS.md @@ -0,0 +1,29 @@ +# Issue 推进笔记 + +更新入口：2026-09-28T20:34Z 用户请求。代码、回归、CI 与现场验收分开记录；不以局部测试关闭总 issue。 + +## 基线与协作 + +- main：476e64f04ed1d9c7dcab4a41c4f3cdb72faad6a7，tree 40ca207a3c49be347120e058e47ed328ec4e3098。 +- main 完整 CI36472507327、Recovery safety36472507300 已返回 success。 +- #155 独立推进到0e4dbe777053e354bcf4845b851a0c9b86ab8310，修正原验签导入与换行语法；专项36480031762 success。尚未独立核其产物；完整36480031758与主线冲突另核。 +- 本分支 fix/executor-core-20260929-2045 不写 #155/#145 原分支。只读源码导出工作流 work/issue-workspace-20260929 不进主线。 +- 已阅读权威设计、有效完成归档、仓库贡献规则及当前源码。#83旧正文不作为当前提交状态。 + +## 本批次 #156 + +- [x] 核对依赖：executor 反向导入 batch/discovery/communication，持有整棵 Application；packet 依赖 plugins.schemas，共同协议的所有权分散。 +- [x] 先提交独立架构及真实签名/投影/回放回归。 +- [ ] 云端记录失败基线（不是本地运行）。 +- [ ] 单一批处理/事件标识函数所有权；明确窄投影与事务通知接口；HTTP/SSH 同一装配配置。 +- [ ] 共用请求/结果、基础schema和wire codec中立化；URL专用限制仍由传输层负责。 +- [ ] 云端修复后聚焦回归、全四分片/精确node-ID/conformance/build/Recovery safety。 +- [ ] 精确头审查并合并；随后更新对应 issue。没有通过证据前不标完成。 + +## 尚需继续 + +#160由已存在的fix/architecture-boundaries-20260929分支推进，本会话不覆盖。 + +#157 存储中立会话；#158 客户端/服务端依赖；#159 市场插件依赖；#68–70恢复本机命令、当前权限与现场验收；#71–72市场矩阵；#76–82权限/个人空间/读取/内容/存储/通知/客户端验收；#83配置/Registry/逐条证据；#85其余模型收敛。 + +#64/#65/#84缺实际授权的生产代理/日志、真实旧库快照、容量与恢复材料，不能用CI替代。#68/#69/#70的独立新鲜检查点、备份退役、真实控制台与promotion仍必须分别验收。本次不部署、不操作真实Root/PIN、资金、外发或备份销毁。 diff --git a/tests/test_architecture_ports.py b/tests/test_architecture_ports.py new file mode 100644 index 00000000..38fc27e3 --- /dev/null +++ b/tests/test_architecture_ports.py @@ -0,0 +1,129 @@ +"""Core/adapter boundaries plus real signed calls through the composed ports.""" +import ast +import importlib +import importlib.util +import inspect +from datetime import timedelta +from pathlib import Path +import subprocess +import sys + +import pytest + +from msg.core.codec import digest, wire +from msg.core.executor import OperationExecutor +from msg.core.requests import request_for +from test_service import NOW, register + + +ROOT = Path(__file__).resolve().parents[1] / 'src' / 'msg' + + +def imports(path): + tree = ast.parse(path.read_text()) + return [node.module for node in ast.walk(tree) + if isinstance(node, ast.ImportFrom) and node.module] + + +def test_executor_has_no_plugin_or_application_backdoor(): + source = (ROOT / 'core/executor.py').read_text() + assert not any(name.startswith(('msg.plugins', 'msg.application')) + for name in imports(ROOT / 'core/executor.py')) + assert not any(isinstance(node, ast.Attribute) and node.attr == 'application' + for node in ast.walk(ast.parse(source))) + + +def test_batch_rules_and_event_identity_have_one_neutral_owner(): + assert importlib.util.find_spec('msg.core.batching') is not None + assert importlib.util.find_spec('msg.core.events') is not None + batching = importlib.import_module('msg.core.batching') + events = importlib.import_module('msg.core.events') + from msg.plugins import batch, communication + assert batch.packets is batching.packets + assert communication.event_id is events.event_id + assert not any(name.startswith('msg.plugins') for name in imports(ROOT / 'core/batching.py')) + request = request_for('content.post_create', {}, 'https://example.invalid', + subject='u_test', request_id='stable-request') + assert events.event_id(request, 'u_test') == 'e_' + digest(('u_test', 'stable-request'))[7:39] + + +def test_core_protocol_imports_without_loading_plugins_or_transports(): + code = """ +import importlib.abc +import sys +class NoOuterLayer(importlib.abc.MetaPathFinder): + def find_spec(self, fullname, path=None, target=None): + if fullname.startswith(('msg.plugins', 'msg.transports', 'msg.application')): + raise AssertionError('outer layer imported: ' + fullname) +sys.meta_path.insert(0, NoOuterLayer()) +from msg.core import executor, batching, packet +assert executor.result_wire is packet.result_wire +""" + result = subprocess.run([sys.executable, '-c', code], text=True, capture_output=True) + assert result.returncode == 0, result.stderr + + +def test_wire_contracts_and_primitive_schemas_are_single_source(): + assert importlib.util.find_spec('msg.core.packet') is not None + assert importlib.util.find_spec('msg.core.schemas') is not None + packet = importlib.import_module('msg.core.packet') + schemas = importlib.import_module('msg.core.schemas') + from msg.transports import packet as transport + from msg.plugins import schemas as plugin_schemas + for name in ('REQUEST_SCHEMA', 'RESULT_SCHEMA', 'decode_packet', 'decode_result'): + assert getattr(packet, name) is getattr(transport, name) + assert schemas.obj is plugin_schemas.obj + assert schemas.SIGNATURE is plugin_schemas.SIGNATURE + + +@pytest.mark.asyncio +async def test_composition_reuses_ports_for_alternate_authenticator(installed): + app, _ = installed + assert hasattr(app, 'new_executor') + executor = app.new_executor(app.authenticator) + assert not hasattr(executor, 'application') + assert executor.max_request_bytes == app.settings.server.limits.max_request_bytes + assert executor.result_projection == app.executor.result_projection + assert executor.event_notifications == app.executor.event_notifications + assert executor.response_hook == app.executor.response_hook + assert executor.recovery_drill_marker == app.executor.recovery_drill_marker + app.executor.recovery_quarantined = True + assert app.new_executor(app.authenticator).recovery_quarantined is True + + +@pytest.mark.asyncio +async def test_standalone_executor_projection_failure_rolls_back(installed): + app, _ = installed + key, uid, _ = await register(app, 'port-agent') + executor = OperationExecutor(app.registry, app.metadata, app.contents, app.authenticator, + app.authorizer, app.clock, app.receipt_signer) + packet = request_for('content.post_create', {'parent': '/main', 'body': 'not published'}, + app.settings.service_url, subject=uid, signer=key, request_id='missing-projection-port', + return_fields=('id',), expires_at=NOW + timedelta(seconds=90)) + result = await executor.execute(packet) + assert result.status == 'error' and result.error.code == 'projection_unavailable', wire(result) + async with app.metadata.transaction(write=False) as tx: + assert tx.one("SELECT COUNT(*) FROM resources WHERE type='post'")[0] == 0 + assert tx.one('SELECT COUNT(*) FROM results WHERE request_id=?', (packet.request_id,))[0] == 0 + + +@pytest.mark.asyncio +async def test_explicit_projection_port_preserves_signed_execution_and_replay(installed): + app, _ = installed + assert 'result_projection' in inspect.signature(OperationExecutor).parameters + key, uid, _ = await register(app, 'projected-agent') + executor = OperationExecutor(app.registry, app.metadata, app.contents, app.authenticator, + app.authorizer, app.clock, app.receipt_signer, + max_request_bytes=app.settings.server.limits.max_request_bytes, + result_projection=app.executor.result_projection, + event_notifications=app.executor.event_notifications) + packet = request_for('content.post_create', {'parent': '/main', 'body': 'projected'}, + app.settings.service_url, subject=uid, signer=key, request_id='explicit-projection-port', + return_fields=('id',), expires_at=NOW + timedelta(seconds=90)) + first = await executor.execute(packet) + replay = await executor.execute(packet) + assert first.status == replay.status == 'ok', wire(first) + assert first.data['projection'][0]['id'] == first.resources[0].id + assert replay.replayed and replay.resources == first.resources and replay.receipt == first.receipt + async with app.metadata.transaction(write=False) as tx: + assert tx.one("SELECT COUNT(*) FROM resources WHERE type='post'")[0] == 1 From 10f46fd300ef0fa4890ca0ea7aad3d10bca7f7a5 Mon Sep 17 00:00:00 2001 From: LIghtJUNction Date: Mon, 28 Sep 2026 20:48:19 +0000 Subject: [PATCH 02/11] refactor(core): isolate executor ports and shared wire contracts Refs #156 #83 #85. Preserve canonical encodings, batch exclusions, transactional notifications and public import aliases. --- docs/ISSUE_PROGRESS.md | 6 +++ src/msg/application.py | 28 +++++++++++-- src/msg/core/batching.py | 23 +++++++++++ src/msg/core/events.py | 6 +++ src/msg/core/execution_ports.py | 20 +++++++++ src/msg/core/executor.py | 41 +++++++++--------- src/msg/core/packet.py | 71 ++++++++++++++++++++++++++++++++ src/msg/core/schemas.py | 30 ++++++++++++++ src/msg/extensions/ssh.py | 7 +--- src/msg/plugins/batch.py | 25 +---------- src/msg/plugins/communication.py | 3 +- src/msg/plugins/schemas.py | 35 +++------------- src/msg/transports/packet.py | 71 ++++---------------------------- tests/issue_78_80/conftest.py | 9 +++- 14 files changed, 223 insertions(+), 152 deletions(-) create mode 100644 src/msg/core/batching.py create mode 100644 src/msg/core/events.py create mode 100644 src/msg/core/execution_ports.py create mode 100644 src/msg/core/packet.py create mode 100644 src/msg/core/schemas.py diff --git a/docs/ISSUE_PROGRESS.md b/docs/ISSUE_PROGRESS.md index e73da20e..357ffb2d 100644 --- a/docs/ISSUE_PROGRESS.md +++ b/docs/ISSUE_PROGRESS.md @@ -27,3 +27,9 @@ #157 存储中立会话；#158 客户端/服务端依赖；#159 市场插件依赖；#68–70恢复本机命令、当前权限与现场验收；#71–72市场矩阵；#76–82权限/个人空间/读取/内容/存储/通知/客户端验收；#83配置/Registry/逐条证据；#85其余模型收敛。 #64/#65/#84缺实际授权的生产代理/日志、真实旧库快照、容量与恢复材料，不能用CI替代。#68/#69/#70的独立新鲜检查点、备份退役、真实控制台与promotion仍必须分别验收。本次不部署、不操作真实Root/PIN、资金、外发或备份销毁。 + +## 当前提交进度 + +失败基线已提交453d6f858e19136261278c7042c13598035aef1a，tree ae0b4f5e31dc1844c1abd0142da898b75b7779d4。云端专项36481310584和完整36481310527已触发；回读时仍排队，未当作通过。 + +实现采用固定的ResultProjection和TransactionalEventNotifications接口，不引入任意插件钩子。core.batching/core.events/core.packet/core.schemas分别只有一份真实逻辑；旧公开导入路径兼容重导出。URL秘密检查和gzip只在传输层，未移到所有入口。Application.new_executor同时装配HTTP/SSH的请求限制、秘密释放和恢复隔离，不再给executor整棵Application。未运行本地项目测试，尚无修复后通过结论。 diff --git a/src/msg/application.py b/src/msg/application.py index 591920d2..1862a579 100644 --- a/src/msg/application.py +++ b/src/msg/application.py @@ -119,15 +119,35 @@ async def load(self): self.authenticator=AuthenticationService(self.registry,self.certificates,self.settings.service_url,self.clock, self.primary_ceiling,self.temporary_ceiling) self.authorizer=AuthorizationService(self.registry,self.certificates) - self.executor=OperationExecutor(self.registry,self.metadata,self.contents,self.authenticator,self.authorizer, - self.clock,self.receipt_signer) - self.executor.application=self + self.executor=self.new_executor(self.authenticator) self.executor.recovery_drill_marker=marker self.executor.recovery_quarantined=quarantined - self.executor.response_hook=self._secrets_for_caller self._loaded=True return self + def new_executor(self, authenticator=None): + """Compose identical limits/ports for HTTP and the restricted SSH entry.""" + executor=OperationExecutor(self.registry,self.metadata,self.contents, + self.authenticator if authenticator is None else authenticator, + self.authorizer,self.clock,self.receipt_signer, + max_request_bytes=self.settings.server.limits.max_request_bytes, + result_projection=self._result_projection, + event_notifications=self._event_notifications) + executor.response_hook=self._secrets_for_caller + executor.recovery_drill_marker=self.settings.config_dir/'recovery-drill.json' + if self.executor is not None: + executor.recovery_quarantined=self.executor.recovery_quarantined + return executor + + async def _result_projection(self, context, request, session, resource, *, fields): + from msg.plugins.discovery import read_projection + return await read_projection(self,context,request,session,resource.id, + revision=resource.revision,fields=fields) + + async def _event_notifications(self, session, event): + from msg.plugins.communication import enqueue_domain_webhooks + await enqueue_domain_webhooks(self,session,event) + async def online_issuer(self,tx): issuer=tx.setting('online_ca_certificate') require(issuer is not None,'issuer_not_ready') diff --git a/src/msg/core/batching.py b/src/msg/core/batching.py new file mode 100644 index 00000000..d1b7bf2f --- /dev/null +++ b/src/msg/core/batching.py @@ -0,0 +1,23 @@ +"""The one child-envelope validator for atomic and independent batches.""" +from msg.core.codec import canonical +from msg.core.errors import require +from msg.core.packet import decode_packet +from msg.core.requests import SECRET_DELIVERY_MIN_VERSION + +# Credential delivery has response-only secrets; it requires its own call. +NO_BATCH=frozenset(SECRET_DELIVERY_MIN_VERSION) | {'identity.register','identity.upgrade'} + + +def packets(registry,request,subject,max_bytes=None): + max_bytes=max_bytes or len(canonical(request)) + values=[] + for value in request.arguments['requests']: + packet=decode_packet(value,max_bytes) + spec=registry.operation(packet.operation,packet.contract_version) + require(spec.effect in {'read','transaction'} and not packet.operation.startswith('batch.') + and packet.operation not in NO_BATCH and 'network' in spec.entries,'operation_not_batchable') + require(packet.subject==subject,'batch_subject_mismatch') + values.append(packet) + require(len({p.request_id for p in values})==len(values),'duplicate_batch_request_id') + require(request.request_id not in {p.request_id for p in values},'recursive_request_id') + return values diff --git a/src/msg/core/events.py b/src/msg/core/events.py new file mode 100644 index 00000000..c43b242f --- /dev/null +++ b/src/msg/core/events.py @@ -0,0 +1,6 @@ +"""Deterministic committed-operation event identity shared by all producers.""" +from msg.core.codec import digest + + +def event_id(request, subject): + return 'e_' + digest((subject, request.request_id))[7:39] diff --git a/src/msg/core/execution_ports.py b/src/msg/core/execution_ports.py new file mode 100644 index 00000000..6f1bd92f --- /dev/null +++ b/src/msg/core/execution_ports.py @@ -0,0 +1,20 @@ +"""Fixed executor dependencies, not plugin hooks or a workflow interface. + +ResultProjection is a permission-checked read in the existing transaction. +TransactionalEventNotifications may enqueue durable jobs in that transaction; +it must never perform external I/O or commit it. Execution, auth and receipt +ownership remain in OperationExecutor. +""" +from typing import Protocol +from msg.core.contracts import MetadataSession +from msg.core.models import Event, ExecutionContext, JsonMap, OperationRequest, ResourceRef + + +class ResultProjection(Protocol): + async def __call__(self, context: ExecutionContext, request: OperationRequest, + session: MetadataSession, resource: ResourceRef, + *, fields: tuple[str, ...]) -> JsonMap: ... + + +class TransactionalEventNotifications(Protocol): + async def __call__(self, session: MetadataSession, event: Event) -> None: ... diff --git a/src/msg/core/executor.py b/src/msg/core/executor.py index d00ac08d..87ea5881 100644 --- a/src/msg/core/executor.py +++ b/src/msg/core/executor.py @@ -5,6 +5,10 @@ from dataclasses import replace from uuid import uuid4 from msg.core.codec import wire,digest +from msg.core.batching import packets +from msg.core.events import event_id +from msg.core.packet import result_wire +from msg.core.execution_ports import ResultProjection, TransactionalEventNotifications from msg.core.errors import Failure,require from msg.core.models import ExecutionContext,HandlerOutput,OperationResult,OperationError,Event,AccessRequirement from msg.core.requests import SECRET_DELIVERY_MIN_VERSION,receipt_bytes @@ -13,14 +17,21 @@ class OperationExecutor: - def __init__(self,registry,metadata,contents,authenticator,authorizer,clock,receipt_signer): + def __init__(self,registry,metadata,contents,authenticator,authorizer,clock,receipt_signer, + *, max_request_bytes: int | None = None, + result_projection: ResultProjection | None = None, + event_notifications: TransactionalEventNotifications | None = None): self.registry,self.metadata,self.contents=registry,metadata,contents self.authenticator,self.authorizer,self.clock=authenticator,authorizer,clock self.receipt_signer=receipt_signer self.response_hook=None self.recovery_drill_marker=None self.recovery_quarantined=False - self.application=None + require(max_request_bytes is None or (type(max_request_bytes) is int and max_request_bytes>0), + 'invalid_request_limit') + self.max_request_bytes=max_request_bytes + self.result_projection=result_projection + self.event_notifications=event_notifications def recovery_drill_active(self): marker=self.recovery_drill_marker @@ -49,9 +60,8 @@ async def execute(self,request, *, entry='network'): if spec.name=='batch.atomic': # Validate the child set before an old cached parent result # can bypass the handler's secret-delivery exclusion. - from msg.plugins.batch import packets packets(self.registry,request,principal.subject, - self.application.settings.server.limits.max_request_bytes) + self.max_request_bytes) context=ExecutionContext(request_id=request.request_id,principal=principal,entry=entry, now=self.clock(),deadline_monotonic=time.monotonic()+30) checks=await spec.requirements(request,session) @@ -94,9 +104,8 @@ async def execute(self,request, *, entry='network'): if exc.code not in {'not_found'}:raise output=await spec.handler(context,request,session) if request.return_fields: - from msg.plugins.discovery import read_projection - require(output.resources,'projection_unavailable') - projections=[await read_projection(self.application,context,request,session,ref.id,revision=ref.revision, + require(output.resources and self.result_projection is not None,'projection_unavailable') + projections=[await self.result_projection(context,request,session,ref, fields=request.return_fields) for ref in output.resources] output=replace(output,data={**(output.data or {}),'projection':projections}) if audited: @@ -114,14 +123,12 @@ async def execute(self,request, *, entry='network'): resources=output.resources,committed_at=self.clock() if spec.effect!='read' else None, data=output.data,output=output.output) if spec.effect!='read': - from msg.plugins.communication import event_id event=Event(id=event_id(request,principal.subject),type=spec.name,time=result.committed_at,request_id=request.request_id, actor=principal.actor,subject=principal.subject,resources=output.resources, data={'operation':spec.name}) await session.append_event(event) - from msg.plugins.communication import WEBHOOK_DOMAIN_EVENTS,enqueue_domain_webhooks - if event.type in WEBHOOK_DOMAIN_EVENTS and self.application is not None: - await enqueue_domain_webhooks(self.application,session,event) + if self.event_notifications is not None: + await self.event_notifications(session,event) result=replace(result,receipt=self.receipt_signer.sign(receipt_bytes(result),purpose='receipt')) await session.save_result(principal.subject,request.payload_digest,result) # Only after the enclosing transaction commits may success reach the adapter. @@ -150,12 +157,10 @@ async def _independent(self,request,spec,entry): Interrupted batches are resumed by resending the original signed children. No child is reported rolled back merely because its sibling failed. """ - from msg.plugins.batch import packets - from msg.plugins.communication import event_id async with self.metadata.transaction(write=True) as tx: principal=await self.authenticator.authenticate(request,tx,entry=entry) await self.authorizer._ceiling(principal,f'{spec.name}@{spec.version}',principal.subject,tx) - children=packets(self.registry,request,principal.subject) + children=packets(self.registry,request,principal.subject,self.max_request_bytes) existing=tx.one('SELECT digest FROM batches WHERE subject=? AND request_id=?',(principal.subject,request.request_id)) require(existing is None or existing[0]==request.payload_digest,'idempotency_conflict') previous=await tx.request_result(principal.subject,request.request_id,request.payload_digest) @@ -186,11 +191,3 @@ async def _independent(self,request,spec,entry): request_id=request.request_id,actor=principal.actor,subject=principal.subject,resources=result.resources,data={'operation':spec.name})) await tx.save_result(principal.subject,request.payload_digest,result) return replace(result,prefer_cli=request.source!='msg',cli_url='/AGENTS.md' if request.source!='msg' else None) - - -def result_wire(result): - value=wire(result,compact=True) - for name in ('replayed','prefer_cli'): - if not value.get(name): - value.pop(name,None) - return value diff --git a/src/msg/core/packet.py b/src/msg/core/packet.py new file mode 100644 index 00000000..97cc4b29 --- /dev/null +++ b/src/msg/core/packet.py @@ -0,0 +1,71 @@ +"""Transport-neutral request/result contracts and bounded canonical decoding.""" +from __future__ import annotations +from collections.abc import Mapping +import re +from msg.core.codec import decode, loads, canonical, wire +from msg.core.errors import require +from msg.core.models import OperationRequest, OperationResult +from msg.core.schemas import obj, IDENTIFIER, STRING, BYTES, REF, SIGNATURE + +REQUEST_SCHEMA=obj({ + 'request_id':{'type':'string','minLength':1,'maxLength':128},'protocol_version':{'const':1}, + 'operation':{'type':'string','pattern':'^[a-z][a-z0-9_]*(\\.[a-z][a-z0-9_]*)+$','maxLength':128}, + 'contract_version':{'type':'integer','minimum':1},'target_service':{'type':'string','maxLength':2048}, + 'subject':{'type':['string','null'],'maxLength':160},'arguments':{'type':'object'}, + 'expected_generations':{'type':'array','maxItems':64,'items':{'type':'array','prefixItems':[IDENTIFIER,{'type':'integer','minimum':0}], + 'minItems':2,'maxItems':2}}, + 'expires_at':{'type':['string','null'],'maxLength':40}, + 'payload_digest':{'type':'string','pattern':'^sha256:[0-9a-f]{64}$'}, + 'proof':{'anyOf':[{'type':'null'},obj({'signature':SIGNATURE,'certificates':{'type':'array','items':IDENTIFIER,'maxItems':16}},('signature','certificates')), + obj({'credential_id':IDENTIFIER,'token':BYTES},('credential_id','token'))]}, + 'return_fields':{'type':'array','maxItems':30,'items':STRING},'source':{'enum':['msg','manual','mcp','unknown']} +},('request_id','protocol_version','operation','contract_version','target_service','subject','arguments', + 'expected_generations','expires_at','payload_digest','proof')) +RESULT_SCHEMA={'type':'object','properties':{ + 'request_id':STRING,'operation':STRING,'status':{'enum':['ok','accepted','error','uncertain']}, + 'actor':STRING,'subject':STRING,'resources':{'type':'array','items':REF},'committed_at':STRING, + 'replayed':{'type':'boolean'},'receipt':SIGNATURE,'error':{'type':'object'},'data':{'type':'object'}, + 'output':REF,'prefer_cli':{'type':'boolean'},'cli_url':STRING}, + 'required':['request_id','operation','status'],'additionalProperties':False} + + +def decode_packet(value,max_bytes=1048576): + from jsonschema import Draft202012Validator + if isinstance(value,(bytes,str)): + require(len(value.encode() if isinstance(value,str) else value)<=max_bytes,'request_too_large') + value=loads(value) + value=wire(value) + require(len(canonical(value))<=max_bytes,'request_too_large') + error=next(Draft202012Validator(REQUEST_SCHEMA).iter_errors(value),None) + require(error is None,'invalid_request_envelope', '.'.join(map(str,error.path)) if error else None) + packet=decode(OperationRequest,value) + require(len(set(rid for rid,generation in packet.expected_generations))==len(packet.expected_generations), + 'duplicate_expected_generation') + return packet + + +def safe_error_code(value, fallback): + """Only bounded contract identifiers, never exception text or credential URLs.""" + return (value if isinstance(value, str) and + re.fullmatch(r'[a-z][a-z0-9_]{0,95}', value) else fallback) + + +def decode_result(value): + # Reject malformed signed/unsigned results rather than rewriting their error + # fields. Valid receipts and persisted result bytes are left unchanged. + require(isinstance(value, Mapping), 'invalid_result_envelope') + error = value.get('error') + if error is not None: + require(isinstance(error, Mapping) and safe_error_code(error.get('code'), None) is not None, + 'invalid_result_envelope') + # Compact results omit anonymous actor/subject; these required domain fields + # are restored explicitly, not inferred from a transport connection. + return decode(OperationResult,dict({'actor':None,'subject':None},**value)) + + +def result_wire(result): + value=wire(result,compact=True) + for name in ('replayed','prefer_cli'): + if not value.get(name): + value.pop(name,None) + return value diff --git a/src/msg/core/schemas.py b/src/msg/core/schemas.py new file mode 100644 index 00000000..97e00ffe --- /dev/null +++ b/src/msg/core/schemas.py @@ -0,0 +1,30 @@ +"""Small explicit JSON schemas; no open-ended executable configuration.""" +STRING={'type':'string'} +IDENTIFIER={'type':'string','minLength':1,'maxLength':160} +INTEGER={'type':'integer','minimum':0} +BOOLEAN={'type':'boolean'} +BYTES={'type':'string','pattern':'^[A-Za-z0-9_-]*$'} +REF={'type':'object','properties':{'id':IDENTIFIER,'revision':{'type':['string','null']}},'required':['id'],'additionalProperties':False} +SCOPE={'type':'object','properties':{'resource_id':IDENTIFIER,'descendants':BOOLEAN},'required':['resource_id'],'additionalProperties':False} +NETWORK_CONSTRAINTS={'type':'object','properties':{ + 'hosts':{'type':'array','items':STRING},'schemes':{'type':'array','items':{'enum':['http','https']}}, + 'methods':{'type':'array','items':{'enum':['GET','HEAD','POST','PUT','PATCH','DELETE','OPTIONS']}}, + 'ports':{'type':'array','items':{'type':'integer','minimum':1,'maximum':65535}}, + 'timeout_ms':{'type':'integer','minimum':1},'max_response_bytes':{'type':'integer','minimum':1}, + 'max_redirects':{'type':'integer','minimum':0}},'additionalProperties':False} +GRANT={'type':'object','properties':{'capability':IDENTIFIER,'version':{'const':1},'scope':SCOPE, + 'operations':{'type':'array','items':IDENTIFIER,'minItems':1,'uniqueItems':True}, + 'constraints':{'type':'object'}},'required':['capability','version','scope','operations','constraints'], + 'additionalProperties':False} +GRANTS={'type':'array','items':GRANT} +ISSUANCE={'type':['object','null'],'properties':{'issue_grants':GRANTS,'max_cert_ttl_seconds':INTEGER, + 'max_child_ca_depth':INTEGER,'max_delegation_depth':INTEGER}, + 'required':['issue_grants','max_cert_ttl_seconds','max_child_ca_depth','max_delegation_depth'],'additionalProperties':False} +SIGNATURE={'type':'object','properties':{'key_id':IDENTIFIER,'algorithm':{'const':'ed25519'},'value':BYTES}, + 'required':['key_id','algorithm','value'],'additionalProperties':False} +OUTPUT={'type':'object','properties':{'resources':{'type':'array','items':REF},'data':{'type':['object','null']}, + 'output':{'anyOf':[REF,{'type':'null'}]}},'required':['resources','data','output'],'additionalProperties':False} + + +def obj(properties=None,required=()): + return {'type':'object','properties':properties or {},'required':list(required),'additionalProperties':False} diff --git a/src/msg/extensions/ssh.py b/src/msg/extensions/ssh.py index d12baadc..1d438593 100644 --- a/src/msg/extensions/ssh.py +++ b/src/msg/extensions/ssh.py @@ -21,7 +21,7 @@ from msg.constants import ROOT_SUBJECT from msg.core.codec import canonical, decode, digest, wire from msg.core.errors import Failure, require -from msg.core.executor import OperationExecutor, result_wire +from msg.core.packet import result_wire from msg.core.models import Credential, Principal, Signature, SignatureProof, CapabilityGrant, HandlerOutput, EffectJob, ResourceRef from msg.core.requests import request_for, payload_fields from msg.plugins.common import operation_id, check_access, new_id, resolve @@ -229,10 +229,7 @@ async def forced_session(config_dir, credential_id): try: command = parse_command(os.environ.get('SSH_ORIGINAL_COMMAND', '')) auth = SSHAuthenticator(app, credential_id) - executor = OperationExecutor(app.registry, app.metadata, app.contents, auth, app.authorizer, app.clock, app.receipt_signer) - # The normal secret delivery hook remains gated by signed operations. - executor.application = app - executor.response_hook = app._secrets_for_caller + executor = app.new_executor(auth) async with app.metadata.transaction(write=False) as tx: principal = await ssh_principal(app, credential_id, tx) if command[0] in {'call','packet'}: diff --git a/src/msg/plugins/batch.py b/src/msg/plugins/batch.py index beb05dbf..b54122db 100644 --- a/src/msg/plugins/batch.py +++ b/src/msg/plugins/batch.py @@ -1,32 +1,11 @@ """Bounded batching is a protocol primitive, not an executable workflow.""" from __future__ import annotations -from msg.core.codec import wire,canonical from msg.core.errors import Failure,require -from msg.core.executor import result_wire -from msg.core.requests import SECRET_DELIVERY_MIN_VERSION +from msg.core.packet import result_wire, REQUEST_SCHEMA +from msg.core.batching import NO_BATCH, packets from msg.core.models import HandlerOutput from msg.plugins.common import registration,operation_id from msg.plugins.schemas import obj -from msg.transports.packet import REQUEST_SCHEMA,decode_packet - -# Credential delivery has response-only secrets; it requires its own call. -NO_BATCH=frozenset(SECRET_DELIVERY_MIN_VERSION) | {'identity.register','identity.upgrade'} - - -def packets(registry,request,subject,max_bytes=None): - max_bytes=max_bytes or len(canonical(request)) - values=[] - for value in request.arguments['requests']: - packet=decode_packet(value,max_bytes) - spec=registry.operation(packet.operation,packet.contract_version) - require(spec.effect in {'read','transaction'} and not packet.operation.startswith('batch.') - and packet.operation not in NO_BATCH and 'network' in spec.entries,'operation_not_batchable') - require(packet.subject==subject,'batch_subject_mismatch') - values.append(packet) - require(len({p.request_id for p in values})==len(values),'duplicate_batch_request_id') - require(request.request_id not in {p.request_id for p in values},'recursive_request_id') - return values - def install(app): op,finish=registration(app,'batch',('identity',)) diff --git a/src/msg/plugins/communication.py b/src/msg/plugins/communication.py index d704aa7e..a97ce67a 100644 --- a/src/msg/plugins/communication.py +++ b/src/msg/plugins/communication.py @@ -17,8 +17,7 @@ from msg.storage.capacity import require_webhook_capacity -def event_id(request,subject): - return 'e_'+digest((subject,request.request_id))[7:39] +from msg.core.events import event_id # A deliberately small event vocabulary. An Event is an audit fact, not blanket diff --git a/src/msg/plugins/schemas.py b/src/msg/plugins/schemas.py index 97e00ffe..275fc60e 100644 --- a/src/msg/plugins/schemas.py +++ b/src/msg/plugins/schemas.py @@ -1,30 +1,5 @@ -"""Small explicit JSON schemas; no open-ended executable configuration.""" -STRING={'type':'string'} -IDENTIFIER={'type':'string','minLength':1,'maxLength':160} -INTEGER={'type':'integer','minimum':0} -BOOLEAN={'type':'boolean'} -BYTES={'type':'string','pattern':'^[A-Za-z0-9_-]*$'} -REF={'type':'object','properties':{'id':IDENTIFIER,'revision':{'type':['string','null']}},'required':['id'],'additionalProperties':False} -SCOPE={'type':'object','properties':{'resource_id':IDENTIFIER,'descendants':BOOLEAN},'required':['resource_id'],'additionalProperties':False} -NETWORK_CONSTRAINTS={'type':'object','properties':{ - 'hosts':{'type':'array','items':STRING},'schemes':{'type':'array','items':{'enum':['http','https']}}, - 'methods':{'type':'array','items':{'enum':['GET','HEAD','POST','PUT','PATCH','DELETE','OPTIONS']}}, - 'ports':{'type':'array','items':{'type':'integer','minimum':1,'maximum':65535}}, - 'timeout_ms':{'type':'integer','minimum':1},'max_response_bytes':{'type':'integer','minimum':1}, - 'max_redirects':{'type':'integer','minimum':0}},'additionalProperties':False} -GRANT={'type':'object','properties':{'capability':IDENTIFIER,'version':{'const':1},'scope':SCOPE, - 'operations':{'type':'array','items':IDENTIFIER,'minItems':1,'uniqueItems':True}, - 'constraints':{'type':'object'}},'required':['capability','version','scope','operations','constraints'], - 'additionalProperties':False} -GRANTS={'type':'array','items':GRANT} -ISSUANCE={'type':['object','null'],'properties':{'issue_grants':GRANTS,'max_cert_ttl_seconds':INTEGER, - 'max_child_ca_depth':INTEGER,'max_delegation_depth':INTEGER}, - 'required':['issue_grants','max_cert_ttl_seconds','max_child_ca_depth','max_delegation_depth'],'additionalProperties':False} -SIGNATURE={'type':'object','properties':{'key_id':IDENTIFIER,'algorithm':{'const':'ed25519'},'value':BYTES}, - 'required':['key_id','algorithm','value'],'additionalProperties':False} -OUTPUT={'type':'object','properties':{'resources':{'type':'array','items':REF},'data':{'type':['object','null']}, - 'output':{'anyOf':[REF,{'type':'null'}]}},'required':['resources','data','output'],'additionalProperties':False} - - -def obj(properties=None,required=()): - return {'type':'object','properties':properties or {},'required':list(required),'additionalProperties':False} +"""Compatibility exports; shared wire schemas are owned by msg.core.schemas.""" +from msg.core.schemas import ( + STRING, IDENTIFIER, INTEGER, BOOLEAN, BYTES, REF, SCOPE, NETWORK_CONSTRAINTS, + GRANT, GRANTS, ISSUANCE, SIGNATURE, OUTPUT, obj, +) diff --git a/src/msg/transports/packet.py b/src/msg/transports/packet.py index 84dc47ea..dccaff9b 100644 --- a/src/msg/transports/packet.py +++ b/src/msg/transports/packet.py @@ -1,38 +1,15 @@ -"""Bounded wire decoding shared by every network adapter.""" +"""URL and compression boundaries; common wire contracts live in core.packet.""" from __future__ import annotations -from collections.abc import Mapping -import re import zlib -from msg.core.codec import decode,loads,unb64,canonical,wire -from msg.core.errors import require,Failure +from msg.core.codec import unb64 +from msg.core.errors import require, Failure from msg.core.requests import SECRET_DELIVERY_MIN_VERSION -from msg.core.models import OperationRequest,OperationResult,TokenProof -from msg.plugins.schemas import obj,IDENTIFIER,STRING,BYTES,REF,SIGNATURE +from msg.core.models import OperationRequest, TokenProof +from msg.core.packet import ( + REQUEST_SCHEMA, RESULT_SCHEMA, decode_packet, decode_result, safe_error_code, +) from msg.transports.url_safety import contains_secret_fields -REQUEST_SCHEMA=obj({ - 'request_id':{'type':'string','minLength':1,'maxLength':128},'protocol_version':{'const':1}, - 'operation':{'type':'string','pattern':'^[a-z][a-z0-9_]*(\\.[a-z][a-z0-9_]*)+$','maxLength':128}, - 'contract_version':{'type':'integer','minimum':1},'target_service':{'type':'string','maxLength':2048}, - 'subject':{'type':['string','null'],'maxLength':160},'arguments':{'type':'object'}, - 'expected_generations':{'type':'array','maxItems':64,'items':{'type':'array','prefixItems':[IDENTIFIER,{'type':'integer','minimum':0}], - 'minItems':2,'maxItems':2}}, - 'expires_at':{'type':['string','null'],'maxLength':40}, - 'payload_digest':{'type':'string','pattern':'^sha256:[0-9a-f]{64}$'}, - 'proof':{'anyOf':[{'type':'null'},obj({'signature':SIGNATURE,'certificates':{'type':'array','items':IDENTIFIER,'maxItems':16}},('signature','certificates')), - obj({'credential_id':IDENTIFIER,'token':BYTES},('credential_id','token'))]}, - 'return_fields':{'type':'array','maxItems':30,'items':STRING},'source':{'enum':['msg','manual','mcp','unknown']} -},('request_id','protocol_version','operation','contract_version','target_service','subject','arguments', - 'expected_generations','expires_at','payload_digest','proof')) -RESULT_SCHEMA={'type':'object','properties':{ - 'request_id':STRING,'operation':STRING,'status':{'enum':['ok','accepted','error','uncertain']}, - 'actor':STRING,'subject':STRING,'resources':{'type':'array','items':REF},'committed_at':STRING, - 'replayed':{'type':'boolean'},'receipt':SIGNATURE,'error':{'type':'object'},'data':{'type':'object'}, - 'output':REF,'prefer_cli':{'type':'boolean'},'cli_url':STRING}, - 'required':['request_id','operation','status'],'additionalProperties':False} - - - def require_url_safe_packet(packet: OperationRequest) -> None: """Reject credentials and recovery claims before a GET URL is dispatched. @@ -47,40 +24,6 @@ def require_url_safe_packet(packet: OperationRequest) -> None: require(not contains_secret_fields(packet.arguments), 'secure_channel_required') -def decode_packet(value,max_bytes=1048576): - from jsonschema import Draft202012Validator - if isinstance(value,(bytes,str)): - require(len(value.encode() if isinstance(value,str) else value)<=max_bytes,'request_too_large') - value=loads(value) - value=wire(value) - require(len(canonical(value))<=max_bytes,'request_too_large') - error=next(Draft202012Validator(REQUEST_SCHEMA).iter_errors(value),None) - require(error is None,'invalid_request_envelope', '.'.join(map(str,error.path)) if error else None) - packet=decode(OperationRequest,value) - require(len(set(rid for rid,generation in packet.expected_generations))==len(packet.expected_generations), - 'duplicate_expected_generation') - return packet - - -def safe_error_code(value, fallback): - """Only bounded contract identifiers, never exception text or credential URLs.""" - return (value if isinstance(value, str) and - re.fullmatch(r'[a-z][a-z0-9_]{0,95}', value) else fallback) - - -def decode_result(value): - # Reject malformed signed/unsigned results rather than rewriting their error - # fields. Valid receipts and persisted result bytes are left unchanged. - require(isinstance(value, Mapping), 'invalid_result_envelope') - error = value.get('error') - if error is not None: - require(isinstance(error, Mapping) and safe_error_code(error.get('code'), None) is not None, - 'invalid_result_envelope') - # Compact results omit anonymous actor/subject; these required domain fields - # are restored explicitly, not inferred from a transport connection. - return decode(OperationResult,dict({'actor':None,'subject':None},**value)) - - def gunzip(raw,limit): inflater=zlib.decompressobj(wbits=31) try: diff --git a/tests/issue_78_80/conftest.py b/tests/issue_78_80/conftest.py index ada0f4a7..578e5576 100644 --- a/tests/issue_78_80/conftest.py +++ b/tests/issue_78_80/conftest.py @@ -125,9 +125,14 @@ async def invoke(operation,args,version=1, *, expected=(),context=None): app.authorizer=AuthorizationService(registry,app.certificates) app.authenticator=AuthenticationService(registry,app.certificates,app.settings.service_url, app.clock,lambda:base_grants(registry),lambda:base_grants(registry)) + async def result_projection(context, request, session, resource, *, fields): + return await discovery.read_projection(app,context,request,session,resource.id, + revision=resource.revision,fields=fields) + app.executor=OperationExecutor(registry,metadata,app.contents,app.authenticator, - app.authorizer,app.clock,Ed25519Signer.generate()) - app.executor.application=app + app.authorizer,app.clock,Ed25519Signer.generate(), + max_request_bytes=app.settings.server.limits.max_request_bytes, + result_projection=result_projection) app._loaded=True def packet(operation,args,version=1, *, signed=True,**kwargs): From 7f8752e37fe5b5a8021f192b87e731caec2bbc2d Mon Sep 17 00:00:00 2001 From: LIghtJUNction Date: Tue, 29 Sep 2026 04:48:24 +0800 Subject: [PATCH 03/11] test: define backend-independent metadata session contracts --- .../workflows/metadata-session-boundary.yml | 66 +++++++ docs/METADATA_SESSION_PROGRESS.md | 27 +++ tests/test_metadata_session_boundary.py | 186 ++++++++++++++++++ 3 files changed, 279 insertions(+) create mode 100644 .github/workflows/metadata-session-boundary.yml create mode 100644 docs/METADATA_SESSION_PROGRESS.md create mode 100644 tests/test_metadata_session_boundary.py diff --git a/.github/workflows/metadata-session-boundary.yml b/.github/workflows/metadata-session-boundary.yml new file mode 100644 index 00000000..5a3bf086 --- /dev/null +++ b/.github/workflows/metadata-session-boundary.yml @@ -0,0 +1,66 @@ +name: Metadata session contracts +on: + push: + branches: [fix/metadata-session-boundary-20260929] + workflow_dispatch: +permissions: + contents: read +concurrency: + group: metadata-session-${{ github.ref }} + cancel-in-progress: false +jobs: + contracts: + runs-on: ubuntu-latest + timeout-minutes: 20 + services: + postgres: + image: postgres:16 + env: + POSTGRES_USER: msgtest + POSTGRES_PASSWORD: msgtest + POSTGRES_DB: postgres + ports: ['5432:5432'] + options: >- + --health-cmd "pg_isready -U msgtest" + --health-interval 10s --health-timeout 5s --health-retries 5 + env: + MSG_TEST_POSTGRES_URL_TEMPLATE: postgresql://msgtest:msgtest@127.0.0.1:5432/{database} + steps: + - uses: actions/checkout@v7 + with: + persist-credentials: false + - uses: actions/setup-python@v7 + with: + python-version: '3.15' + allow-prereleases: true + cache: pip + - name: Install isolated test dependencies + run: | + sudo apt-get update -qq + sudo apt-get install -y -qq age + python -m pip install -e '.[dev]' + - name: Record exact source + env: + SOURCE_SHA: ${{ github.sha }} + run: | + mkdir -p artifacts + test "$(git rev-parse HEAD)" = "$SOURCE_SHA" + python - <<'PY' + import json, os, subprocess + from pathlib import Path + git=lambda *args: subprocess.check_output(['git',*args],text=True).strip() + Path('artifacts/source.json').write_text(json.dumps({'commit':git('rev-parse','HEAD'), + 'tree':git('rev-parse','HEAD^{tree}'),'event_sha':os.environ['SOURCE_SHA'], + 'dirty':bool(git('status','--porcelain'))},indent=2)+'\n') + PY + - name: Run the same contract on both real database adapters + run: python -m pytest tests/test_metadata_session_boundary.py --junitxml=artifacts/metadata-session.xml + - name: Preserve PostgreSQL locks, migration and recovery regressions + run: | + python -m pytest tests/test_postgres.py tests/test_security_storage_rollback.py tests/test_security_rollback_fences.py tests/test_ledger_accounts.py tests/test_ledger_account_isolation.py tests/test_ledger_migration_json.py tests/test_ledger_migration_references.py tests/test_recovery_safety_units.py tests/test_recovery_metadata.py --junitxml=artifacts/metadata-regressions.xml + - uses: actions/upload-artifact@v4 + if: always() + with: + name: metadata-session-${{ github.sha }} + path: artifacts/ + if-no-files-found: error diff --git a/docs/METADATA_SESSION_PROGRESS.md b/docs/METADATA_SESSION_PROGRESS.md new file mode 100644 index 00000000..3d94c200 --- /dev/null +++ b/docs/METADATA_SESSION_PROGRESS.md @@ -0,0 +1,27 @@ +# Metadata session boundary progress + +## Coordination and baseline + +- Issue #157 only; broader #65/#69/#85 production and architectural acceptance remain separate. +- Base main: `476e64f04ed1d9c7dcab4a41c4f3cdb72faad6a7`, tree `40ca207a3c49be347120e058e47ed328ec4e3098`. +- Branch: `fix/metadata-session-boundary-20260929`. +- #83 comment 5878201914 records the coordination change. The overlapping regression-only #164 is closed without merging; #165 owns #156/#160 and #155 owns recovery. Do not overwrite either branch. +- Main source was read from CI36472507327 artifact10992168743. Its downloaded SHA-256 matches `9abbc69988b8ffcb45f9931dc87e291145a7680f275bf7aadbedb7bed2189d10`. + +## Stage 1: regression-first + +This commit adds 4 architectural contracts and 6 shared behavioral tests, each behavioral test parameterized over real PostgreSQL and SQLite (16 tests total). Source implementation is unchanged. The focused cloud workflow records exact commit/tree and JUnit; no new pass result is claimed before execution. + +Coverage: independent PostgreSQL import, neutral ownership, declared MetadataSession methods, resource generations/path/paging, persisted result/event/audit/job identities, savepoint compensation, read-only/closed/cross-task guards, unique-constraint rollback and driver parameter/literal handling. + +## Planned boundary + +Before: `PostgresSession -> SqliteSession -> SQLite connection + shared model operations`. + +After: PostgreSQL and SQLite independently implement the driver operation of a neutral MetadataSessionBase. That base owns only model/session invariants and rollback-callback ordering; it does not own a connection, transaction commit, schema, migration, placeholder translation or driver exception handling. + +PostgreSQL keeps its direct/session-pooled connection requirement, connection-scoped advisory lock held through compensation, nested savepoints, pending-effect signals only after commit and exact SQL/error adapter. SQLite keeps its own connection, pragmas, error mapping, writer file fence and test/fake stores. + +## Required evidence before closure + +Obtain actual red evidence, then apply the extraction without schema/migration/domain changes. Run the shared contracts on both adapters, the existing PostgreSQL concurrency/migration/recovery suites, and the complete four-shard node-ID/conformance/build gate on the final head. Verify the tested tree before merging. Production restore/backup/operator evidence is not simulated or marked complete by these tests. diff --git a/tests/test_metadata_session_boundary.py b/tests/test_metadata_session_boundary.py new file mode 100644 index 00000000..9a753fc7 --- /dev/null +++ b/tests/test_metadata_session_boundary.py @@ -0,0 +1,186 @@ +"""One storage contract exercised independently against PostgreSQL and SQLite.""" +from __future__ import annotations + +import asyncio +from dataclasses import replace +from datetime import UTC, datetime +import importlib +import inspect +from pathlib import Path +import subprocess +import sys + +import pytest + +from msg.core.contracts import MetadataSession, MetadataStore +from msg.core.errors import Failure +from msg.core.models import AuditEvent, EffectJob, Event, OperationResult, Principal +from test_postgres import item + + +def test_postgres_import_does_not_load_a_sqlite_adapter_or_driver(): + code = ''' +import importlib.abc, sys +class NoSqlite(importlib.abc.MetaPathFinder): + def find_spec(self, fullname, path=None, target=None): + if fullname == 'msg.storage.sqlite' or fullname == 'sqlite3' or fullname.startswith('sqlite3.'): + raise AssertionError('PostgreSQL imported SQLite: ' + fullname) +sys.meta_path.insert(0, NoSqlite()) +import msg.storage.postgres +assert 'msg.storage.sqlite' not in sys.modules +assert 'sqlite3' not in sys.modules +''' + result = subprocess.run([sys.executable, '-c', code], capture_output=True, text=True) + assert result.returncode == 0, result.stderr + + +def test_backends_share_only_the_neutral_session_implementation(): + from msg.storage.postgres import PostgresSession + from msg.storage.sqlite import SqliteSession + common = importlib.import_module('msg.storage.session').MetadataSessionBase + assert not issubclass(PostgresSession, SqliteSession) + assert PostgresSession.__bases__ == SqliteSession.__bases__ == (common,) + for name in ('resource','children','save_result','append_audit','on_rollback','check'): + assert getattr(PostgresSession, name) is getattr(SqliteSession, name) is getattr(common, name) + assert PostgresSession.execute is not SqliteSession.execute + assert 'execute' in common.__abstractmethods__ + + +def test_shared_layer_does_not_own_connections_dialects_schemas_or_commits(): + module = importlib.import_module('msg.storage.session') + text = Path(module.__file__).read_text() + for forbidden in ('sqlite3', 'psycopg', '_connection', 'PRAGMA ', 'CREATE TABLE ', + 'INSERT OR IGNORE', '.commit(', '.rollback(', 'pg_advisory'): + assert forbidden not in text + assert set(module.MetadataSessionBase.__abstractmethods__) == {'execute'} + assert set(inspect.signature(module.MetadataSessionBase).parameters) == {'write'} + + +def test_shared_implementation_covers_the_declared_metadata_contract(): + common = importlib.import_module('msg.storage.session').MetadataSessionBase + for name, member in vars(MetadataSession).items(): + if not name.startswith('_') and callable(member): + assert name in vars(common), name + assert callable(getattr(common, name)), name + assert 'transaction' in vars(MetadataStore) + assert {'one','rows','check','setting','set_setting','job','save_job', + 'organization','path','ancestors','run_rollback_effects'} <= set(vars(common)) + + +@pytest.fixture(params=['postgres','sqlite']) +async def metadata(request, pg_dsn, tmp_path): + from msg.storage.postgres import PostgresMetadataStore + from msg.storage.sqlite import SqliteMetadataStore + store = (PostgresMetadataStore(pg_dsn) if request.param == 'postgres' else + SqliteMetadataStore(tmp_path/'metadata.sqlite')) + try: + yield store + finally: + await store.close() + + +async def test_shared_resource_generation_path_and_paging(metadata): + async with metadata.transaction(write=True) as tx: + await tx.insert(item()) + await tx.insert(item('a','root')) + await tx.insert(item('b','root')) + assert await tx.resolve('/a') == 'a' + assert await tx.path('a') == '/a' + assert [r.id for r in await tx.ancestors('a')] == ['root'] + first = await tx.children('root', limit=1) + assert [r.id for r in first.items] == ['a'] and first.next_cursor == 'a' + assert [r.id for r in (await tx.children('root', cursor=first.next_cursor)).items] == ['b'] + old = await tx.resource('a') + await tx.replace(replace(old, generation=1, mode=0o700), 0) + assert tx.setting('authorization_epoch') == 1 + with pytest.raises(Failure, match='generation_conflict'): + await tx.replace(replace(old, generation=1), 0) + async with metadata.transaction(write=False) as tx: + assert (await tx.resource('a')).generation == 1 + assert (await tx.resource('a')).mode == 0o700 + + +async def test_shared_result_event_audit_and_job_roundtrips(metadata): + now = datetime.now(UTC) + principal = Principal(actor='owner', subject='owner', credential_id=None, + method='local', certificates=(), ceiling=()) + result = OperationResult(request_id='once', operation='test', status='ok', actor=None, subject=None) + event = Event(id='event-one', type='test', time=now, request_id='once', + actor='owner', subject='owner', resources=(), data={}) + audit = AuditEvent(event=event, authority=(), before_digest=None, after_digest=None, + previous_digest=None, entry_digest='', result='ok') + job = EffectJob(id='job-one', event_id=event.id, kind='mail', dedupe_key='unique-job', + principal=principal, operation='mail.send', arguments={}, state='pending', + attempts=0, next_attempt_at=now, lease_until=None) + async with metadata.transaction(write=True) as tx: + await tx.save_result('owner','digest-one',result) + await tx.append_event(event) + await tx.append_audit(audit) + await tx.append_audit(replace(audit, event=replace(event,id='event-two'))) + await tx.enqueue(job) + async with metadata.transaction(write=False) as tx: + assert await tx.request_result('owner','once','digest-one') == result + with pytest.raises(Failure, match='idempotency_conflict'): + await tx.request_result('owner','once','another-digest') + rows = tx.rows('SELECT digest,previous FROM audit ORDER BY seq') + assert rows[0][1] is None and rows[1][1] == rows[0][0] + assert await tx.job(job.id) == job + assert tx.one('SELECT id FROM events WHERE id=?',(event.id,))[0] == event.id + + +async def test_shared_savepoints_and_compensation_keep_their_scope(metadata): + effects = [] + async def compensate(value): + effects.append(value) + async with metadata.transaction(write=True) as outer: + outer.set_setting('outer', True) + outer.on_rollback(lambda: compensate('outer')) + with pytest.raises(ValueError, match='inner'): + async with metadata.transaction(write=True) as inner: + assert inner is outer + inner.set_setting('inner', True) + inner.on_rollback(lambda: compensate('inner')) + raise ValueError('inner') + assert effects == ['inner'] and outer.setting('inner') is None + assert effects == ['inner'] + with pytest.raises(ValueError, match='whole'): + async with metadata.transaction(write=True) as tx: + tx.set_setting('whole', True) + tx.on_rollback(lambda: compensate('whole')) + raise ValueError('whole') + async with metadata.transaction(write=False) as tx: + assert tx.setting('outer') is True and tx.setting('whole') is None + assert effects == ['inner','whole'] + + +async def test_shared_read_only_closed_and_cross_task_guards(metadata): + async with metadata.transaction(write=False) as tx: + with pytest.raises(Failure, match='read_only_transaction'): + tx.set_setting('forbidden', True) + with pytest.raises(Failure, match='read_only_transaction'): + async with metadata.transaction(write=True): + pytest.fail('read-only transaction was upgraded') + async def other_task(): + tx.setting('anything') + with pytest.raises(Failure, match='transaction_cross_task'): + await asyncio.create_task(other_task()) + with pytest.raises(Failure, match='transaction_closed'): + tx.setting('anything') + + +async def test_each_driver_translates_unique_errors_and_rolls_back(metadata): + async with metadata.transaction(write=True) as tx: + await tx.insert(item()) + with pytest.raises(Failure, match='constraint_conflict'): + async with metadata.transaction(write=True) as tx: + await tx.insert(item('child','root')) + await tx.insert(item('child','root')) + async with metadata.transaction(write=False) as tx: + assert tx.one('SELECT COUNT(*) FROM resources')[0] == 1 + assert tx.one('SELECT id FROM resources WHERE id=?',('child',)) is None + + +async def test_each_driver_preserves_literal_percent_and_question_mark(metadata): + async with metadata.transaction(write=False) as tx: + row = tx.one("SELECT '?', ?, '50%'", ('literal?%',)) + assert tuple(row) == ('?', 'literal?%', '50%') From 2daf1e0b81783d8b52685540b041ba9563333282 Mon Sep 17 00:00:00 2001 From: LIghtJUNction Date: Tue, 29 Sep 2026 04:50:21 +0800 Subject: [PATCH 04/11] test(storage): expose session backend and query contract drift Refs #157 #83 #85. Exercise identical lifecycle and query boundaries on SQLite and real PostgreSQL before implementation. --- .github/workflows/storage-session.yml | 81 ++++++++++++++++++++ docs/ISSUE_PROGRESS.md | 14 ++++ tests/test_session_boundary.py | 106 ++++++++++++++++++++++++++ 3 files changed, 201 insertions(+) create mode 100644 .github/workflows/storage-session.yml create mode 100644 docs/ISSUE_PROGRESS.md create mode 100644 tests/test_session_boundary.py diff --git a/.github/workflows/storage-session.yml b/.github/workflows/storage-session.yml new file mode 100644 index 00000000..696c037a --- /dev/null +++ b/.github/workflows/storage-session.yml @@ -0,0 +1,81 @@ +name: Storage session boundaries +on: + push: + branches: [fix/storage-session-20260929-2050] + pull_request: + paths: + - 'src/msg/**' + - 'tests/test_session_boundary.py' + - '.github/workflows/storage-session.yml' + workflow_dispatch: +permissions: + contents: read +concurrency: + group: storage-session-${{ github.ref }} + cancel-in-progress: true +jobs: + focused: + runs-on: ubuntu-latest + timeout-minutes: 12 + services: + postgres: + image: postgres:16 + env: + POSTGRES_USER: msgtest + POSTGRES_PASSWORD: msgtest + POSTGRES_DB: postgres + ports: ['5432:5432'] + options: >- + --health-cmd "pg_isready -U msgtest" + --health-interval 5s --health-timeout 5s --health-retries 10 + valkey: + image: valkey/valkey:9.0.6 + ports: ['6379:6379'] + options: >- + --health-cmd "valkey-cli ping" + --health-interval 5s --health-timeout 5s --health-retries 10 + env: + MSG_TEST_POSTGRES_URL_TEMPLATE: postgresql://msgtest:msgtest@127.0.0.1:5432/{database} + MSG_TEST_VALKEY_URL: redis://127.0.0.1:6379/0 + steps: + - uses: actions/checkout@v7 + with: + persist-credentials: false + - uses: actions/setup-python@v7 + with: + python-version: '3.15' + allow-prereleases: true + cache: pip + - name: Install isolated test dependencies + run: | + sudo apt-get update -qq + sudo apt-get install -y -qq age + python -m pip install -e '.[dev]' + - name: Record source identity + env: + EVENT_HEAD: ${{ github.event.pull_request.head.sha || github.sha }} + run: | + mkdir -p artifacts + python - <<'PY' + import json, os, platform, subprocess + from pathlib import Path + def git(ref): + return subprocess.check_output(['git', 'rev-parse', ref], text=True).strip() + Path('artifacts/source.json').write_text(json.dumps({ + 'head': os.environ['EVENT_HEAD'], 'checkout': git('HEAD'), 'tree': git('HEAD^{tree}'), + 'run': os.environ['GITHUB_RUN_ID'], 'attempt': os.environ['GITHUB_RUN_ATTEMPT'], + 'python': platform.python_version(), 'scope': 'isolated-regression-not-production' + }, indent=2) + '\n') + PY + - name: Exercise boundaries and unchanged security contracts + run: >- + python -m pytest tests/test_session_boundary.py tests/test_foundation.py + tests/test_postgres.py tests/test_security_rollback_fences.py + tests/test_security_storage_rollback.py tests/test_ledger_accounts.py + --junitxml=artifacts/storage-session.xml + - uses: actions/upload-artifact@v4 + if: always() + with: + name: storage-session-evidence + path: artifacts/ + if-no-files-found: error diff --git a/docs/ISSUE_PROGRESS.md b/docs/ISSUE_PROGRESS.md new file mode 100644 index 00000000..4826c066 --- /dev/null +++ b/docs/ISSUE_PROGRESS.md @@ -0,0 +1,14 @@ +# Issue 推进笔记 + +本分支 fix/storage-session-20260929-2050 处理 #157，基于 main476e64f04ed1d9c7dcab4a41c4f3cdb72faad6a7。与 #156 执行器分支、#160 工具接口分支及 #155 恢复分支隔离，合并时将进度归入同一笔记。 + +- [x] 阅读 issue 全文及第二轮补充：既解除后端继承，也明确真实 SQL 会话契约。 +- [x] 提交双后端失败回归：独立构造、真实政策消费者、受限结果视图、只读/跨任务/关闭后拒绝、嵌套回滚。 +- [ ] 云端记录失败基线；实现后运行同一用例，不降低断言。 +- [ ] 中立共享 RelationalSession；各后端保有连接、方言、迁移、事务和错误映射。 +- [ ] 显式 QuerySession/QueryResult，不给 handler 裸连接或提交权；避免增加逐表 Repository/ORM。 +- [ ] 完整四分片/node-ID/conformance/build 与恢复回归；审查精确头后合并。 + +所有项目执行在 GitHub Actions；本地仅源文件编辑、差异与Git树检查。当前没有修复后测试通过结论，没有生产部署、真实数据迁移或恢复放行。 + +#64/#65/#84的生产材料缺口及#68–70现场验收仍开放。其他任务状态以对应最新issue/PR和组合head的证据为准，不以本笔记替代权威设计与有效完成归档。 diff --git a/tests/test_session_boundary.py b/tests/test_session_boundary.py new file mode 100644 index 00000000..0f434981 --- /dev/null +++ b/tests/test_session_boundary.py @@ -0,0 +1,106 @@ +"""The same bounded query and domain contract on real SQLite and PostgreSQL.""" +import asyncio +import importlib +import importlib.util +from pathlib import Path +import subprocess +import sys +from typing import get_type_hints + +import pytest + +from msg.core.errors import Failure +from msg.core.contracts import MetadataSession +from msg.market.policy import DEFAULT_POLICY, load_policy +from msg.storage.postgres import PostgresMetadataStore, PostgresSession +from msg.storage.sqlite import SqliteMetadataStore, SqliteSession + + +@pytest.fixture(params=['sqlite', 'postgres']) +async def store(request, tmp_path): + value = (PostgresMetadataStore(request.getfixturevalue('pg_dsn')) + if request.param == 'postgres' else SqliteMetadataStore(tmp_path / 'metadata.db')) + yield value + await value.close() + + +def test_sessions_are_siblings_and_common_base_does_not_own_a_driver(): + assert not issubclass(PostgresSession, SqliteSession) + assert importlib.util.find_spec('msg.storage.session') is not None + common = importlib.import_module('msg.storage.session').RelationalSession + assert issubclass(PostgresSession, common) and issubclass(SqliteSession, common) + code = """ +import importlib.abc +import sys +class NoSQLite(importlib.abc.MetaPathFinder): + def find_spec(self, fullname, path=None, target=None): + if fullname in {'msg.storage.sqlite', 'sqlite3'}: + raise AssertionError('production storage imported ' + fullname) +sys.meta_path.insert(0, NoSQLite()) +from msg.storage.postgres import PostgresSession +""" + child = subprocess.run([sys.executable, '-c', code], capture_output=True, text=True) + assert child.returncode == 0, child.stderr + source = Path(importlib.import_module('msg.storage.session').__file__).read_text() + assert 'import sqlite3' not in source and 'import psycopg' not in source + assert '_connection' not in source + + +def test_declared_session_covers_real_query_consumer_without_commit_rights(): + assert importlib.util.find_spec('msg.core.query') is not None + query = importlib.import_module('msg.core.query') + assert get_type_hints(load_policy)['tx'] is query.QuerySession + for name in ('execute', 'one', 'rows', 'setting', 'set_setting'): + assert hasattr(MetadataSession, name), name + assert hasattr(query.QuerySession, name), name + for name in ('connection', 'commit', 'rollback'): + assert not hasattr(MetadataSession, name) + assert not hasattr(query.QuerySession, name) + + +async def test_query_result_is_a_guarded_view_not_a_driver_cursor(store): + async with store.transaction(write=False) as tx: + result = tx.execute('SELECT 1') + assert not hasattr(result, 'connection') + assert not hasattr(result, 'commit') + assert result.fetchone() == (1,) + assert result.fetchone() is None + assert list(tx.execute('SELECT 2 UNION ALL SELECT 3')) == [(2,), (3,)] + assert tx.rows('SELECT 4') == [(4,)] + assert tx.one('SELECT 5') == (5,) + with pytest.raises(Failure, match='transaction_closed'): + result.fetchall() + + +async def test_query_result_and_session_refuse_cross_task_and_read_only_writes(store): + async with store.transaction(write=False) as tx: + result = tx.execute('SELECT 1') + async def misuse_result(): + with pytest.raises(Failure, match='transaction_cross_task'): + result.fetchone() + async def misuse_session(): + with pytest.raises(Failure, match='transaction_cross_task'): + tx.one('SELECT 1') + await asyncio.gather(misuse_result(), misuse_session()) + with pytest.raises(Failure, match='read_only_transaction'): + tx.set_setting('forbidden', True) + assert result.fetchone() == (1,) + + +async def test_declared_policy_consumer_and_nested_rollback_work_on_both_backends(store): + assert importlib.util.find_spec('msg.core.query') is not None + query = importlib.import_module('msg.core.query') + async with store.transaction(write=True) as tx: + assert isinstance(tx, query.QuerySession) + assert load_policy(tx, DEFAULT_POLICY['id']) == DEFAULT_POLICY + tx.set_setting('outer', 'keep') + with pytest.raises(ValueError, match='abort inner'): + async with store.transaction(write=True) as child: + child.set_setting('inner', 'drop') + raise ValueError('abort inner') + assert tx.setting('inner') is None + tx.set_setting('after-inner', True) + async with store.transaction(write=False) as tx: + assert tx.setting('outer') == 'keep' + assert tx.setting('after-inner') is True + assert tx.setting('inner') is None From 1dc01c0b120642fd0438266085109cdaa6132d46 Mon Sep 17 00:00:00 2001 From: LIghtJUNction Date: Tue, 29 Sep 2026 04:53:35 +0800 Subject: [PATCH 05/11] ci: assemble and audit the exact metadata session extraction --- .../workflows/assemble-metadata-session.yml | 251 ++++++++++++++++++ 1 file changed, 251 insertions(+) create mode 100644 .github/workflows/assemble-metadata-session.yml diff --git a/.github/workflows/assemble-metadata-session.yml b/.github/workflows/assemble-metadata-session.yml new file mode 100644 index 00000000..056816c7 --- /dev/null +++ b/.github/workflows/assemble-metadata-session.yml @@ -0,0 +1,251 @@ +name: Assemble reviewed metadata session extraction +on: + push: + branches: [build/metadata-session-boundary-166] + paths: [.github/workflows/assemble-metadata-session.yml] +permissions: + contents: write +jobs: + assemble: + if: github.repository == 'TokenNotIncluded/msg.lmm.best' && github.ref == 'refs/heads/build/metadata-session-boundary-166' + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - uses: actions/checkout@v7 + with: + fetch-depth: 0 + - name: Require the unchanged regression-first source + env: + INPUT_SHA: ${{ github.sha }} + run: | + test "$(git rev-parse HEAD)" = "$INPUT_SHA" + git merge-base --is-ancestor 7f8752e37fe5b5a8021f192b87e731caec2bbc2d HEAD + git diff --exit-code 7f8752e37fe5b5a8021f192b87e731caec2bbc2d -- src tests docs + - name: Extract reviewed session methods without executing project code + run: | + python - <<'PY' + import ast, difflib, hashlib, json, subprocess + from pathlib import Path + + paths = { + 'sqlite': 'src/msg/storage/sqlite.py', + 'postgres': 'src/msg/storage/postgres.py', + 'common': 'src/msg/storage/session.py', + 'notes': 'docs/METADATA_SESSION_PROGRESS.md', + } + expected = { + 'sqlite':'880def3daaf59d3ce66d2e5e980a710c200eb988', + 'postgres':'5ac02cff5b7b048a84e4812c60502ffba889c9c1', + } + before = {name:Path(path).read_text() if Path(path).exists() else '' + for name,path in paths.items()} + for name, sha in expected.items(): + actual=subprocess.check_output(['git','hash-object',paths[name]],text=True).strip() + assert actual == sha, (name,actual,sha) + assert not before['common'], 'neutral module already exists' + + def replace_once(text, old, new): + assert text.count(old)==1, ('unexpected source',old[:100]) + return text.replace(old,new) + + old=before['sqlite'] + start=old.index('class SqliteSession:') + end=old.index('\n\nclass SqliteMetadataStore:') + session=old[start:end] + execute_start=session.index(' def execute(') + execute_end=session.index(' def one(') + driver_execute=session[execute_start:execute_end] + common=replace_once(session,'class SqliteSession:', + 'class MetadataSessionBase(ABC):') + common=replace_once(common, + ' def __init__(self, connection, *, write: bool):\n self._connection, self.write = connection, write', + ' def __init__(self, *, write: bool):\n self.write = write') + common=replace_once(common,driver_execute, + ' @abstractmethod\n' + ' def execute(self, sql: str, parameters: Sequence[object] = (), *,\n' + ' write: bool = False) -> SqlCursor:\n' + ' """Run a bound statement after checking this session access."""\n' + ' raise NotImplementedError\n\n') + models=old[old.index('from msg.core.models import ('):old.index('\n_SCHEMA =')] + common_header='''"""Shared metadata model operations over a backend-owned statement adapter. + + The base owns task/read-only guards, model invariants and compensation + ordering. Drivers own transactions, locks, bound-parameter translation, + exception mapping and durability. No driver handle belongs in this layer. + The internal qmark SQL convention is not a public arbitrary-query API. + """ + from __future__ import annotations + + import asyncio + from abc import ABC, abstractmethod + from collections.abc import Iterator, Sequence + from dataclasses import replace + from typing import Protocol + + from msg.core.codec import canonical, decode, digest, loads, wire + from msg.core.errors import Failure, require + ''' + cursor=''' + class SqlCursor(Protocol): + """Only the cursor behavior used by the shared metadata methods.""" + @property + def rowcount(self) -> int: ... + def fetchone(self) -> tuple[object, ...] | None: ... + def fetchall(self) -> list[tuple[object, ...]]: ... + def __iter__(self) -> Iterator[tuple[object, ...]]: ... + + + ''' + after={} + after['common']=common_header+models+'\n'+cursor+common+'\n' + driver='''class SqliteSession(MetadataSessionBase): + def __init__(self, connection: sqlite3.Connection, *, write: bool): + super().__init__(write=write) + self._connection = connection + + '''+driver_execute.rstrip() + sqlite=old[:start]+driver+old[end:] + sqlite=replace_once(sqlite,'from dataclasses import replace\n','') + sqlite=replace_once(sqlite,'from msg.core.codec import canonical, decode, digest, loads, wire\n','') + sqlite=replace_once(sqlite,models,'from msg.storage.session import MetadataSessionBase\n') + sqlite=replace_once(sqlite, + '"""One transactional metadata session; handlers never own commit rights."""', + '"""SQLite test adapter; shared metadata invariants live in storage.session."""') + after['sqlite']=sqlite + postgres=replace_once(before['postgres'], + 'from msg.storage.sqlite import SqliteSession', + 'from msg.storage.session import MetadataSessionBase') + postgres=replace_once(postgres,'class PostgresSession(SqliteSession):', + 'class PostgresSession(MetadataSessionBase):') + postgres=replace_once(postgres, + ' super().__init__(connection, write=write)', + ' super().__init__(write=write)\n self._connection = connection') + after['postgres']=postgres + after['notes']=before['notes']+''' + + ## Red evidence and implementation + + Focused run **36481798533** tested exact regression commit + `7f8752e37fe5b5a8021f192b87e731caec2bbc2d` / tree + `686dbddc1bdfc52d97c127bdb7338f002cabf735`: **4 failed, 12 passed, + 0 errors, 0 skipped**. All failures were new architecture assertions; + all six shared behavior tests passed independently on both adapters. + Artifact10997260030 was downloaded and its SHA-256 matched the provider: + `50ae1ff1b71684d60e0ff133cde41c6cb81658c0855aeb9c376fff110dc27e2e`. + + The extraction changes only `storage/session.py`, `storage/sqlite.py` + and `storage/postgres.py`. `MetadataSessionBase` now owns the existing + resource/identity/result/event/audit/job methods, task/read-only guards + and rollback compensation. Its sole abstract method is bound statement + execution; `SqlCursor` explicitly describes the small real cursor port. + Both concrete sessions own their own driver handle and statement adapter. + + `core.contracts.MetadataSession` remains the structural business-facing + contract and is not weakened. Existing storage-local helpers (`one`, + `rows`, `check`, path/ancestor lookup, settings and job updates) remain + explicit concrete session methods; they do not grant callers commit + rights. The internal qmark convention is translated only by each driver. + No new ORM, repository service, configuration matrix or migration exists. + + The assembly workflow checks exact input blob IDs and compares every + extracted method except the constructor/statement port with the original + method AST. It also checks both complete Store classes, FakeMetadataStore, + driver execute/enqueue methods, PostgreSQL SQL translation, and both + schema literals for exact structural equality. It uploads this audit, + source patch and source commit/tree. Assembly is not a substitute for + green cloud tests. One-shot tooling stays on the build branch and is + removed from the submitted tree before final validation. + + PostgreSQL connection-scoped advisory locking through compensation, + savepoints, cancellation handling, migration, SQL/error adaptation and + post-commit effect signals are unchanged. SQLite retains its own file + writer fence, pragmas, connection, schema and exception mapping. No schema, + historical ID, ledger entry, signature or receipt bytes are rewritten. + + Final focused/full/recovery results are still pending at this commit. + Do not close #157 or broader production-evidence issues from this stage. + ''' + + def parse(text): + return ast.parse(text) + def cls(text,name): + return next(n for n in parse(text).body if isinstance(n,ast.ClassDef) and n.name==name) + def methods(text,name): + return {n.name:n for n in cls(text,name).body + if isinstance(n,(ast.FunctionDef,ast.AsyncFunctionDef))} + def dump(node): + return ast.dump(node,include_attributes=False) + def schema(text): + return next(n.value for n in parse(text).body if isinstance(n,ast.Assign) + and any(isinstance(t,ast.Name) and t.id=='_SCHEMA' for t in n.targets)) + same=[] + original_methods=methods(before['sqlite'],'SqliteSession') + shared_methods=methods(after['common'],'MetadataSessionBase') + assert set(original_methods)==set(shared_methods) + for name,node in original_methods.items(): + if name not in {'__init__','execute'}: + assert dump(node)==dump(shared_methods[name]),name + same.append('shared.'+name) + for backend, name in [('sqlite','SqliteMetadataStore'),('sqlite','FakeMetadataStore'), + ('postgres','PostgresMetadataStore')]: + assert dump(cls(before[backend],name))==dump(cls(after[backend],name)),name + same.append(name) + for backend,name in [('sqlite','SqliteSession'),('postgres','PostgresSession')]: + left=methods(before[backend],name) + right=methods(after[backend],name) + for method in ['execute']+(['enqueue'] if backend=='postgres' else []): + assert dump(left[method])==dump(right[method]),(backend,method) + same.append(backend+'.'+method) + assert dump(schema(before[backend]))==dump(schema(after[backend])) + same.append(backend+'._SCHEMA') + old_sql=next(n for n in parse(before['postgres']).body if isinstance(n,ast.FunctionDef) and n.name=='_postgres_sql') + new_sql=next(n for n in parse(after['postgres']).body if isinstance(n,ast.FunctionDef) and n.name=='_postgres_sql') + assert dump(old_sql)==dump(new_sql) + same.append('postgres._postgres_sql') + for forbidden in ('sqlite3','psycopg','_connection','PRAGMA ','CREATE TABLE ', + 'INSERT OR IGNORE','.commit(','.rollback(','pg_advisory'): + assert forbidden not in after['common'],forbidden + for name,text in after.items(): + Path(paths[name]).write_text(text) + out=Path('/tmp/metadata-session-assembly') + out.mkdir(parents=True,exist_ok=True) + patch=''.join(''.join(difflib.unified_diff(before[name].splitlines(True),after[name].splitlines(True), + fromfile='a/'+paths[name] if before[name] else '/dev/null',tofile='b/'+paths[name])) for name in after) + (out/'changes.patch').write_text(patch) + (out/'audit.json').write_text(json.dumps({'baseline_blobs':expected, + 'structurally_unchanged':same,'changed_paths':[paths[n] for n in after], + 'patch_sha256':hashlib.sha256(patch.encode()).hexdigest()},indent=2)+'\n') + PY + - name: Commit only the audited source extraction + run: | + git add src/msg/storage/session.py src/msg/storage/sqlite.py src/msg/storage/postgres.py docs/METADATA_SESSION_PROGRESS.md + git diff --cached --check + python - <<'PY' + import json, subprocess + from pathlib import Path + expected=set(json.loads(Path('/tmp/metadata-session-assembly/audit.json').read_text())['changed_paths']) + actual=set(subprocess.check_output(['git','diff','--cached','--name-only'],text=True).splitlines()) + assert actual == expected, (actual,expected) + PY + git -c user.name='github-actions[bot]' -c user.email='41898282+github-actions[bot]@users.noreply.github.com' commit -m 'refactor: separate shared metadata invariants from concrete database sessions' + - name: Publish dedicated build branch without force or workflow changes + env: + INPUT_SHA: ${{ github.sha }} + run: | + test "$(git ls-remote origin refs/heads/build/metadata-session-boundary-166 | cut -f1)" = "$INPUT_SHA" + git push origin HEAD:refs/heads/build/metadata-session-boundary-166 + python - <<'PY' + import json, os, subprocess + from pathlib import Path + git=lambda *args: subprocess.check_output(['git',*args],text=True).strip() + Path('/tmp/metadata-session-assembly/source.json').write_text(json.dumps({ + 'input':os.environ['INPUT_SHA'],'commit':git('rev-parse','HEAD'), + 'tree':git('rev-parse','HEAD^{tree}'),'dirty':bool(git('status','--porcelain'))},indent=2)+'\n') + PY + git archive --format=tar.gz -o /tmp/metadata-session-assembly/source.tar.gz HEAD + - uses: actions/upload-artifact@v4 + if: always() + with: + name: metadata-session-assembly + path: /tmp/metadata-session-assembly/ + if-no-files-found: error From 20b6d7e3c6c2d22a8f9e1ae7f2f89e1a1812dc27 Mon Sep 17 00:00:00 2001 From: LIghtJUNction Date: Tue, 29 Sep 2026 04:54:21 +0800 Subject: [PATCH 06/11] ci: deduplicate branch verification and record red boundary evidence Refs #156 #83 #85. Preserve all full-suite gates while grouping same-repository push and PR runs together. --- .github/workflows/architecture-ports.yml | 2 +- .github/workflows/ci.yml | 2 +- docs/ISSUE_PROGRESS.md | 16 +++++++++++++--- 3 files changed, 15 insertions(+), 5 deletions(-) diff --git a/.github/workflows/architecture-ports.yml b/.github/workflows/architecture-ports.yml index 2d63634c..d9135870 100644 --- a/.github/workflows/architecture-ports.yml +++ b/.github/workflows/architecture-ports.yml @@ -11,7 +11,7 @@ on: permissions: contents: read concurrency: - group: architecture-ports-${{ github.ref }} + group: architecture-ports-${{ github.event.pull_request.head.repo.full_name || github.repository }}-${{ github.head_ref || github.ref_name }} cancel-in-progress: true jobs: focused: diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fec5fb8d..22b3ed01 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -6,7 +6,7 @@ on: permissions: contents: read concurrency: - group: rewrite-${{ github.ref }} + group: rewrite-${{ github.event.pull_request.head.repo.full_name || github.repository }}-${{ github.head_ref || github.ref_name }} cancel-in-progress: true jobs: shards: diff --git a/docs/ISSUE_PROGRESS.md b/docs/ISSUE_PROGRESS.md index 357ffb2d..ee1214a0 100644 --- a/docs/ISSUE_PROGRESS.md +++ b/docs/ISSUE_PROGRESS.md @@ -14,9 +14,9 @@ - [x] 核对依赖：executor 反向导入 batch/discovery/communication，持有整棵 Application；packet 依赖 plugins.schemas，共同协议的所有权分散。 - [x] 先提交独立架构及真实签名/投影/回放回归。 -- [ ] 云端记录失败基线（不是本地运行）。 -- [ ] 单一批处理/事件标识函数所有权；明确窄投影与事务通知接口；HTTP/SSH 同一装配配置。 -- [ ] 共用请求/结果、基础schema和wire codec中立化；URL专用限制仍由传输层负责。 +- [x] 云端记录失败基线（不是本地运行）。 +- [x] 单一批处理/事件标识函数所有权；明确窄投影与事务通知接口；HTTP/SSH 同一装配配置。 +- [x] 共用请求/结果、基础schema和wire codec中立化；URL专用限制仍由传输层负责。 - [ ] 云端修复后聚焦回归、全四分片/精确node-ID/conformance/build/Recovery safety。 - [ ] 精确头审查并合并；随后更新对应 issue。没有通过证据前不标完成。 @@ -33,3 +33,13 @@ 失败基线已提交453d6f858e19136261278c7042c13598035aef1a，tree ae0b4f5e31dc1844c1abd0142da898b75b7779d4。云端专项36481310584和完整36481310527已触发；回读时仍排队，未当作通过。 实现采用固定的ResultProjection和TransactionalEventNotifications接口，不引入任意插件钩子。core.batching/core.events/core.packet/core.schemas分别只有一份真实逻辑；旧公开导入路径兼容重导出。URL秘密检查和gzip只在传输层，未移到所有入口。Application.new_executor同时装配HTTP/SSH的请求限制、秘密释放和恢复隔离，不再给executor整棵Application。未运行本地项目测试，尚无修复后通过结论。 + +## 已核验的失败基线与实现头 + +专项36481310584/job109127358396：165 tests，7 failed / 158 passed / 0 errors / 0 skipped。7个新增用例全部准确失败，原有158个全部通过。已下载artifact10997043729并核对ZIP SHA256 c2e1978649b6e251e38703befb0fc674fbb336db55fa075113af0edb07d81fb7；source.json绑定453d6f8/tree ae0b4f5、Python3.15.0rc2、attempt1。 + +已生成实现提交10f46fd300ef0fa4890ca0ea7aad3d10bca7f7a5/tree e9e33a3286f6d4efd19a421834bba813448f6a79，父提交为失败基线；云端发布步骤验证脚本哈希与完整预期Git树后只推暂存分支。辅助工作流和脚本不进入产品分支。 + +CI仅调整并发分组：同一来源仓库+分支的push/PR运行互相取代，避免重复占四个runner；不同fork同名分支不互相取消。没有删测试、放宽断言、缩小全量选择或移除conformance/build/恢复gate。修复后的完整结果仍待回读。 + +#155完整CI36480031758现已四分片、conformance、build、精确node-ID gate均success，但仍需其组合head验收；本会话不覆盖该分支。#157开始独立测试分支，#160仍由另一分支负责。 From 4a4bac67a253588bba4623c67f45f61830060a76 Mon Sep 17 00:00:00 2001 From: LIghtJUNction Date: Tue, 29 Sep 2026 04:56:53 +0800 Subject: [PATCH 07/11] ci: normalize extracted source EOF without changing the audited methods --- .../workflows/assemble-metadata-session.yml | 211 ++---------------- 1 file changed, 18 insertions(+), 193 deletions(-) diff --git a/.github/workflows/assemble-metadata-session.yml b/.github/workflows/assemble-metadata-session.yml index 056816c7..a108093f 100644 --- a/.github/workflows/assemble-metadata-session.yml +++ b/.github/workflows/assemble-metadata-session.yml @@ -21,200 +21,23 @@ jobs: test "$(git rev-parse HEAD)" = "$INPUT_SHA" git merge-base --is-ancestor 7f8752e37fe5b5a8021f192b87e731caec2bbc2d HEAD git diff --exit-code 7f8752e37fe5b5a8021f192b87e731caec2bbc2d -- src tests docs - - name: Extract reviewed session methods without executing project code + - name: Replay the exact audited extraction with its trailing-blank-line fix run: | python - <<'PY' - import ast, difflib, hashlib, json, subprocess - from pathlib import Path - - paths = { - 'sqlite': 'src/msg/storage/sqlite.py', - 'postgres': 'src/msg/storage/postgres.py', - 'common': 'src/msg/storage/session.py', - 'notes': 'docs/METADATA_SESSION_PROGRESS.md', - } - expected = { - 'sqlite':'880def3daaf59d3ce66d2e5e980a710c200eb988', - 'postgres':'5ac02cff5b7b048a84e4812c60502ffba889c9c1', - } - before = {name:Path(path).read_text() if Path(path).exists() else '' - for name,path in paths.items()} - for name, sha in expected.items(): - actual=subprocess.check_output(['git','hash-object',paths[name]],text=True).strip() - assert actual == sha, (name,actual,sha) - assert not before['common'], 'neutral module already exists' - - def replace_once(text, old, new): - assert text.count(old)==1, ('unexpected source',old[:100]) - return text.replace(old,new) - - old=before['sqlite'] - start=old.index('class SqliteSession:') - end=old.index('\n\nclass SqliteMetadataStore:') - session=old[start:end] - execute_start=session.index(' def execute(') - execute_end=session.index(' def one(') - driver_execute=session[execute_start:execute_end] - common=replace_once(session,'class SqliteSession:', - 'class MetadataSessionBase(ABC):') - common=replace_once(common, - ' def __init__(self, connection, *, write: bool):\n self._connection, self.write = connection, write', - ' def __init__(self, *, write: bool):\n self.write = write') - common=replace_once(common,driver_execute, - ' @abstractmethod\n' - ' def execute(self, sql: str, parameters: Sequence[object] = (), *,\n' - ' write: bool = False) -> SqlCursor:\n' - ' """Run a bound statement after checking this session access."""\n' - ' raise NotImplementedError\n\n') - models=old[old.index('from msg.core.models import ('):old.index('\n_SCHEMA =')] - common_header='''"""Shared metadata model operations over a backend-owned statement adapter. - - The base owns task/read-only guards, model invariants and compensation - ordering. Drivers own transactions, locks, bound-parameter translation, - exception mapping and durability. No driver handle belongs in this layer. - The internal qmark SQL convention is not a public arbitrary-query API. - """ - from __future__ import annotations - - import asyncio - from abc import ABC, abstractmethod - from collections.abc import Iterator, Sequence - from dataclasses import replace - from typing import Protocol - - from msg.core.codec import canonical, decode, digest, loads, wire - from msg.core.errors import Failure, require - ''' - cursor=''' - class SqlCursor(Protocol): - """Only the cursor behavior used by the shared metadata methods.""" - @property - def rowcount(self) -> int: ... - def fetchone(self) -> tuple[object, ...] | None: ... - def fetchall(self) -> list[tuple[object, ...]]: ... - def __iter__(self) -> Iterator[tuple[object, ...]]: ... - - - ''' - after={} - after['common']=common_header+models+'\n'+cursor+common+'\n' - driver='''class SqliteSession(MetadataSessionBase): - def __init__(self, connection: sqlite3.Connection, *, write: bool): - super().__init__(write=write) - self._connection = connection - - '''+driver_execute.rstrip() - sqlite=old[:start]+driver+old[end:] - sqlite=replace_once(sqlite,'from dataclasses import replace\n','') - sqlite=replace_once(sqlite,'from msg.core.codec import canonical, decode, digest, loads, wire\n','') - sqlite=replace_once(sqlite,models,'from msg.storage.session import MetadataSessionBase\n') - sqlite=replace_once(sqlite, - '"""One transactional metadata session; handlers never own commit rights."""', - '"""SQLite test adapter; shared metadata invariants live in storage.session."""') - after['sqlite']=sqlite - postgres=replace_once(before['postgres'], - 'from msg.storage.sqlite import SqliteSession', - 'from msg.storage.session import MetadataSessionBase') - postgres=replace_once(postgres,'class PostgresSession(SqliteSession):', - 'class PostgresSession(MetadataSessionBase):') - postgres=replace_once(postgres, - ' super().__init__(connection, write=write)', - ' super().__init__(write=write)\n self._connection = connection') - after['postgres']=postgres - after['notes']=before['notes']+''' - - ## Red evidence and implementation - - Focused run **36481798533** tested exact regression commit - `7f8752e37fe5b5a8021f192b87e731caec2bbc2d` / tree - `686dbddc1bdfc52d97c127bdb7338f002cabf735`: **4 failed, 12 passed, - 0 errors, 0 skipped**. All failures were new architecture assertions; - all six shared behavior tests passed independently on both adapters. - Artifact10997260030 was downloaded and its SHA-256 matched the provider: - `50ae1ff1b71684d60e0ff133cde41c6cb81658c0855aeb9c376fff110dc27e2e`. - - The extraction changes only `storage/session.py`, `storage/sqlite.py` - and `storage/postgres.py`. `MetadataSessionBase` now owns the existing - resource/identity/result/event/audit/job methods, task/read-only guards - and rollback compensation. Its sole abstract method is bound statement - execution; `SqlCursor` explicitly describes the small real cursor port. - Both concrete sessions own their own driver handle and statement adapter. - - `core.contracts.MetadataSession` remains the structural business-facing - contract and is not weakened. Existing storage-local helpers (`one`, - `rows`, `check`, path/ancestor lookup, settings and job updates) remain - explicit concrete session methods; they do not grant callers commit - rights. The internal qmark convention is translated only by each driver. - No new ORM, repository service, configuration matrix or migration exists. - - The assembly workflow checks exact input blob IDs and compares every - extracted method except the constructor/statement port with the original - method AST. It also checks both complete Store classes, FakeMetadataStore, - driver execute/enqueue methods, PostgreSQL SQL translation, and both - schema literals for exact structural equality. It uploads this audit, - source patch and source commit/tree. Assembly is not a substitute for - green cloud tests. One-shot tooling stays on the build branch and is - removed from the submitted tree before final validation. - - PostgreSQL connection-scoped advisory locking through compensation, - savepoints, cancellation handling, migration, SQL/error adaptation and - post-commit effect signals are unchanged. SQLite retains its own file - writer fence, pragmas, connection, schema and exception mapping. No schema, - historical ID, ledger entry, signature or receipt bytes are rewritten. - - Final focused/full/recovery results are still pending at this commit. - Do not close #157 or broader production-evidence issues from this stage. - ''' - - def parse(text): - return ast.parse(text) - def cls(text,name): - return next(n for n in parse(text).body if isinstance(n,ast.ClassDef) and n.name==name) - def methods(text,name): - return {n.name:n for n in cls(text,name).body - if isinstance(n,(ast.FunctionDef,ast.AsyncFunctionDef))} - def dump(node): - return ast.dump(node,include_attributes=False) - def schema(text): - return next(n.value for n in parse(text).body if isinstance(n,ast.Assign) - and any(isinstance(t,ast.Name) and t.id=='_SCHEMA' for t in n.targets)) - same=[] - original_methods=methods(before['sqlite'],'SqliteSession') - shared_methods=methods(after['common'],'MetadataSessionBase') - assert set(original_methods)==set(shared_methods) - for name,node in original_methods.items(): - if name not in {'__init__','execute'}: - assert dump(node)==dump(shared_methods[name]),name - same.append('shared.'+name) - for backend, name in [('sqlite','SqliteMetadataStore'),('sqlite','FakeMetadataStore'), - ('postgres','PostgresMetadataStore')]: - assert dump(cls(before[backend],name))==dump(cls(after[backend],name)),name - same.append(name) - for backend,name in [('sqlite','SqliteSession'),('postgres','PostgresSession')]: - left=methods(before[backend],name) - right=methods(after[backend],name) - for method in ['execute']+(['enqueue'] if backend=='postgres' else []): - assert dump(left[method])==dump(right[method]),(backend,method) - same.append(backend+'.'+method) - assert dump(schema(before[backend]))==dump(schema(after[backend])) - same.append(backend+'._SCHEMA') - old_sql=next(n for n in parse(before['postgres']).body if isinstance(n,ast.FunctionDef) and n.name=='_postgres_sql') - new_sql=next(n for n in parse(after['postgres']).body if isinstance(n,ast.FunctionDef) and n.name=='_postgres_sql') - assert dump(old_sql)==dump(new_sql) - same.append('postgres._postgres_sql') - for forbidden in ('sqlite3','psycopg','_connection','PRAGMA ','CREATE TABLE ', - 'INSERT OR IGNORE','.commit(','.rollback(','pg_advisory'): - assert forbidden not in after['common'],forbidden - for name,text in after.items(): - Path(paths[name]).write_text(text) - out=Path('/tmp/metadata-session-assembly') - out.mkdir(parents=True,exist_ok=True) - patch=''.join(''.join(difflib.unified_diff(before[name].splitlines(True),after[name].splitlines(True), - fromfile='a/'+paths[name] if before[name] else '/dev/null',tofile='b/'+paths[name])) for name in after) - (out/'changes.patch').write_text(patch) - (out/'audit.json').write_text(json.dumps({'baseline_blobs':expected, - 'structurally_unchanged':same,'changed_paths':[paths[n] for n in after], - 'patch_sha256':hashlib.sha256(patch.encode()).hexdigest()},indent=2)+'\n') + import subprocess, textwrap + recipe_commit='1dc01c0b120642fd0438266085109cdaa6132d46' + recipe_path='.github/workflows/assemble-metadata-session.yml' + workflow=subprocess.check_output(['git','show',recipe_commit+':'+recipe_path],text=True) + step=workflow.index(' - name: Extract reviewed session methods without executing project code\n') + opening=" python - <<'PY'\n" + begin=workflow.index(opening,step)+len(opening) + end=workflow.index(' PY\n',begin) + source=textwrap.dedent(workflow[begin:end]) + old="after['common']=common_header+models+'\\n'+cursor+common+'\\n'" + new="after['common']=common_header+models+'\\n'+cursor+common.rstrip()+'\\n'" + assert source.count(old)==1, 'unexpected pinned recipe' + source=source.replace(old,new) + exec(compile(source,recipe_commit+':metadata-session-extraction','exec')) PY - name: Commit only the audited source extraction run: | @@ -240,7 +63,9 @@ jobs: git=lambda *args: subprocess.check_output(['git',*args],text=True).strip() Path('/tmp/metadata-session-assembly/source.json').write_text(json.dumps({ 'input':os.environ['INPUT_SHA'],'commit':git('rev-parse','HEAD'), - 'tree':git('rev-parse','HEAD^{tree}'),'dirty':bool(git('status','--porcelain'))},indent=2)+'\n') + 'tree':git('rev-parse','HEAD^{tree}'),'dirty':bool(git('status','--porcelain')), + 'recipe_commit':'1dc01c0b120642fd0438266085109cdaa6132d46', + 'recipe_adjustment':'trim one extra blank line at common module EOF'},indent=2)+'\n') PY git archive --format=tar.gz -o /tmp/metadata-session-assembly/source.tar.gz HEAD - uses: actions/upload-artifact@v4 From 5474eaf475e7f844244413b1f3e5109d4add4dce Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 28 Sep 2026 20:59:58 +0000 Subject: [PATCH 08/11] refactor: separate shared metadata invariants from concrete database sessions --- docs/METADATA_SESSION_PROGRESS.md | 43 ++++ src/msg/storage/postgres.py | 7 +- src/msg/storage/session.py | 414 ++++++++++++++++++++++++++++++ src/msg/storage/sqlite.py | 390 +--------------------------- 4 files changed, 467 insertions(+), 387 deletions(-) create mode 100644 src/msg/storage/session.py diff --git a/docs/METADATA_SESSION_PROGRESS.md b/docs/METADATA_SESSION_PROGRESS.md index 3d94c200..c0d405c6 100644 --- a/docs/METADATA_SESSION_PROGRESS.md +++ b/docs/METADATA_SESSION_PROGRESS.md @@ -25,3 +25,46 @@ PostgreSQL keeps its direct/session-pooled connection requirement, connection-sc ## Required evidence before closure Obtain actual red evidence, then apply the extraction without schema/migration/domain changes. Run the shared contracts on both adapters, the existing PostgreSQL concurrency/migration/recovery suites, and the complete four-shard node-ID/conformance/build gate on the final head. Verify the tested tree before merging. Production restore/backup/operator evidence is not simulated or marked complete by these tests. + + +## Red evidence and implementation + +Focused run **36481798533** tested exact regression commit +`7f8752e37fe5b5a8021f192b87e731caec2bbc2d` / tree +`686dbddc1bdfc52d97c127bdb7338f002cabf735`: **4 failed, 12 passed, +0 errors, 0 skipped**. All failures were new architecture assertions; +all six shared behavior tests passed independently on both adapters. +Artifact10997260030 was downloaded and its SHA-256 matched the provider: +`50ae1ff1b71684d60e0ff133cde41c6cb81658c0855aeb9c376fff110dc27e2e`. + +The extraction changes only `storage/session.py`, `storage/sqlite.py` +and `storage/postgres.py`. `MetadataSessionBase` now owns the existing +resource/identity/result/event/audit/job methods, task/read-only guards +and rollback compensation. Its sole abstract method is bound statement +execution; `SqlCursor` explicitly describes the small real cursor port. +Both concrete sessions own their own driver handle and statement adapter. + +`core.contracts.MetadataSession` remains the structural business-facing +contract and is not weakened. Existing storage-local helpers (`one`, +`rows`, `check`, path/ancestor lookup, settings and job updates) remain +explicit concrete session methods; they do not grant callers commit +rights. The internal qmark convention is translated only by each driver. +No new ORM, repository service, configuration matrix or migration exists. + +The assembly workflow checks exact input blob IDs and compares every +extracted method except the constructor/statement port with the original +method AST. It also checks both complete Store classes, FakeMetadataStore, +driver execute/enqueue methods, PostgreSQL SQL translation, and both +schema literals for exact structural equality. It uploads this audit, +source patch and source commit/tree. Assembly is not a substitute for +green cloud tests. One-shot tooling stays on the build branch and is +removed from the submitted tree before final validation. + +PostgreSQL connection-scoped advisory locking through compensation, +savepoints, cancellation handling, migration, SQL/error adaptation and +post-commit effect signals are unchanged. SQLite retains its own file +writer fence, pragmas, connection, schema and exception mapping. No schema, +historical ID, ledger entry, signature or receipt bytes are rewritten. + +Final focused/full/recovery results are still pending at this commit. +Do not close #157 or broader production-evidence issues from this stage. diff --git a/src/msg/storage/postgres.py b/src/msg/storage/postgres.py index 5ac02cff..a0891512 100644 --- a/src/msg/storage/postgres.py +++ b/src/msg/storage/postgres.py @@ -23,7 +23,7 @@ from psycopg.conninfo import conninfo_to_dict, make_conninfo from msg.core.errors import Failure -from msg.storage.sqlite import SqliteSession +from msg.storage.session import MetadataSessionBase _SCHEMA = """ CREATE TABLE IF NOT EXISTS schema_version (version INTEGER PRIMARY KEY); @@ -509,9 +509,10 @@ def _postgres_sql(sql: str, *, has_parameters: bool = False) -> str: return ''.join(output) -class PostgresSession(SqliteSession): +class PostgresSession(MetadataSessionBase): def __init__(self, connection, *, write: bool): - super().__init__(connection, write=write) + super().__init__(write=write) + self._connection = connection self.pending_effect_ids: list[str] = [] def execute(self, sql, parameters=(), *, write=False): diff --git a/src/msg/storage/session.py b/src/msg/storage/session.py new file mode 100644 index 00000000..45b48b41 --- /dev/null +++ b/src/msg/storage/session.py @@ -0,0 +1,414 @@ +"""Shared metadata model operations over a backend-owned statement adapter. + +The base owns task/read-only guards, model invariants and compensation +ordering. Drivers own transactions, locks, bound-parameter translation, +exception mapping and durability. No driver handle belongs in this layer. +The internal qmark SQL convention is not a public arbitrary-query API. +""" +from __future__ import annotations + +import asyncio +from abc import ABC, abstractmethod +from collections.abc import Iterator, Sequence +from dataclasses import replace +from typing import Protocol + +from msg.core.codec import canonical, decode, digest, loads, wire +from msg.core.errors import Failure, require +from msg.core.models import ( + AuditEvent, Certificate, CertificateRequest, CertificateRequestState, Credential, + EffectJob, EmailSettings, Event, Membership, OperationResult, Organization, Page, + Resource, ResourceRef, Revision, Subject, TransferChunk, TransferSession, +) + + +class SqlCursor(Protocol): + """Only the cursor behavior used by the shared metadata methods.""" + @property + def rowcount(self) -> int: ... + def fetchone(self) -> tuple[object, ...] | None: ... + def fetchall(self) -> list[tuple[object, ...]]: ... + def __iter__(self) -> Iterator[tuple[object, ...]]: ... + + +class MetadataSessionBase(ABC): + def __init__(self, *, write: bool): + self.write = write + self.owner_task = asyncio.current_task() + self.closed = False + self.rollback_effects = [] + + def on_rollback(self, effect): + """Compensate external work if this transaction or savepoint aborts.""" + self.check(write=True) + self.rollback_effects.append(effect) + + async def run_rollback_effects(self, cause, start=0): + effects = self.rollback_effects[start:] + del self.rollback_effects[start:] + if not effects: + return + + async def compensate(): + for effect in reversed(effects): + try: + await effect() + except BaseException as exc: + # Keep the primary error and try every independent cleanup. + cause.add_note('rollback compensation failed: ' + type(exc).__name__) + + # A second cancellation must not abandon a pin operation (including its + # worker thread) and release the writer fence before it finishes. + pending = asyncio.create_task(compensate()) + while not pending.done(): + try: + await asyncio.shield(pending) + except asyncio.CancelledError: + pass + pending.result() + + def check(self, write=False): + require(not self.closed, "transaction_closed") + require(asyncio.current_task() is self.owner_task, "transaction_cross_task") + if write: + require(self.write, "read_only_transaction") + + @abstractmethod + def execute(self, sql: str, parameters: Sequence[object] = (), *, + write: bool = False) -> SqlCursor: + """Run a bound statement after checking this session access.""" + raise NotImplementedError + + def one(self, sql, parameters=()): + return self.execute(sql, parameters).fetchone() + + def rows(self, sql, parameters=()): + return self.execute(sql, parameters).fetchall() + + async def resource(self, id): + row = self.one("SELECT body FROM resources WHERE id=?", (id,)) + require(row is not None, "not_found") + return decode(Resource, loads(row[0])) + + async def resolve(self, path): + require(isinstance(path, str) and path.startswith("/"), "invalid_path") + parts = path.rstrip("/").split("/")[1:] + require(all(p not in {".", ".."} and "\\" not in p and "\x00" not in p for p in parts), "invalid_path") + if len(parts) == 2 and parts[0] == "_id": + return (await self.resource(parts[1])).id + row = self.one("SELECT id FROM resources WHERE parent IS NULL") + require(row is not None, "not_initialized") + rid = row[0] + if path == "/": + return rid + for part in parts: + require(bool(part), "invalid_path") + row = self.one("SELECT id FROM resources WHERE parent=? AND name=?", (rid, part)) + require(row is not None, "not_found") + rid = row[0] + return rid + + async def path(self, id): + segments, seen = [], set() + r = await self.resource(id) + while r.parent is not None: + require(r.id not in seen, "parent_cycle") + seen.add(r.id) + segments.append(r.name) + r = await self.resource(r.parent) + return "/" + "/".join(reversed(segments)) + + async def ancestors(self, id): + result, seen = [], set() + r = await self.resource(id) + while r.parent is not None: + require(r.id not in seen, "parent_cycle") + seen.add(r.id) + r = await self.resource(r.parent) + result.append(r) + return tuple(reversed(result)) + + async def children(self, parent, cursor=None, limit=50): + require(type(limit) is int and 1 <= limit <= 500, "invalid_limit") + rows = self.rows("SELECT body FROM resources WHERE parent=? AND id>? ORDER BY id LIMIT ?", + (parent, cursor or "", limit + 1)) + values = tuple(decode(Resource, loads(row[0])) for row in rows[:limit]) + return Page(items=values, next_cursor=values[-1].id if len(rows) > limit else None) + + async def revision(self, ref): + rid = ref.revision or (await self.resource(ref.id)).revision + row = self.one("SELECT body FROM revisions WHERE id=? AND resource_id=?", (rid, ref.id)) + require(row is not None, "revision_not_found") + return decode(Revision, loads(row[0])) + + async def history(self, id, cursor=None, limit=50): + require(1 <= limit <= 500, "invalid_limit") + rows = self.rows("SELECT body FROM revisions WHERE resource_id=? AND id>? ORDER BY id LIMIT ?", + (id, cursor or "", limit+1)) + values = tuple(decode(Revision, loads(r[0])) for r in rows[:limit]) + return Page(items=values, next_cursor=values[-1].id if len(rows)>limit else None) + + async def insert(self, resource): + self.check(True) + if resource.parent is not None: + await self.resource(resource.parent) + data = wire(resource) + self.execute("INSERT INTO resources VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?)", ( + resource.id, resource.type, resource.name, resource.parent, resource.owner, + resource.group, resource.mode, resource.generation, resource.revision, + resource.state, data['created_at'], data['modified_at'], canonical(resource).decode()), write=True) + for tag in resource.tags: + self.execute("INSERT INTO resource_tags VALUES (?,?)", (resource.id,tag), write=True) + + async def replace(self, resource, expected_generation): + self.check(True) + old = await self.resource(resource.id) + require(old.generation == expected_generation, "generation_conflict", + details={"generation":old.generation, "revision":old.revision}) + require(resource.generation == expected_generation + 1, "invalid_generation") + require(resource.type == old.type and resource.created_at == old.created_at and + resource.created_by == old.created_by, "immutable_creation_fact") + if resource.parent is not None: + require(resource.parent != resource.id, "parent_cycle") + parent = await self.resource(resource.parent) + require(all(a.id != resource.id for a in await self.ancestors(parent.id)), "parent_cycle") + if any(getattr(resource,k)!=getattr(old,k) for k in ('parent','owner','group','mode','state')): + self.set_setting('authorization_epoch',self.setting('authorization_epoch',0)+1) + data = wire(resource) + changed = self.execute("""UPDATE resources SET name=?,parent=?,owner=?,grp=?,mode=?, + generation=?,revision=?,state=?,modified_at=?,body=? WHERE id=? AND generation=?""", + (resource.name, resource.parent, resource.owner, resource.group, resource.mode, + resource.generation, resource.revision, resource.state, data['modified_at'], + canonical(resource).decode(), resource.id, expected_generation), write=True).rowcount + require(changed == 1, "generation_conflict") + if resource.tags != old.tags: + self.execute("DELETE FROM resource_tags WHERE resource_id=?", (resource.id,), write=True) + for tag in resource.tags: + self.execute("INSERT INTO resource_tags VALUES (?,?)", (resource.id,tag), write=True) + + async def append_revision(self, revision): + self.check(True) + for parent in revision.parents: + await self.revision(ResourceRef(id=revision.resource_id, revision=parent)) + self.execute("INSERT INTO revisions VALUES (?,?,?,?)", ( + revision.id, revision.resource_id, wire(revision.created_at), canonical(revision).decode()), write=True) + for relation in revision.relations: + self.execute("INSERT INTO relations VALUES (?,?,?,?,?,?)", ( + revision.id, revision.resource_id, relation.type, relation.target.id, + relation.target.revision, canonical(relation).decode()), write=True) + + async def subject(self, id): + row = self.one("SELECT body FROM identities WHERE id=? AND kind='subject'", (id,)) + require(row is not None, "subject_not_found") + return decode(Subject, loads(row[0])) + + async def organization(self, id): + row = self.one("SELECT body FROM identities WHERE id=? AND kind='organization'", (id,)) + require(row is not None, "group_not_found") + return decode(Organization, loads(row[0])) + + async def credential(self, id): + row = self.one("SELECT body FROM credentials WHERE id=?", (id,)) + require(row is not None, "credential_not_found") + return decode(Credential, loads(row[0])) + + async def certificate(self, id): + row = self.one("SELECT body FROM certificates WHERE id=?", (id,)) + require(row is not None, "certificate_not_found") + return decode(Certificate, loads(row[0])) + + async def certificate_revoked(self, id): + row = self.one("SELECT revoked FROM certificates WHERE id=?", (id,)) + require(row is not None, "certificate_not_found") + return bool(row[0]) + + async def memberships(self, subject): + if subject is None: + return () + rows=self.rows("""SELECT m.org,m.body FROM memberships m + JOIN resources r ON r.id=m.org + WHERE m.subject=? AND r.type='organization' AND r.state='active' + ORDER BY m.org""",(subject,)) + memberships=((org,decode(Membership,loads(body))) for org,body in rows) + # A stale membership row must not revive an archived/missing organization + # or borrow an organization/subject identifier from a mismatched body. + active=tuple(member for org,member in memberships if member.status=='active' + and member.organization_id==org and member.subject_id==subject + and org!='g_public') + row=self.one("SELECT body FROM identities WHERE id=? AND kind='subject'",(subject,)) + if row is None: + return active + identity=decode(Subject,loads(row[0])) + if identity.kind in {'registered','custodial','system'} and not identity.local_only: + # /&public is a virtual group for formal subjects. Historical rows + # may remain in storage, but never create a duplicate authorization. + return (Membership(organization_id='g_public',subject_id=subject, + role='member',version=0),*active) + return active + + async def request_result(self, subject, id, digest): + row = self.one("SELECT digest,body FROM results WHERE subject=? AND request_id=?", (subject,id)) + if row is None: + return None + require(row[0] == digest, "idempotency_conflict") + return decode(OperationResult, loads(row[1])) + + async def save_result(self, subject, digest, result): + from msg.storage.capacity import require_result_capacity + require_result_capacity(self) + self.execute("INSERT INTO results VALUES (?,?,?,?)", ( + subject, result.request_id, digest, canonical(result).decode()), write=True) + + async def update_identity(self, record, expected_generation): + self.check(True) + if isinstance(record, Membership): + self.set_setting('authorization_epoch',self.setting('authorization_epoch',0)+1) + table, where, args = 'memberships', 'org=? AND subject=?', (record.organization_id,record.subject_id) + generation = record.version + elif isinstance(record, EmailSettings): + table, where, args = 'emails', 'subject=?', (record.subject_id,) + generation = expected_generation + 1 + else: + table, where, args = 'identities', 'id=?', (record.resource_id,) + generation = record.auth_version if isinstance(record, Subject) else record.membership_version + row = self.one(f"SELECT generation FROM {table} WHERE {where}", args) + require((row is None and expected_generation == -1) or + (row is not None and row[0] == expected_generation), "generation_conflict") + payload = canonical(record).decode() + if row is None: + if table == 'identities': + self.execute("INSERT INTO identities VALUES (?,?,?,?)", (*args, + 'subject' if isinstance(record,Subject) else 'organization', generation,payload), write=True) + elif table == 'memberships': + self.execute("INSERT INTO memberships VALUES (?,?,?,?)", (*args,generation,payload), write=True) + else: + self.execute("INSERT INTO emails VALUES (?,?,?)", (*args,generation,payload), write=True) + else: + self.execute(f"UPDATE {table} SET generation=?,body=? WHERE {where}", + (generation,payload,*args), write=True) + + async def save_credential(self, credential, expected_auth_version): + subject = await self.subject(credential.subject_id) + require(subject.auth_version == expected_auth_version, "auth_version_conflict") + old=self.one("SELECT body FROM credentials WHERE id=?",(credential.id,)) + if old is not None: + previous=decode(Credential,loads(old[0])) + require(previous.subject_id==credential.subject_id and previous.kind==credential.kind and + previous.verifier==credential.verifier,"credential_identity_immutable") + self.execute("INSERT INTO credentials VALUES (?,?,?) ON CONFLICT(id) DO UPDATE SET body=excluded.body", + (credential.id,credential.subject_id,canonical(credential).decode()),write=True) + + async def csr(self, id): + row = self.one("SELECT body FROM csrs WHERE id=?",(id,)) + require(row is not None,"csr_not_found") + return decode(CertificateRequest,loads(row[0])) + + async def csr_state(self, id): + row=self.one("SELECT state_body FROM csrs WHERE id=?",(id,)) + require(row is not None,"csr_not_found") + return decode(CertificateRequestState,loads(row[0])) + + async def save_csr(self, request): + state=CertificateRequestState(request_id=request.resource_id,status='pending',generation=0) + self.execute("INSERT INTO csrs VALUES (?,?,?,?,?)",(request.resource_id,0,'pending', + canonical(request).decode(),canonical(state).decode()),write=True) + + async def transition_csr(self, state, expected_generation): + require(state.generation == expected_generation+1,"invalid_generation") + old=await self.csr_state(state.request_id) + require(old.status=='pending',"csr_not_pending") + require(old.generation==expected_generation,"generation_conflict") + self.execute("UPDATE csrs SET generation=?,state=?,state_body=? WHERE id=? AND generation=?", + (state.generation,state.status,canonical(state).decode(),state.request_id,expected_generation),write=True) + + async def register_certificate(self, certificate, csr_id, expected_generation): + self.execute("INSERT INTO certificates VALUES (?,?,?,?,?)",( + certificate.resource_id,certificate.subject_id,certificate.parent_certificate_id,0, + canonical(certificate).decode()),write=True) + if csr_id is not None: + self.set_setting('certificate_request:'+certificate.resource_id,csr_id) + await self.transition_csr(CertificateRequestState(request_id=csr_id,status='issued', + generation=expected_generation+1,certificate_id=certificate.resource_id),expected_generation) + + async def revoke_certificate(self, id, event): + await self.certificate(id) + self.execute("UPDATE certificates SET revoked=1 WHERE id=?",(id,),write=True) + self.set_setting('authorization_epoch',self.setting('authorization_epoch',0)+1) + await self.append_audit(event) + + async def transfer(self,id): + row=self.one("SELECT body FROM transfers WHERE id=?",(id,)) + require(row is not None,"transfer_not_found") + return decode(TransferSession,loads(row[0])) + + async def save_transfer(self,session,expected_generation): + if expected_generation is None: + self.execute("INSERT INTO transfers (id,subject,generation,body) VALUES (?,?,?,?)", + (session.id,session.subject_id,session.generation,canonical(session).decode()),write=True) + else: + require(session.generation==expected_generation+1,"invalid_generation") + count=self.execute("UPDATE transfers SET generation=?,body=? WHERE id=? AND generation=?", + (session.generation,canonical(session).decode(),session.id,expected_generation),write=True).rowcount + require(count==1,"generation_conflict") + + async def put_chunk(self,chunk): + self.check(True) + start,end=chunk.offset,chunk.offset+chunk.content.size + rows=self.rows("SELECT offset,length,body FROM chunks WHERE transfer_id=? AND offset?", + (chunk.transfer_id,end,start)) + if rows: + require(len(rows)==1 and rows[0][0]==start and rows[0][1]==chunk.content.size and + decode(TransferChunk,loads(rows[0][2])).content==chunk.content,"chunk_conflict") + return + self.execute("INSERT INTO chunks VALUES (?,?,?,?)",(chunk.transfer_id,start,chunk.content.size, + canonical(chunk).decode()),write=True) + + async def missing_ranges(self,id,cursor=None,limit=50): + session=await self.transfer(id) + require(session.expected_size is not None,"size_required") + require(1<=limit<=500,"invalid_limit") + start=int(cursor or 0) + require(0<=start<=session.expected_size,"invalid_cursor") + gaps=[] + for row in self.execute("SELECT offset,length FROM chunks WHERE transfer_id=? AND offset+length>? ORDER BY offset",(id,start)): + if row[0]>start: + gaps.append((start,row[0])) + if len(gaps)>limit: + return Page(items=tuple(gaps[:limit]),next_cursor=str(gaps[limit][0])) + start=max(start,row[0]+row[1]) + if startlimit else None) + + async def append_event(self,event): + self.execute("INSERT INTO events (id,body) VALUES (?,?)",(event.id,canonical(event).decode()),write=True) + + async def append_audit(self,event): + previous=self.one("SELECT digest FROM audit ORDER BY seq DESC LIMIT 1") + prev=previous[0] if previous else None + data=wire(replace(event,previous_digest=prev,entry_digest='')) + data.pop('entry_digest') + entry=replace(event,previous_digest=prev,entry_digest=digest(data)) + self.execute("INSERT INTO audit (digest,previous,body) VALUES (?,?,?)", + (entry.entry_digest,prev,canonical(entry).decode()),write=True) + + async def enqueue(self,job): + self.execute("INSERT INTO jobs VALUES (?,?,?,?,?,?)",(job.id,job.dedupe_key,job.kind, + job.state,wire(job.next_attempt_at),canonical(job).decode()),write=True) + + async def job(self,id): + row=self.one("SELECT body FROM jobs WHERE id=?",(id,)) + require(row is not None,'job_not_found') + return decode(EffectJob,loads(row[0])) + + async def save_job(self,job): + self.execute("UPDATE jobs SET state=?,next_at=?,body=? WHERE id=?",(job.state, + wire(job.next_attempt_at),canonical(job).decode(),job.id),write=True) + + def setting(self,key,default=None): + row=self.one("SELECT value FROM settings WHERE key=?",(key,)) + return loads(row[0]) if row else default + + def set_setting(self,key,value): + self.execute("INSERT INTO settings VALUES (?,?) ON CONFLICT(key) DO UPDATE SET value=excluded.value", + (key,canonical(value).decode()),write=True) diff --git a/src/msg/storage/sqlite.py b/src/msg/storage/sqlite.py index 880def3d..b67cac03 100644 --- a/src/msg/storage/sqlite.py +++ b/src/msg/storage/sqlite.py @@ -1,4 +1,4 @@ -"""One transactional metadata session; handlers never own commit rights.""" +"""SQLite test adapter; shared metadata invariants live in storage.session.""" from __future__ import annotations import asyncio @@ -9,16 +9,10 @@ import tempfile import uuid from contextlib import asynccontextmanager -from dataclasses import replace from pathlib import Path -from msg.core.codec import canonical, decode, digest, loads, wire from msg.core.errors import Failure, require -from msg.core.models import ( - AuditEvent, Certificate, CertificateRequest, CertificateRequestState, Credential, - EffectJob, EmailSettings, Event, Membership, OperationResult, Organization, Page, - Resource, ResourceRef, Revision, Subject, TransferChunk, TransferSession, -) +from msg.storage.session import MetadataSessionBase _SCHEMA = """ PRAGMA foreign_keys=ON; @@ -219,47 +213,10 @@ """ -class SqliteSession: - def __init__(self, connection, *, write: bool): - self._connection, self.write = connection, write - self.owner_task = asyncio.current_task() - self.closed = False - self.rollback_effects = [] - - def on_rollback(self, effect): - """Compensate external work if this transaction or savepoint aborts.""" - self.check(write=True) - self.rollback_effects.append(effect) - - async def run_rollback_effects(self, cause, start=0): - effects = self.rollback_effects[start:] - del self.rollback_effects[start:] - if not effects: - return - - async def compensate(): - for effect in reversed(effects): - try: - await effect() - except BaseException as exc: - # Keep the primary error and try every independent cleanup. - cause.add_note('rollback compensation failed: ' + type(exc).__name__) - - # A second cancellation must not abandon a pin operation (including its - # worker thread) and release the writer fence before it finishes. - pending = asyncio.create_task(compensate()) - while not pending.done(): - try: - await asyncio.shield(pending) - except asyncio.CancelledError: - pass - pending.result() - - def check(self, write=False): - require(not self.closed, "transaction_closed") - require(asyncio.current_task() is self.owner_task, "transaction_cross_task") - if write: - require(self.write, "read_only_transaction") +class SqliteSession(MetadataSessionBase): + def __init__(self, connection: sqlite3.Connection, *, write: bool): + super().__init__(write=write) + self._connection = connection def execute(self, sql, parameters=(), *, write=False): self.check(write) @@ -268,341 +225,6 @@ def execute(self, sql, parameters=(), *, write=False): except sqlite3.IntegrityError as exc: raise Failure("constraint_conflict") from exc - def one(self, sql, parameters=()): - return self.execute(sql, parameters).fetchone() - - def rows(self, sql, parameters=()): - return self.execute(sql, parameters).fetchall() - - async def resource(self, id): - row = self.one("SELECT body FROM resources WHERE id=?", (id,)) - require(row is not None, "not_found") - return decode(Resource, loads(row[0])) - - async def resolve(self, path): - require(isinstance(path, str) and path.startswith("/"), "invalid_path") - parts = path.rstrip("/").split("/")[1:] - require(all(p not in {".", ".."} and "\\" not in p and "\x00" not in p for p in parts), "invalid_path") - if len(parts) == 2 and parts[0] == "_id": - return (await self.resource(parts[1])).id - row = self.one("SELECT id FROM resources WHERE parent IS NULL") - require(row is not None, "not_initialized") - rid = row[0] - if path == "/": - return rid - for part in parts: - require(bool(part), "invalid_path") - row = self.one("SELECT id FROM resources WHERE parent=? AND name=?", (rid, part)) - require(row is not None, "not_found") - rid = row[0] - return rid - - async def path(self, id): - segments, seen = [], set() - r = await self.resource(id) - while r.parent is not None: - require(r.id not in seen, "parent_cycle") - seen.add(r.id) - segments.append(r.name) - r = await self.resource(r.parent) - return "/" + "/".join(reversed(segments)) - - async def ancestors(self, id): - result, seen = [], set() - r = await self.resource(id) - while r.parent is not None: - require(r.id not in seen, "parent_cycle") - seen.add(r.id) - r = await self.resource(r.parent) - result.append(r) - return tuple(reversed(result)) - - async def children(self, parent, cursor=None, limit=50): - require(type(limit) is int and 1 <= limit <= 500, "invalid_limit") - rows = self.rows("SELECT body FROM resources WHERE parent=? AND id>? ORDER BY id LIMIT ?", - (parent, cursor or "", limit + 1)) - values = tuple(decode(Resource, loads(row[0])) for row in rows[:limit]) - return Page(items=values, next_cursor=values[-1].id if len(rows) > limit else None) - - async def revision(self, ref): - rid = ref.revision or (await self.resource(ref.id)).revision - row = self.one("SELECT body FROM revisions WHERE id=? AND resource_id=?", (rid, ref.id)) - require(row is not None, "revision_not_found") - return decode(Revision, loads(row[0])) - - async def history(self, id, cursor=None, limit=50): - require(1 <= limit <= 500, "invalid_limit") - rows = self.rows("SELECT body FROM revisions WHERE resource_id=? AND id>? ORDER BY id LIMIT ?", - (id, cursor or "", limit+1)) - values = tuple(decode(Revision, loads(r[0])) for r in rows[:limit]) - return Page(items=values, next_cursor=values[-1].id if len(rows)>limit else None) - - async def insert(self, resource): - self.check(True) - if resource.parent is not None: - await self.resource(resource.parent) - data = wire(resource) - self.execute("INSERT INTO resources VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?)", ( - resource.id, resource.type, resource.name, resource.parent, resource.owner, - resource.group, resource.mode, resource.generation, resource.revision, - resource.state, data['created_at'], data['modified_at'], canonical(resource).decode()), write=True) - for tag in resource.tags: - self.execute("INSERT INTO resource_tags VALUES (?,?)", (resource.id,tag), write=True) - - async def replace(self, resource, expected_generation): - self.check(True) - old = await self.resource(resource.id) - require(old.generation == expected_generation, "generation_conflict", - details={"generation":old.generation, "revision":old.revision}) - require(resource.generation == expected_generation + 1, "invalid_generation") - require(resource.type == old.type and resource.created_at == old.created_at and - resource.created_by == old.created_by, "immutable_creation_fact") - if resource.parent is not None: - require(resource.parent != resource.id, "parent_cycle") - parent = await self.resource(resource.parent) - require(all(a.id != resource.id for a in await self.ancestors(parent.id)), "parent_cycle") - if any(getattr(resource,k)!=getattr(old,k) for k in ('parent','owner','group','mode','state')): - self.set_setting('authorization_epoch',self.setting('authorization_epoch',0)+1) - data = wire(resource) - changed = self.execute("""UPDATE resources SET name=?,parent=?,owner=?,grp=?,mode=?, - generation=?,revision=?,state=?,modified_at=?,body=? WHERE id=? AND generation=?""", - (resource.name, resource.parent, resource.owner, resource.group, resource.mode, - resource.generation, resource.revision, resource.state, data['modified_at'], - canonical(resource).decode(), resource.id, expected_generation), write=True).rowcount - require(changed == 1, "generation_conflict") - if resource.tags != old.tags: - self.execute("DELETE FROM resource_tags WHERE resource_id=?", (resource.id,), write=True) - for tag in resource.tags: - self.execute("INSERT INTO resource_tags VALUES (?,?)", (resource.id,tag), write=True) - - async def append_revision(self, revision): - self.check(True) - for parent in revision.parents: - await self.revision(ResourceRef(id=revision.resource_id, revision=parent)) - self.execute("INSERT INTO revisions VALUES (?,?,?,?)", ( - revision.id, revision.resource_id, wire(revision.created_at), canonical(revision).decode()), write=True) - for relation in revision.relations: - self.execute("INSERT INTO relations VALUES (?,?,?,?,?,?)", ( - revision.id, revision.resource_id, relation.type, relation.target.id, - relation.target.revision, canonical(relation).decode()), write=True) - - async def subject(self, id): - row = self.one("SELECT body FROM identities WHERE id=? AND kind='subject'", (id,)) - require(row is not None, "subject_not_found") - return decode(Subject, loads(row[0])) - - async def organization(self, id): - row = self.one("SELECT body FROM identities WHERE id=? AND kind='organization'", (id,)) - require(row is not None, "group_not_found") - return decode(Organization, loads(row[0])) - - async def credential(self, id): - row = self.one("SELECT body FROM credentials WHERE id=?", (id,)) - require(row is not None, "credential_not_found") - return decode(Credential, loads(row[0])) - - async def certificate(self, id): - row = self.one("SELECT body FROM certificates WHERE id=?", (id,)) - require(row is not None, "certificate_not_found") - return decode(Certificate, loads(row[0])) - - async def certificate_revoked(self, id): - row = self.one("SELECT revoked FROM certificates WHERE id=?", (id,)) - require(row is not None, "certificate_not_found") - return bool(row[0]) - - async def memberships(self, subject): - if subject is None: - return () - rows=self.rows("""SELECT m.org,m.body FROM memberships m - JOIN resources r ON r.id=m.org - WHERE m.subject=? AND r.type='organization' AND r.state='active' - ORDER BY m.org""",(subject,)) - memberships=((org,decode(Membership,loads(body))) for org,body in rows) - # A stale membership row must not revive an archived/missing organization - # or borrow an organization/subject identifier from a mismatched body. - active=tuple(member for org,member in memberships if member.status=='active' - and member.organization_id==org and member.subject_id==subject - and org!='g_public') - row=self.one("SELECT body FROM identities WHERE id=? AND kind='subject'",(subject,)) - if row is None: - return active - identity=decode(Subject,loads(row[0])) - if identity.kind in {'registered','custodial','system'} and not identity.local_only: - # /&public is a virtual group for formal subjects. Historical rows - # may remain in storage, but never create a duplicate authorization. - return (Membership(organization_id='g_public',subject_id=subject, - role='member',version=0),*active) - return active - - async def request_result(self, subject, id, digest): - row = self.one("SELECT digest,body FROM results WHERE subject=? AND request_id=?", (subject,id)) - if row is None: - return None - require(row[0] == digest, "idempotency_conflict") - return decode(OperationResult, loads(row[1])) - - async def save_result(self, subject, digest, result): - from msg.storage.capacity import require_result_capacity - require_result_capacity(self) - self.execute("INSERT INTO results VALUES (?,?,?,?)", ( - subject, result.request_id, digest, canonical(result).decode()), write=True) - - async def update_identity(self, record, expected_generation): - self.check(True) - if isinstance(record, Membership): - self.set_setting('authorization_epoch',self.setting('authorization_epoch',0)+1) - table, where, args = 'memberships', 'org=? AND subject=?', (record.organization_id,record.subject_id) - generation = record.version - elif isinstance(record, EmailSettings): - table, where, args = 'emails', 'subject=?', (record.subject_id,) - generation = expected_generation + 1 - else: - table, where, args = 'identities', 'id=?', (record.resource_id,) - generation = record.auth_version if isinstance(record, Subject) else record.membership_version - row = self.one(f"SELECT generation FROM {table} WHERE {where}", args) - require((row is None and expected_generation == -1) or - (row is not None and row[0] == expected_generation), "generation_conflict") - payload = canonical(record).decode() - if row is None: - if table == 'identities': - self.execute("INSERT INTO identities VALUES (?,?,?,?)", (*args, - 'subject' if isinstance(record,Subject) else 'organization', generation,payload), write=True) - elif table == 'memberships': - self.execute("INSERT INTO memberships VALUES (?,?,?,?)", (*args,generation,payload), write=True) - else: - self.execute("INSERT INTO emails VALUES (?,?,?)", (*args,generation,payload), write=True) - else: - self.execute(f"UPDATE {table} SET generation=?,body=? WHERE {where}", - (generation,payload,*args), write=True) - - async def save_credential(self, credential, expected_auth_version): - subject = await self.subject(credential.subject_id) - require(subject.auth_version == expected_auth_version, "auth_version_conflict") - old=self.one("SELECT body FROM credentials WHERE id=?",(credential.id,)) - if old is not None: - previous=decode(Credential,loads(old[0])) - require(previous.subject_id==credential.subject_id and previous.kind==credential.kind and - previous.verifier==credential.verifier,"credential_identity_immutable") - self.execute("INSERT INTO credentials VALUES (?,?,?) ON CONFLICT(id) DO UPDATE SET body=excluded.body", - (credential.id,credential.subject_id,canonical(credential).decode()),write=True) - - async def csr(self, id): - row = self.one("SELECT body FROM csrs WHERE id=?",(id,)) - require(row is not None,"csr_not_found") - return decode(CertificateRequest,loads(row[0])) - - async def csr_state(self, id): - row=self.one("SELECT state_body FROM csrs WHERE id=?",(id,)) - require(row is not None,"csr_not_found") - return decode(CertificateRequestState,loads(row[0])) - - async def save_csr(self, request): - state=CertificateRequestState(request_id=request.resource_id,status='pending',generation=0) - self.execute("INSERT INTO csrs VALUES (?,?,?,?,?)",(request.resource_id,0,'pending', - canonical(request).decode(),canonical(state).decode()),write=True) - - async def transition_csr(self, state, expected_generation): - require(state.generation == expected_generation+1,"invalid_generation") - old=await self.csr_state(state.request_id) - require(old.status=='pending',"csr_not_pending") - require(old.generation==expected_generation,"generation_conflict") - self.execute("UPDATE csrs SET generation=?,state=?,state_body=? WHERE id=? AND generation=?", - (state.generation,state.status,canonical(state).decode(),state.request_id,expected_generation),write=True) - - async def register_certificate(self, certificate, csr_id, expected_generation): - self.execute("INSERT INTO certificates VALUES (?,?,?,?,?)",( - certificate.resource_id,certificate.subject_id,certificate.parent_certificate_id,0, - canonical(certificate).decode()),write=True) - if csr_id is not None: - self.set_setting('certificate_request:'+certificate.resource_id,csr_id) - await self.transition_csr(CertificateRequestState(request_id=csr_id,status='issued', - generation=expected_generation+1,certificate_id=certificate.resource_id),expected_generation) - - async def revoke_certificate(self, id, event): - await self.certificate(id) - self.execute("UPDATE certificates SET revoked=1 WHERE id=?",(id,),write=True) - self.set_setting('authorization_epoch',self.setting('authorization_epoch',0)+1) - await self.append_audit(event) - - async def transfer(self,id): - row=self.one("SELECT body FROM transfers WHERE id=?",(id,)) - require(row is not None,"transfer_not_found") - return decode(TransferSession,loads(row[0])) - - async def save_transfer(self,session,expected_generation): - if expected_generation is None: - self.execute("INSERT INTO transfers (id,subject,generation,body) VALUES (?,?,?,?)", - (session.id,session.subject_id,session.generation,canonical(session).decode()),write=True) - else: - require(session.generation==expected_generation+1,"invalid_generation") - count=self.execute("UPDATE transfers SET generation=?,body=? WHERE id=? AND generation=?", - (session.generation,canonical(session).decode(),session.id,expected_generation),write=True).rowcount - require(count==1,"generation_conflict") - - async def put_chunk(self,chunk): - self.check(True) - start,end=chunk.offset,chunk.offset+chunk.content.size - rows=self.rows("SELECT offset,length,body FROM chunks WHERE transfer_id=? AND offset?", - (chunk.transfer_id,end,start)) - if rows: - require(len(rows)==1 and rows[0][0]==start and rows[0][1]==chunk.content.size and - decode(TransferChunk,loads(rows[0][2])).content==chunk.content,"chunk_conflict") - return - self.execute("INSERT INTO chunks VALUES (?,?,?,?)",(chunk.transfer_id,start,chunk.content.size, - canonical(chunk).decode()),write=True) - - async def missing_ranges(self,id,cursor=None,limit=50): - session=await self.transfer(id) - require(session.expected_size is not None,"size_required") - require(1<=limit<=500,"invalid_limit") - start=int(cursor or 0) - require(0<=start<=session.expected_size,"invalid_cursor") - gaps=[] - for row in self.execute("SELECT offset,length FROM chunks WHERE transfer_id=? AND offset+length>? ORDER BY offset",(id,start)): - if row[0]>start: - gaps.append((start,row[0])) - if len(gaps)>limit: - return Page(items=tuple(gaps[:limit]),next_cursor=str(gaps[limit][0])) - start=max(start,row[0]+row[1]) - if startlimit else None) - - async def append_event(self,event): - self.execute("INSERT INTO events (id,body) VALUES (?,?)",(event.id,canonical(event).decode()),write=True) - - async def append_audit(self,event): - previous=self.one("SELECT digest FROM audit ORDER BY seq DESC LIMIT 1") - prev=previous[0] if previous else None - data=wire(replace(event,previous_digest=prev,entry_digest='')) - data.pop('entry_digest') - entry=replace(event,previous_digest=prev,entry_digest=digest(data)) - self.execute("INSERT INTO audit (digest,previous,body) VALUES (?,?,?)", - (entry.entry_digest,prev,canonical(entry).decode()),write=True) - - async def enqueue(self,job): - self.execute("INSERT INTO jobs VALUES (?,?,?,?,?,?)",(job.id,job.dedupe_key,job.kind, - job.state,wire(job.next_attempt_at),canonical(job).decode()),write=True) - - async def job(self,id): - row=self.one("SELECT body FROM jobs WHERE id=?",(id,)) - require(row is not None,'job_not_found') - return decode(EffectJob,loads(row[0])) - - async def save_job(self,job): - self.execute("UPDATE jobs SET state=?,next_at=?,body=? WHERE id=?",(job.state, - wire(job.next_attempt_at),canonical(job).decode(),job.id),write=True) - - def setting(self,key,default=None): - row=self.one("SELECT value FROM settings WHERE key=?",(key,)) - return loads(row[0]) if row else default - - def set_setting(self,key,value): - self.execute("INSERT INTO settings VALUES (?,?) ON CONFLICT(key) DO UPDATE SET value=excluded.value", - (key,canonical(value).decode()),write=True) - - class SqliteMetadataStore: def __init__(self,path: Path, *, busy_timeout: float=10): self.path=Path(path) From 4a6b852961b5e9745c1fea8d37117a7a7c1bde58 Mon Sep 17 00:00:00 2001 From: LIghtJUNction Date: Mon, 28 Sep 2026 21:00:31 +0000 Subject: [PATCH 09/11] refactor(storage): share neutral sessions and expose guarded query ports Refs #157 #83 #85. Preserve driver transactions, rollback fences, SQL and migration behavior. --- docs/ISSUE_PROGRESS.md | 12 +- src/msg/core/contracts.py | 3 +- src/msg/core/query.py | 44 ++++ src/msg/market/policy.py | 3 +- src/msg/storage/postgres.py | 15 +- src/msg/storage/query.py | 39 ++++ src/msg/storage/session.py | 403 ++++++++++++++++++++++++++++++++++++ src/msg/storage/sqlite.py | 392 +---------------------------------- 8 files changed, 519 insertions(+), 392 deletions(-) create mode 100644 src/msg/core/query.py create mode 100644 src/msg/storage/query.py create mode 100644 src/msg/storage/session.py diff --git a/docs/ISSUE_PROGRESS.md b/docs/ISSUE_PROGRESS.md index 4826c066..956cdfa6 100644 --- a/docs/ISSUE_PROGRESS.md +++ b/docs/ISSUE_PROGRESS.md @@ -5,10 +5,18 @@ - [x] 阅读 issue 全文及第二轮补充：既解除后端继承，也明确真实 SQL 会话契约。 - [x] 提交双后端失败回归：独立构造、真实政策消费者、受限结果视图、只读/跨任务/关闭后拒绝、嵌套回滚。 - [ ] 云端记录失败基线；实现后运行同一用例，不降低断言。 -- [ ] 中立共享 RelationalSession；各后端保有连接、方言、迁移、事务和错误映射。 -- [ ] 显式 QuerySession/QueryResult，不给 handler 裸连接或提交权；避免增加逐表 Repository/ORM。 +- [x] 中立共享 RelationalSession；各后端保有连接、方言、迁移、事务和错误映射。 +- [x] 显式 QuerySession/QueryResult，不给 handler 裸连接或提交权；避免增加逐表 Repository/ORM。 - [ ] 完整四分片/node-ID/conformance/build 与恢复回归；审查精确头后合并。 所有项目执行在 GitHub Actions；本地仅源文件编辑、差异与Git树检查。当前没有修复后测试通过结论，没有生产部署、真实数据迁移或恢复放行。 #64/#65/#84的生产材料缺口及#68–70现场验收仍开放。其他任务状态以对应最新issue/PR和组合head的证据为准，不以本笔记替代权威设计与有效完成归档。 + +## 实现切片（尚待云端验收） + +SqliteSession/PostgresSession改为同一RelationalSession的兄弟类；共用域方法不再持有连接或导入数据库驱动。所有原SQL、schema、迁移、隔离、写栅栏、恢复补偿、提交后信号保留。QuerySession显式列出实际查询方法；MetadataSession继承它，真实market.policy.load_policy以该接口标注。 + +execute返回受会话约束的QueryResult而不是驱动游标，不公开connection/commit；rowcount/fetchone/fetchall/迭代均重验任务和生命周期。没有把进程内Python当作安全沙箱，没有增加ORM/逐表Repository/任意SQL网络接口。 + +实现待同一红基线用例、真实PG/SQLite、账本、回滚补偿和完整四分片验收。所有执行在云端，未进行本地项目测试。 diff --git a/src/msg/core/contracts.py b/src/msg/core/contracts.py index 912b1caa..106df748 100644 --- a/src/msg/core/contracts.py +++ b/src/msg/core/contracts.py @@ -1,8 +1,9 @@ from __future__ import annotations from .models import * +from .query import QuerySession -class MetadataSession(Protocol): +class MetadataSession(QuerySession, Protocol): def on_rollback(self, effect: Callable[[], Awaitable[None]]) -> None: ... async def resource(self, id: ResourceId) -> Resource: ... async def resolve(self, path: str) -> ResourceId: ... diff --git a/src/msg/core/query.py b/src/msg/core/query.py new file mode 100644 index 00000000..13991aa9 --- /dev/null +++ b/src/msg/core/query.py @@ -0,0 +1,44 @@ +"""Explicit SQL query port for trusted server-side domain consumers. + +Queries use the shared positional '?' subset; a backend may translate it. +This is not an ORM or an untrusted-SQL interface. Handlers/requirements, local +administrative use cases and workers may consume it within the transaction +provided by MetadataStore; adapters/clients must not construct sessions. +Writes require a write transaction and write=True. Results cannot escape its +owning task or lifetime. Integrity conflicts are constraint_conflict, and +PostgreSQL lock/deadlock timeouts are retryable server_busy; adapter-specific +programming errors still abort the transaction. No commit or connection port +is exposed. Driver and database protections remain the authority for SQL. +""" +from __future__ import annotations + +from collections.abc import Iterator, Mapping, Sequence +from decimal import Decimal +from typing import Protocol, runtime_checkable + +# PostgreSQL SUM(bigint) returns Decimal; ordinary stored times/JSON are text. +type SqlValue = None | bool | int | float | str | bytes | Decimal +type SqlRow = tuple[SqlValue, ...] +type SqlParameters = Sequence[SqlValue] +type SettingValue = ( + None | bool | int | float | str | list[SettingValue] + | tuple[SettingValue, ...] | Mapping[str, SettingValue] +) + + +class QueryResult(Protocol): + @property + def rowcount(self) -> int: ... + def fetchone(self) -> SqlRow | None: ... + def fetchall(self) -> list[SqlRow]: ... + def __iter__(self) -> Iterator[SqlRow]: ... + + +@runtime_checkable +class QuerySession(Protocol): + def execute(self, sql: str, parameters: SqlParameters = (), *, + write: bool = False) -> QueryResult: ... + def one(self, sql: str, parameters: SqlParameters = ()) -> SqlRow | None: ... + def rows(self, sql: str, parameters: SqlParameters = ()) -> list[SqlRow]: ... + def setting(self, key: str, default: SettingValue = None) -> SettingValue: ... + def set_setting(self, key: str, value: SettingValue) -> None: ... diff --git a/src/msg/market/policy.py b/src/msg/market/policy.py index 3b414cca..9717a4da 100644 --- a/src/msg/market/policy.py +++ b/src/msg/market/policy.py @@ -3,6 +3,7 @@ from msg.core.codec import canonical, digest, loads from msg.core.errors import require +from msg.core.query import QuerySession DEFAULT_POLICY = { 'id': 'dispute-v1', 'version': 1, @@ -44,7 +45,7 @@ def validate(policy): return policy -def load_policy(tx, policy_id): +def load_policy(tx: QuerySession, policy_id: str): row = tx.one('SELECT body,digest FROM arbitration_policies WHERE id=?', (policy_id,)) if row: policy = loads(row[0]) diff --git a/src/msg/storage/postgres.py b/src/msg/storage/postgres.py index 5ac02cff..624326e8 100644 --- a/src/msg/storage/postgres.py +++ b/src/msg/storage/postgres.py @@ -23,7 +23,9 @@ from psycopg.conninfo import conninfo_to_dict, make_conninfo from msg.core.errors import Failure -from msg.storage.sqlite import SqliteSession +from msg.core.query import QueryResult, SqlParameters +from msg.storage.query import SessionQueryResult +from msg.storage.session import RelationalSession _SCHEMA = """ CREATE TABLE IF NOT EXISTS schema_version (version INTEGER PRIMARY KEY); @@ -509,18 +511,21 @@ def _postgres_sql(sql: str, *, has_parameters: bool = False) -> str: return ''.join(output) -class PostgresSession(SqliteSession): +class PostgresSession(RelationalSession): def __init__(self, connection, *, write: bool): - super().__init__(connection, write=write) + super().__init__(write=write) + self._connection = connection self.pending_effect_ids: list[str] = [] - def execute(self, sql, parameters=(), *, write=False): + def execute(self, sql: str, parameters: SqlParameters = (), *, + write: bool = False) -> QueryResult: self.check(write) try: # With an empty parameter tuple psycopg still parses literal '%' as a # placeholder; SQL such as LIKE 'policy:%' must be sent unchanged. - return self._connection.execute( + cursor = self._connection.execute( _postgres_sql(sql, has_parameters=bool(parameters)), parameters or None) + return SessionQueryResult(cursor, self.check) except psycopg.errors.IntegrityError as exc: raise Failure("constraint_conflict") from exc except (psycopg.errors.DeadlockDetected, psycopg.errors.LockNotAvailable, diff --git a/src/msg/storage/query.py b/src/msg/storage/query.py new file mode 100644 index 00000000..092787e8 --- /dev/null +++ b/src/msg/storage/query.py @@ -0,0 +1,39 @@ +"""Session-bound query results without a public driver cursor/connection.""" +from __future__ import annotations + +from collections.abc import Callable, Iterator +from typing import Protocol +from msg.core.query import SqlRow + + +class _Cursor(Protocol): + @property + def rowcount(self) -> int: ... + def fetchone(self) -> SqlRow | None: ... + def fetchall(self) -> list[SqlRow]: ... + + +class SessionQueryResult: + """A narrow view, not a sandbox for trusted in-process Python consumers.""" + __slots__ = ('_cursor', '_check') + + def __init__(self, cursor: _Cursor, check: Callable[[], None]): + self._cursor = cursor + self._check = check + + @property + def rowcount(self) -> int: + self._check() + return self._cursor.rowcount + + def fetchone(self) -> SqlRow | None: + self._check() + return self._cursor.fetchone() + + def fetchall(self) -> list[SqlRow]: + self._check() + return self._cursor.fetchall() + + def __iter__(self) -> Iterator[SqlRow]: + while (row := self.fetchone()) is not None: + yield row diff --git a/src/msg/storage/session.py b/src/msg/storage/session.py new file mode 100644 index 00000000..554b754d --- /dev/null +++ b/src/msg/storage/session.py @@ -0,0 +1,403 @@ +"""Shared relational domain methods, with no driver or transaction ownership. + +This SQL-oriented monolith shares the supported placeholder/query subset, not +an imaginary database-independent object store. Adapters own SQL translation, +schema migration, connections, writer fences, savepoints and commit/rollback. +Only this session's owning task can access its transaction or results. +""" +from __future__ import annotations + +import asyncio +from abc import ABC, abstractmethod +from dataclasses import replace +from msg.core.codec import canonical, decode, digest, loads, wire +from msg.core.errors import require +from msg.core.query import QueryResult, SettingValue, SqlParameters, SqlRow +from msg.core.models import ( + AuditEvent, Certificate, CertificateRequest, CertificateRequestState, Credential, + EffectJob, EmailSettings, Event, Membership, OperationResult, Organization, Page, + Resource, ResourceRef, Revision, Subject, TransferChunk, TransferSession, +) + + +class RelationalSession(ABC): + def __init__(self, *, write: bool): + self.write = write + self.owner_task = asyncio.current_task() + self.closed = False + self.rollback_effects = [] + + def on_rollback(self, effect): + """Compensate external work if this transaction or savepoint aborts.""" + self.check(write=True) + self.rollback_effects.append(effect) + + async def run_rollback_effects(self, cause, start=0): + effects = self.rollback_effects[start:] + del self.rollback_effects[start:] + if not effects: + return + + async def compensate(): + for effect in reversed(effects): + try: + await effect() + except BaseException as exc: + # Keep the primary error and try every independent cleanup. + cause.add_note('rollback compensation failed: ' + type(exc).__name__) + + # A second cancellation must not abandon a pin operation (including its + # worker thread) and release the writer fence before it finishes. + pending = asyncio.create_task(compensate()) + while not pending.done(): + try: + await asyncio.shield(pending) + except asyncio.CancelledError: + pass + pending.result() + + def check(self, write=False): + require(not self.closed, "transaction_closed") + require(asyncio.current_task() is self.owner_task, "transaction_cross_task") + if write: + require(self.write, "read_only_transaction") + + @abstractmethod + def execute(self, sql: str, parameters: SqlParameters = (), *, + write: bool = False) -> QueryResult: + """Run adapter-normalized SQL in this store-owned transaction.""" + raise NotImplementedError + + def one(self, sql: str, parameters: SqlParameters = ()) -> SqlRow | None: + return self.execute(sql, parameters).fetchone() + + def rows(self, sql: str, parameters: SqlParameters = ()) -> list[SqlRow]: + return self.execute(sql, parameters).fetchall() + + async def resource(self, id): + row = self.one("SELECT body FROM resources WHERE id=?", (id,)) + require(row is not None, "not_found") + return decode(Resource, loads(row[0])) + + async def resolve(self, path): + require(isinstance(path, str) and path.startswith("/"), "invalid_path") + parts = path.rstrip("/").split("/")[1:] + require(all(p not in {".", ".."} and "\\" not in p and "\x00" not in p for p in parts), "invalid_path") + if len(parts) == 2 and parts[0] == "_id": + return (await self.resource(parts[1])).id + row = self.one("SELECT id FROM resources WHERE parent IS NULL") + require(row is not None, "not_initialized") + rid = row[0] + if path == "/": + return rid + for part in parts: + require(bool(part), "invalid_path") + row = self.one("SELECT id FROM resources WHERE parent=? AND name=?", (rid, part)) + require(row is not None, "not_found") + rid = row[0] + return rid + + async def path(self, id): + segments, seen = [], set() + r = await self.resource(id) + while r.parent is not None: + require(r.id not in seen, "parent_cycle") + seen.add(r.id) + segments.append(r.name) + r = await self.resource(r.parent) + return "/" + "/".join(reversed(segments)) + + async def ancestors(self, id): + result, seen = [], set() + r = await self.resource(id) + while r.parent is not None: + require(r.id not in seen, "parent_cycle") + seen.add(r.id) + r = await self.resource(r.parent) + result.append(r) + return tuple(reversed(result)) + + async def children(self, parent, cursor=None, limit=50): + require(type(limit) is int and 1 <= limit <= 500, "invalid_limit") + rows = self.rows("SELECT body FROM resources WHERE parent=? AND id>? ORDER BY id LIMIT ?", + (parent, cursor or "", limit + 1)) + values = tuple(decode(Resource, loads(row[0])) for row in rows[:limit]) + return Page(items=values, next_cursor=values[-1].id if len(rows) > limit else None) + + async def revision(self, ref): + rid = ref.revision or (await self.resource(ref.id)).revision + row = self.one("SELECT body FROM revisions WHERE id=? AND resource_id=?", (rid, ref.id)) + require(row is not None, "revision_not_found") + return decode(Revision, loads(row[0])) + + async def history(self, id, cursor=None, limit=50): + require(1 <= limit <= 500, "invalid_limit") + rows = self.rows("SELECT body FROM revisions WHERE resource_id=? AND id>? ORDER BY id LIMIT ?", + (id, cursor or "", limit+1)) + values = tuple(decode(Revision, loads(r[0])) for r in rows[:limit]) + return Page(items=values, next_cursor=values[-1].id if len(rows)>limit else None) + + async def insert(self, resource): + self.check(True) + if resource.parent is not None: + await self.resource(resource.parent) + data = wire(resource) + self.execute("INSERT INTO resources VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?)", ( + resource.id, resource.type, resource.name, resource.parent, resource.owner, + resource.group, resource.mode, resource.generation, resource.revision, + resource.state, data['created_at'], data['modified_at'], canonical(resource).decode()), write=True) + for tag in resource.tags: + self.execute("INSERT INTO resource_tags VALUES (?,?)", (resource.id,tag), write=True) + + async def replace(self, resource, expected_generation): + self.check(True) + old = await self.resource(resource.id) + require(old.generation == expected_generation, "generation_conflict", + details={"generation":old.generation, "revision":old.revision}) + require(resource.generation == expected_generation + 1, "invalid_generation") + require(resource.type == old.type and resource.created_at == old.created_at and + resource.created_by == old.created_by, "immutable_creation_fact") + if resource.parent is not None: + require(resource.parent != resource.id, "parent_cycle") + parent = await self.resource(resource.parent) + require(all(a.id != resource.id for a in await self.ancestors(parent.id)), "parent_cycle") + if any(getattr(resource,k)!=getattr(old,k) for k in ('parent','owner','group','mode','state')): + self.set_setting('authorization_epoch',self.setting('authorization_epoch',0)+1) + data = wire(resource) + changed = self.execute("""UPDATE resources SET name=?,parent=?,owner=?,grp=?,mode=?, + generation=?,revision=?,state=?,modified_at=?,body=? WHERE id=? AND generation=?""", + (resource.name, resource.parent, resource.owner, resource.group, resource.mode, + resource.generation, resource.revision, resource.state, data['modified_at'], + canonical(resource).decode(), resource.id, expected_generation), write=True).rowcount + require(changed == 1, "generation_conflict") + if resource.tags != old.tags: + self.execute("DELETE FROM resource_tags WHERE resource_id=?", (resource.id,), write=True) + for tag in resource.tags: + self.execute("INSERT INTO resource_tags VALUES (?,?)", (resource.id,tag), write=True) + + async def append_revision(self, revision): + self.check(True) + for parent in revision.parents: + await self.revision(ResourceRef(id=revision.resource_id, revision=parent)) + self.execute("INSERT INTO revisions VALUES (?,?,?,?)", ( + revision.id, revision.resource_id, wire(revision.created_at), canonical(revision).decode()), write=True) + for relation in revision.relations: + self.execute("INSERT INTO relations VALUES (?,?,?,?,?,?)", ( + revision.id, revision.resource_id, relation.type, relation.target.id, + relation.target.revision, canonical(relation).decode()), write=True) + + async def subject(self, id): + row = self.one("SELECT body FROM identities WHERE id=? AND kind='subject'", (id,)) + require(row is not None, "subject_not_found") + return decode(Subject, loads(row[0])) + + async def organization(self, id): + row = self.one("SELECT body FROM identities WHERE id=? AND kind='organization'", (id,)) + require(row is not None, "group_not_found") + return decode(Organization, loads(row[0])) + + async def credential(self, id): + row = self.one("SELECT body FROM credentials WHERE id=?", (id,)) + require(row is not None, "credential_not_found") + return decode(Credential, loads(row[0])) + + async def certificate(self, id): + row = self.one("SELECT body FROM certificates WHERE id=?", (id,)) + require(row is not None, "certificate_not_found") + return decode(Certificate, loads(row[0])) + + async def certificate_revoked(self, id): + row = self.one("SELECT revoked FROM certificates WHERE id=?", (id,)) + require(row is not None, "certificate_not_found") + return bool(row[0]) + + async def memberships(self, subject): + if subject is None: + return () + rows=self.rows("""SELECT m.org,m.body FROM memberships m + JOIN resources r ON r.id=m.org + WHERE m.subject=? AND r.type='organization' AND r.state='active' + ORDER BY m.org""",(subject,)) + memberships=((org,decode(Membership,loads(body))) for org,body in rows) + # A stale membership row must not revive an archived/missing organization + # or borrow an organization/subject identifier from a mismatched body. + active=tuple(member for org,member in memberships if member.status=='active' + and member.organization_id==org and member.subject_id==subject + and org!='g_public') + row=self.one("SELECT body FROM identities WHERE id=? AND kind='subject'",(subject,)) + if row is None: + return active + identity=decode(Subject,loads(row[0])) + if identity.kind in {'registered','custodial','system'} and not identity.local_only: + # /&public is a virtual group for formal subjects. Historical rows + # may remain in storage, but never create a duplicate authorization. + return (Membership(organization_id='g_public',subject_id=subject, + role='member',version=0),*active) + return active + + async def request_result(self, subject, id, digest): + row = self.one("SELECT digest,body FROM results WHERE subject=? AND request_id=?", (subject,id)) + if row is None: + return None + require(row[0] == digest, "idempotency_conflict") + return decode(OperationResult, loads(row[1])) + + async def save_result(self, subject, digest, result): + from msg.storage.capacity import require_result_capacity + require_result_capacity(self) + self.execute("INSERT INTO results VALUES (?,?,?,?)", ( + subject, result.request_id, digest, canonical(result).decode()), write=True) + + async def update_identity(self, record, expected_generation): + self.check(True) + if isinstance(record, Membership): + self.set_setting('authorization_epoch',self.setting('authorization_epoch',0)+1) + table, where, args = 'memberships', 'org=? AND subject=?', (record.organization_id,record.subject_id) + generation = record.version + elif isinstance(record, EmailSettings): + table, where, args = 'emails', 'subject=?', (record.subject_id,) + generation = expected_generation + 1 + else: + table, where, args = 'identities', 'id=?', (record.resource_id,) + generation = record.auth_version if isinstance(record, Subject) else record.membership_version + row = self.one(f"SELECT generation FROM {table} WHERE {where}", args) + require((row is None and expected_generation == -1) or + (row is not None and row[0] == expected_generation), "generation_conflict") + payload = canonical(record).decode() + if row is None: + if table == 'identities': + self.execute("INSERT INTO identities VALUES (?,?,?,?)", (*args, + 'subject' if isinstance(record,Subject) else 'organization', generation,payload), write=True) + elif table == 'memberships': + self.execute("INSERT INTO memberships VALUES (?,?,?,?)", (*args,generation,payload), write=True) + else: + self.execute("INSERT INTO emails VALUES (?,?,?)", (*args,generation,payload), write=True) + else: + self.execute(f"UPDATE {table} SET generation=?,body=? WHERE {where}", + (generation,payload,*args), write=True) + + async def save_credential(self, credential, expected_auth_version): + subject = await self.subject(credential.subject_id) + require(subject.auth_version == expected_auth_version, "auth_version_conflict") + old=self.one("SELECT body FROM credentials WHERE id=?",(credential.id,)) + if old is not None: + previous=decode(Credential,loads(old[0])) + require(previous.subject_id==credential.subject_id and previous.kind==credential.kind and + previous.verifier==credential.verifier,"credential_identity_immutable") + self.execute("INSERT INTO credentials VALUES (?,?,?) ON CONFLICT(id) DO UPDATE SET body=excluded.body", + (credential.id,credential.subject_id,canonical(credential).decode()),write=True) + + async def csr(self, id): + row = self.one("SELECT body FROM csrs WHERE id=?",(id,)) + require(row is not None,"csr_not_found") + return decode(CertificateRequest,loads(row[0])) + + async def csr_state(self, id): + row=self.one("SELECT state_body FROM csrs WHERE id=?",(id,)) + require(row is not None,"csr_not_found") + return decode(CertificateRequestState,loads(row[0])) + + async def save_csr(self, request): + state=CertificateRequestState(request_id=request.resource_id,status='pending',generation=0) + self.execute("INSERT INTO csrs VALUES (?,?,?,?,?)",(request.resource_id,0,'pending', + canonical(request).decode(),canonical(state).decode()),write=True) + + async def transition_csr(self, state, expected_generation): + require(state.generation == expected_generation+1,"invalid_generation") + old=await self.csr_state(state.request_id) + require(old.status=='pending',"csr_not_pending") + require(old.generation==expected_generation,"generation_conflict") + self.execute("UPDATE csrs SET generation=?,state=?,state_body=? WHERE id=? AND generation=?", + (state.generation,state.status,canonical(state).decode(),state.request_id,expected_generation),write=True) + + async def register_certificate(self, certificate, csr_id, expected_generation): + self.execute("INSERT INTO certificates VALUES (?,?,?,?,?)",( + certificate.resource_id,certificate.subject_id,certificate.parent_certificate_id,0, + canonical(certificate).decode()),write=True) + if csr_id is not None: + self.set_setting('certificate_request:'+certificate.resource_id,csr_id) + await self.transition_csr(CertificateRequestState(request_id=csr_id,status='issued', + generation=expected_generation+1,certificate_id=certificate.resource_id),expected_generation) + + async def revoke_certificate(self, id, event): + await self.certificate(id) + self.execute("UPDATE certificates SET revoked=1 WHERE id=?",(id,),write=True) + self.set_setting('authorization_epoch',self.setting('authorization_epoch',0)+1) + await self.append_audit(event) + + async def transfer(self,id): + row=self.one("SELECT body FROM transfers WHERE id=?",(id,)) + require(row is not None,"transfer_not_found") + return decode(TransferSession,loads(row[0])) + + async def save_transfer(self,session,expected_generation): + if expected_generation is None: + self.execute("INSERT INTO transfers (id,subject,generation,body) VALUES (?,?,?,?)", + (session.id,session.subject_id,session.generation,canonical(session).decode()),write=True) + else: + require(session.generation==expected_generation+1,"invalid_generation") + count=self.execute("UPDATE transfers SET generation=?,body=? WHERE id=? AND generation=?", + (session.generation,canonical(session).decode(),session.id,expected_generation),write=True).rowcount + require(count==1,"generation_conflict") + + async def put_chunk(self,chunk): + self.check(True) + start,end=chunk.offset,chunk.offset+chunk.content.size + rows=self.rows("SELECT offset,length,body FROM chunks WHERE transfer_id=? AND offset?", + (chunk.transfer_id,end,start)) + if rows: + require(len(rows)==1 and rows[0][0]==start and rows[0][1]==chunk.content.size and + decode(TransferChunk,loads(rows[0][2])).content==chunk.content,"chunk_conflict") + return + self.execute("INSERT INTO chunks VALUES (?,?,?,?)",(chunk.transfer_id,start,chunk.content.size, + canonical(chunk).decode()),write=True) + + async def missing_ranges(self,id,cursor=None,limit=50): + session=await self.transfer(id) + require(session.expected_size is not None,"size_required") + require(1<=limit<=500,"invalid_limit") + start=int(cursor or 0) + require(0<=start<=session.expected_size,"invalid_cursor") + gaps=[] + for row in self.execute("SELECT offset,length FROM chunks WHERE transfer_id=? AND offset+length>? ORDER BY offset",(id,start)): + if row[0]>start: + gaps.append((start,row[0])) + if len(gaps)>limit: + return Page(items=tuple(gaps[:limit]),next_cursor=str(gaps[limit][0])) + start=max(start,row[0]+row[1]) + if startlimit else None) + + async def append_event(self,event): + self.execute("INSERT INTO events (id,body) VALUES (?,?)",(event.id,canonical(event).decode()),write=True) + + async def append_audit(self,event): + previous=self.one("SELECT digest FROM audit ORDER BY seq DESC LIMIT 1") + prev=previous[0] if previous else None + data=wire(replace(event,previous_digest=prev,entry_digest='')) + data.pop('entry_digest') + entry=replace(event,previous_digest=prev,entry_digest=digest(data)) + self.execute("INSERT INTO audit (digest,previous,body) VALUES (?,?,?)", + (entry.entry_digest,prev,canonical(entry).decode()),write=True) + + async def enqueue(self,job): + self.execute("INSERT INTO jobs VALUES (?,?,?,?,?,?)",(job.id,job.dedupe_key,job.kind, + job.state,wire(job.next_attempt_at),canonical(job).decode()),write=True) + + async def job(self,id): + row=self.one("SELECT body FROM jobs WHERE id=?",(id,)) + require(row is not None,'job_not_found') + return decode(EffectJob,loads(row[0])) + + async def save_job(self,job): + self.execute("UPDATE jobs SET state=?,next_at=?,body=? WHERE id=?",(job.state, + wire(job.next_attempt_at),canonical(job).decode(),job.id),write=True) + + def setting(self, key: str, default: SettingValue = None) -> SettingValue: + row=self.one("SELECT value FROM settings WHERE key=?",(key,)) + return loads(row[0]) if row else default + + def set_setting(self, key: str, value: SettingValue) -> None: + self.execute("INSERT INTO settings VALUES (?,?) ON CONFLICT(key) DO UPDATE SET value=excluded.value", + (key,canonical(value).decode()),write=True) diff --git a/src/msg/storage/sqlite.py b/src/msg/storage/sqlite.py index 880def3d..2ec62cdf 100644 --- a/src/msg/storage/sqlite.py +++ b/src/msg/storage/sqlite.py @@ -9,16 +9,12 @@ import tempfile import uuid from contextlib import asynccontextmanager -from dataclasses import replace from pathlib import Path -from msg.core.codec import canonical, decode, digest, loads, wire from msg.core.errors import Failure, require -from msg.core.models import ( - AuditEvent, Certificate, CertificateRequest, CertificateRequestState, Credential, - EffectJob, EmailSettings, Event, Membership, OperationResult, Organization, Page, - Resource, ResourceRef, Revision, Subject, TransferChunk, TransferSession, -) +from msg.core.query import QueryResult, SqlParameters +from msg.storage.query import SessionQueryResult +from msg.storage.session import RelationalSession _SCHEMA = """ PRAGMA foreign_keys=ON; @@ -219,389 +215,19 @@ """ -class SqliteSession: +class SqliteSession(RelationalSession): def __init__(self, connection, *, write: bool): - self._connection, self.write = connection, write - self.owner_task = asyncio.current_task() - self.closed = False - self.rollback_effects = [] + super().__init__(write=write) + self._connection = connection - def on_rollback(self, effect): - """Compensate external work if this transaction or savepoint aborts.""" - self.check(write=True) - self.rollback_effects.append(effect) - - async def run_rollback_effects(self, cause, start=0): - effects = self.rollback_effects[start:] - del self.rollback_effects[start:] - if not effects: - return - - async def compensate(): - for effect in reversed(effects): - try: - await effect() - except BaseException as exc: - # Keep the primary error and try every independent cleanup. - cause.add_note('rollback compensation failed: ' + type(exc).__name__) - - # A second cancellation must not abandon a pin operation (including its - # worker thread) and release the writer fence before it finishes. - pending = asyncio.create_task(compensate()) - while not pending.done(): - try: - await asyncio.shield(pending) - except asyncio.CancelledError: - pass - pending.result() - - def check(self, write=False): - require(not self.closed, "transaction_closed") - require(asyncio.current_task() is self.owner_task, "transaction_cross_task") - if write: - require(self.write, "read_only_transaction") - - def execute(self, sql, parameters=(), *, write=False): + def execute(self, sql: str, parameters: SqlParameters = (), *, + write: bool = False) -> QueryResult: self.check(write) try: - return self._connection.execute(sql, parameters) + return SessionQueryResult(self._connection.execute(sql, parameters), self.check) except sqlite3.IntegrityError as exc: raise Failure("constraint_conflict") from exc - def one(self, sql, parameters=()): - return self.execute(sql, parameters).fetchone() - - def rows(self, sql, parameters=()): - return self.execute(sql, parameters).fetchall() - - async def resource(self, id): - row = self.one("SELECT body FROM resources WHERE id=?", (id,)) - require(row is not None, "not_found") - return decode(Resource, loads(row[0])) - - async def resolve(self, path): - require(isinstance(path, str) and path.startswith("/"), "invalid_path") - parts = path.rstrip("/").split("/")[1:] - require(all(p not in {".", ".."} and "\\" not in p and "\x00" not in p for p in parts), "invalid_path") - if len(parts) == 2 and parts[0] == "_id": - return (await self.resource(parts[1])).id - row = self.one("SELECT id FROM resources WHERE parent IS NULL") - require(row is not None, "not_initialized") - rid = row[0] - if path == "/": - return rid - for part in parts: - require(bool(part), "invalid_path") - row = self.one("SELECT id FROM resources WHERE parent=? AND name=?", (rid, part)) - require(row is not None, "not_found") - rid = row[0] - return rid - - async def path(self, id): - segments, seen = [], set() - r = await self.resource(id) - while r.parent is not None: - require(r.id not in seen, "parent_cycle") - seen.add(r.id) - segments.append(r.name) - r = await self.resource(r.parent) - return "/" + "/".join(reversed(segments)) - - async def ancestors(self, id): - result, seen = [], set() - r = await self.resource(id) - while r.parent is not None: - require(r.id not in seen, "parent_cycle") - seen.add(r.id) - r = await self.resource(r.parent) - result.append(r) - return tuple(reversed(result)) - - async def children(self, parent, cursor=None, limit=50): - require(type(limit) is int and 1 <= limit <= 500, "invalid_limit") - rows = self.rows("SELECT body FROM resources WHERE parent=? AND id>? ORDER BY id LIMIT ?", - (parent, cursor or "", limit + 1)) - values = tuple(decode(Resource, loads(row[0])) for row in rows[:limit]) - return Page(items=values, next_cursor=values[-1].id if len(rows) > limit else None) - - async def revision(self, ref): - rid = ref.revision or (await self.resource(ref.id)).revision - row = self.one("SELECT body FROM revisions WHERE id=? AND resource_id=?", (rid, ref.id)) - require(row is not None, "revision_not_found") - return decode(Revision, loads(row[0])) - - async def history(self, id, cursor=None, limit=50): - require(1 <= limit <= 500, "invalid_limit") - rows = self.rows("SELECT body FROM revisions WHERE resource_id=? AND id>? ORDER BY id LIMIT ?", - (id, cursor or "", limit+1)) - values = tuple(decode(Revision, loads(r[0])) for r in rows[:limit]) - return Page(items=values, next_cursor=values[-1].id if len(rows)>limit else None) - - async def insert(self, resource): - self.check(True) - if resource.parent is not None: - await self.resource(resource.parent) - data = wire(resource) - self.execute("INSERT INTO resources VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?)", ( - resource.id, resource.type, resource.name, resource.parent, resource.owner, - resource.group, resource.mode, resource.generation, resource.revision, - resource.state, data['created_at'], data['modified_at'], canonical(resource).decode()), write=True) - for tag in resource.tags: - self.execute("INSERT INTO resource_tags VALUES (?,?)", (resource.id,tag), write=True) - - async def replace(self, resource, expected_generation): - self.check(True) - old = await self.resource(resource.id) - require(old.generation == expected_generation, "generation_conflict", - details={"generation":old.generation, "revision":old.revision}) - require(resource.generation == expected_generation + 1, "invalid_generation") - require(resource.type == old.type and resource.created_at == old.created_at and - resource.created_by == old.created_by, "immutable_creation_fact") - if resource.parent is not None: - require(resource.parent != resource.id, "parent_cycle") - parent = await self.resource(resource.parent) - require(all(a.id != resource.id for a in await self.ancestors(parent.id)), "parent_cycle") - if any(getattr(resource,k)!=getattr(old,k) for k in ('parent','owner','group','mode','state')): - self.set_setting('authorization_epoch',self.setting('authorization_epoch',0)+1) - data = wire(resource) - changed = self.execute("""UPDATE resources SET name=?,parent=?,owner=?,grp=?,mode=?, - generation=?,revision=?,state=?,modified_at=?,body=? WHERE id=? AND generation=?""", - (resource.name, resource.parent, resource.owner, resource.group, resource.mode, - resource.generation, resource.revision, resource.state, data['modified_at'], - canonical(resource).decode(), resource.id, expected_generation), write=True).rowcount - require(changed == 1, "generation_conflict") - if resource.tags != old.tags: - self.execute("DELETE FROM resource_tags WHERE resource_id=?", (resource.id,), write=True) - for tag in resource.tags: - self.execute("INSERT INTO resource_tags VALUES (?,?)", (resource.id,tag), write=True) - - async def append_revision(self, revision): - self.check(True) - for parent in revision.parents: - await self.revision(ResourceRef(id=revision.resource_id, revision=parent)) - self.execute("INSERT INTO revisions VALUES (?,?,?,?)", ( - revision.id, revision.resource_id, wire(revision.created_at), canonical(revision).decode()), write=True) - for relation in revision.relations: - self.execute("INSERT INTO relations VALUES (?,?,?,?,?,?)", ( - revision.id, revision.resource_id, relation.type, relation.target.id, - relation.target.revision, canonical(relation).decode()), write=True) - - async def subject(self, id): - row = self.one("SELECT body FROM identities WHERE id=? AND kind='subject'", (id,)) - require(row is not None, "subject_not_found") - return decode(Subject, loads(row[0])) - - async def organization(self, id): - row = self.one("SELECT body FROM identities WHERE id=? AND kind='organization'", (id,)) - require(row is not None, "group_not_found") - return decode(Organization, loads(row[0])) - - async def credential(self, id): - row = self.one("SELECT body FROM credentials WHERE id=?", (id,)) - require(row is not None, "credential_not_found") - return decode(Credential, loads(row[0])) - - async def certificate(self, id): - row = self.one("SELECT body FROM certificates WHERE id=?", (id,)) - require(row is not None, "certificate_not_found") - return decode(Certificate, loads(row[0])) - - async def certificate_revoked(self, id): - row = self.one("SELECT revoked FROM certificates WHERE id=?", (id,)) - require(row is not None, "certificate_not_found") - return bool(row[0]) - - async def memberships(self, subject): - if subject is None: - return () - rows=self.rows("""SELECT m.org,m.body FROM memberships m - JOIN resources r ON r.id=m.org - WHERE m.subject=? AND r.type='organization' AND r.state='active' - ORDER BY m.org""",(subject,)) - memberships=((org,decode(Membership,loads(body))) for org,body in rows) - # A stale membership row must not revive an archived/missing organization - # or borrow an organization/subject identifier from a mismatched body. - active=tuple(member for org,member in memberships if member.status=='active' - and member.organization_id==org and member.subject_id==subject - and org!='g_public') - row=self.one("SELECT body FROM identities WHERE id=? AND kind='subject'",(subject,)) - if row is None: - return active - identity=decode(Subject,loads(row[0])) - if identity.kind in {'registered','custodial','system'} and not identity.local_only: - # /&public is a virtual group for formal subjects. Historical rows - # may remain in storage, but never create a duplicate authorization. - return (Membership(organization_id='g_public',subject_id=subject, - role='member',version=0),*active) - return active - - async def request_result(self, subject, id, digest): - row = self.one("SELECT digest,body FROM results WHERE subject=? AND request_id=?", (subject,id)) - if row is None: - return None - require(row[0] == digest, "idempotency_conflict") - return decode(OperationResult, loads(row[1])) - - async def save_result(self, subject, digest, result): - from msg.storage.capacity import require_result_capacity - require_result_capacity(self) - self.execute("INSERT INTO results VALUES (?,?,?,?)", ( - subject, result.request_id, digest, canonical(result).decode()), write=True) - - async def update_identity(self, record, expected_generation): - self.check(True) - if isinstance(record, Membership): - self.set_setting('authorization_epoch',self.setting('authorization_epoch',0)+1) - table, where, args = 'memberships', 'org=? AND subject=?', (record.organization_id,record.subject_id) - generation = record.version - elif isinstance(record, EmailSettings): - table, where, args = 'emails', 'subject=?', (record.subject_id,) - generation = expected_generation + 1 - else: - table, where, args = 'identities', 'id=?', (record.resource_id,) - generation = record.auth_version if isinstance(record, Subject) else record.membership_version - row = self.one(f"SELECT generation FROM {table} WHERE {where}", args) - require((row is None and expected_generation == -1) or - (row is not None and row[0] == expected_generation), "generation_conflict") - payload = canonical(record).decode() - if row is None: - if table == 'identities': - self.execute("INSERT INTO identities VALUES (?,?,?,?)", (*args, - 'subject' if isinstance(record,Subject) else 'organization', generation,payload), write=True) - elif table == 'memberships': - self.execute("INSERT INTO memberships VALUES (?,?,?,?)", (*args,generation,payload), write=True) - else: - self.execute("INSERT INTO emails VALUES (?,?,?)", (*args,generation,payload), write=True) - else: - self.execute(f"UPDATE {table} SET generation=?,body=? WHERE {where}", - (generation,payload,*args), write=True) - - async def save_credential(self, credential, expected_auth_version): - subject = await self.subject(credential.subject_id) - require(subject.auth_version == expected_auth_version, "auth_version_conflict") - old=self.one("SELECT body FROM credentials WHERE id=?",(credential.id,)) - if old is not None: - previous=decode(Credential,loads(old[0])) - require(previous.subject_id==credential.subject_id and previous.kind==credential.kind and - previous.verifier==credential.verifier,"credential_identity_immutable") - self.execute("INSERT INTO credentials VALUES (?,?,?) ON CONFLICT(id) DO UPDATE SET body=excluded.body", - (credential.id,credential.subject_id,canonical(credential).decode()),write=True) - - async def csr(self, id): - row = self.one("SELECT body FROM csrs WHERE id=?",(id,)) - require(row is not None,"csr_not_found") - return decode(CertificateRequest,loads(row[0])) - - async def csr_state(self, id): - row=self.one("SELECT state_body FROM csrs WHERE id=?",(id,)) - require(row is not None,"csr_not_found") - return decode(CertificateRequestState,loads(row[0])) - - async def save_csr(self, request): - state=CertificateRequestState(request_id=request.resource_id,status='pending',generation=0) - self.execute("INSERT INTO csrs VALUES (?,?,?,?,?)",(request.resource_id,0,'pending', - canonical(request).decode(),canonical(state).decode()),write=True) - - async def transition_csr(self, state, expected_generation): - require(state.generation == expected_generation+1,"invalid_generation") - old=await self.csr_state(state.request_id) - require(old.status=='pending',"csr_not_pending") - require(old.generation==expected_generation,"generation_conflict") - self.execute("UPDATE csrs SET generation=?,state=?,state_body=? WHERE id=? AND generation=?", - (state.generation,state.status,canonical(state).decode(),state.request_id,expected_generation),write=True) - - async def register_certificate(self, certificate, csr_id, expected_generation): - self.execute("INSERT INTO certificates VALUES (?,?,?,?,?)",( - certificate.resource_id,certificate.subject_id,certificate.parent_certificate_id,0, - canonical(certificate).decode()),write=True) - if csr_id is not None: - self.set_setting('certificate_request:'+certificate.resource_id,csr_id) - await self.transition_csr(CertificateRequestState(request_id=csr_id,status='issued', - generation=expected_generation+1,certificate_id=certificate.resource_id),expected_generation) - - async def revoke_certificate(self, id, event): - await self.certificate(id) - self.execute("UPDATE certificates SET revoked=1 WHERE id=?",(id,),write=True) - self.set_setting('authorization_epoch',self.setting('authorization_epoch',0)+1) - await self.append_audit(event) - - async def transfer(self,id): - row=self.one("SELECT body FROM transfers WHERE id=?",(id,)) - require(row is not None,"transfer_not_found") - return decode(TransferSession,loads(row[0])) - - async def save_transfer(self,session,expected_generation): - if expected_generation is None: - self.execute("INSERT INTO transfers (id,subject,generation,body) VALUES (?,?,?,?)", - (session.id,session.subject_id,session.generation,canonical(session).decode()),write=True) - else: - require(session.generation==expected_generation+1,"invalid_generation") - count=self.execute("UPDATE transfers SET generation=?,body=? WHERE id=? AND generation=?", - (session.generation,canonical(session).decode(),session.id,expected_generation),write=True).rowcount - require(count==1,"generation_conflict") - - async def put_chunk(self,chunk): - self.check(True) - start,end=chunk.offset,chunk.offset+chunk.content.size - rows=self.rows("SELECT offset,length,body FROM chunks WHERE transfer_id=? AND offset?", - (chunk.transfer_id,end,start)) - if rows: - require(len(rows)==1 and rows[0][0]==start and rows[0][1]==chunk.content.size and - decode(TransferChunk,loads(rows[0][2])).content==chunk.content,"chunk_conflict") - return - self.execute("INSERT INTO chunks VALUES (?,?,?,?)",(chunk.transfer_id,start,chunk.content.size, - canonical(chunk).decode()),write=True) - - async def missing_ranges(self,id,cursor=None,limit=50): - session=await self.transfer(id) - require(session.expected_size is not None,"size_required") - require(1<=limit<=500,"invalid_limit") - start=int(cursor or 0) - require(0<=start<=session.expected_size,"invalid_cursor") - gaps=[] - for row in self.execute("SELECT offset,length FROM chunks WHERE transfer_id=? AND offset+length>? ORDER BY offset",(id,start)): - if row[0]>start: - gaps.append((start,row[0])) - if len(gaps)>limit: - return Page(items=tuple(gaps[:limit]),next_cursor=str(gaps[limit][0])) - start=max(start,row[0]+row[1]) - if startlimit else None) - - async def append_event(self,event): - self.execute("INSERT INTO events (id,body) VALUES (?,?)",(event.id,canonical(event).decode()),write=True) - - async def append_audit(self,event): - previous=self.one("SELECT digest FROM audit ORDER BY seq DESC LIMIT 1") - prev=previous[0] if previous else None - data=wire(replace(event,previous_digest=prev,entry_digest='')) - data.pop('entry_digest') - entry=replace(event,previous_digest=prev,entry_digest=digest(data)) - self.execute("INSERT INTO audit (digest,previous,body) VALUES (?,?,?)", - (entry.entry_digest,prev,canonical(entry).decode()),write=True) - - async def enqueue(self,job): - self.execute("INSERT INTO jobs VALUES (?,?,?,?,?,?)",(job.id,job.dedupe_key,job.kind, - job.state,wire(job.next_attempt_at),canonical(job).decode()),write=True) - - async def job(self,id): - row=self.one("SELECT body FROM jobs WHERE id=?",(id,)) - require(row is not None,'job_not_found') - return decode(EffectJob,loads(row[0])) - - async def save_job(self,job): - self.execute("UPDATE jobs SET state=?,next_at=?,body=? WHERE id=?",(job.state, - wire(job.next_attempt_at),canonical(job).decode(),job.id),write=True) - - def setting(self,key,default=None): - row=self.one("SELECT value FROM settings WHERE key=?",(key,)) - return loads(row[0]) if row else default - - def set_setting(self,key,value): - self.execute("INSERT INTO settings VALUES (?,?) ON CONFLICT(key) DO UPDATE SET value=excluded.value", - (key,canonical(value).decode()),write=True) - class SqliteMetadataStore: def __init__(self,path: Path, *, busy_timeout: float=10): From 0ef416ec99e20604de2398622e72d7f479a84ceb Mon Sep 17 00:00:00 2001 From: LIghtJUNction Date: Tue, 29 Sep 2026 05:02:57 +0800 Subject: [PATCH 10/11] ci: remove one-shot assembly tooling before final metadata validation --- .../workflows/assemble-metadata-session.yml | 76 ------------------- 1 file changed, 76 deletions(-) delete mode 100644 .github/workflows/assemble-metadata-session.yml diff --git a/.github/workflows/assemble-metadata-session.yml b/.github/workflows/assemble-metadata-session.yml deleted file mode 100644 index a108093f..00000000 --- a/.github/workflows/assemble-metadata-session.yml +++ /dev/null @@ -1,76 +0,0 @@ -name: Assemble reviewed metadata session extraction -on: - push: - branches: [build/metadata-session-boundary-166] - paths: [.github/workflows/assemble-metadata-session.yml] -permissions: - contents: write -jobs: - assemble: - if: github.repository == 'TokenNotIncluded/msg.lmm.best' && github.ref == 'refs/heads/build/metadata-session-boundary-166' - runs-on: ubuntu-latest - timeout-minutes: 5 - steps: - - uses: actions/checkout@v7 - with: - fetch-depth: 0 - - name: Require the unchanged regression-first source - env: - INPUT_SHA: ${{ github.sha }} - run: | - test "$(git rev-parse HEAD)" = "$INPUT_SHA" - git merge-base --is-ancestor 7f8752e37fe5b5a8021f192b87e731caec2bbc2d HEAD - git diff --exit-code 7f8752e37fe5b5a8021f192b87e731caec2bbc2d -- src tests docs - - name: Replay the exact audited extraction with its trailing-blank-line fix - run: | - python - <<'PY' - import subprocess, textwrap - recipe_commit='1dc01c0b120642fd0438266085109cdaa6132d46' - recipe_path='.github/workflows/assemble-metadata-session.yml' - workflow=subprocess.check_output(['git','show',recipe_commit+':'+recipe_path],text=True) - step=workflow.index(' - name: Extract reviewed session methods without executing project code\n') - opening=" python - <<'PY'\n" - begin=workflow.index(opening,step)+len(opening) - end=workflow.index(' PY\n',begin) - source=textwrap.dedent(workflow[begin:end]) - old="after['common']=common_header+models+'\\n'+cursor+common+'\\n'" - new="after['common']=common_header+models+'\\n'+cursor+common.rstrip()+'\\n'" - assert source.count(old)==1, 'unexpected pinned recipe' - source=source.replace(old,new) - exec(compile(source,recipe_commit+':metadata-session-extraction','exec')) - PY - - name: Commit only the audited source extraction - run: | - git add src/msg/storage/session.py src/msg/storage/sqlite.py src/msg/storage/postgres.py docs/METADATA_SESSION_PROGRESS.md - git diff --cached --check - python - <<'PY' - import json, subprocess - from pathlib import Path - expected=set(json.loads(Path('/tmp/metadata-session-assembly/audit.json').read_text())['changed_paths']) - actual=set(subprocess.check_output(['git','diff','--cached','--name-only'],text=True).splitlines()) - assert actual == expected, (actual,expected) - PY - git -c user.name='github-actions[bot]' -c user.email='41898282+github-actions[bot]@users.noreply.github.com' commit -m 'refactor: separate shared metadata invariants from concrete database sessions' - - name: Publish dedicated build branch without force or workflow changes - env: - INPUT_SHA: ${{ github.sha }} - run: | - test "$(git ls-remote origin refs/heads/build/metadata-session-boundary-166 | cut -f1)" = "$INPUT_SHA" - git push origin HEAD:refs/heads/build/metadata-session-boundary-166 - python - <<'PY' - import json, os, subprocess - from pathlib import Path - git=lambda *args: subprocess.check_output(['git',*args],text=True).strip() - Path('/tmp/metadata-session-assembly/source.json').write_text(json.dumps({ - 'input':os.environ['INPUT_SHA'],'commit':git('rev-parse','HEAD'), - 'tree':git('rev-parse','HEAD^{tree}'),'dirty':bool(git('status','--porcelain')), - 'recipe_commit':'1dc01c0b120642fd0438266085109cdaa6132d46', - 'recipe_adjustment':'trim one extra blank line at common module EOF'},indent=2)+'\n') - PY - git archive --format=tar.gz -o /tmp/metadata-session-assembly/source.tar.gz HEAD - - uses: actions/upload-artifact@v4 - if: always() - with: - name: metadata-session-assembly - path: /tmp/metadata-session-assembly/ - if-no-files-found: error From 8d17498d324eacf207fcf896ddb9fcc63186b1f1 Mon Sep 17 00:00:00 2001 From: LIghtJUNction Date: Tue, 29 Sep 2026 05:15:19 +0800 Subject: [PATCH 11/11] test: retain shared regression coverage while consolidating duplicate repairs Refs #156 #157 #166 #169 #83. Keep all 16 metadata contracts and three independent assembly/rollback cases on the single integrated implementation; extend existing cloud suites without changing production code or weakening assertions. --- .github/workflows/architecture-ports.yml | 3 +- .github/workflows/storage-session.yml | 3 +- docs/ISSUE_PROGRESS.md | 64 +++----- tests/test_execution_composition.py | 106 +++++++++++++ tests/test_metadata_session_boundary.py | 190 +++++++++++++++++++++++ 5 files changed, 321 insertions(+), 45 deletions(-) create mode 100644 tests/test_execution_composition.py create mode 100644 tests/test_metadata_session_boundary.py diff --git a/.github/workflows/architecture-ports.yml b/.github/workflows/architecture-ports.yml index d9135870..7782b432 100644 --- a/.github/workflows/architecture-ports.yml +++ b/.github/workflows/architecture-ports.yml @@ -6,6 +6,7 @@ on: paths: - 'src/msg/**' - 'tests/test_architecture_ports.py' + - 'tests/test_execution_composition.py' - '.github/workflows/architecture-ports.yml' workflow_dispatch: permissions: @@ -69,7 +70,7 @@ jobs: PY - name: Exercise boundaries and unchanged security contracts run: >- - python -m pytest tests/test_architecture_ports.py tests/test_batch.py + python -m pytest tests/test_architecture_ports.py tests/test_execution_composition.py tests/test_batch.py tests/test_batch_secret_release.py tests/test_tools.py tests/test_webhook_domain_events.py tests/test_read_projection_cache_transports.py tests/test_read_projection_cache_unit.py tests/issue_78_80 --junitxml=artifacts/architecture-ports.xml diff --git a/.github/workflows/storage-session.yml b/.github/workflows/storage-session.yml index fe5b5aaa..ad6dc980 100644 --- a/.github/workflows/storage-session.yml +++ b/.github/workflows/storage-session.yml @@ -6,6 +6,7 @@ on: paths: - 'src/msg/**' - 'tests/test_session_boundary.py' + - 'tests/test_metadata_session_boundary.py' - '.github/workflows/storage-session.yml' workflow_dispatch: permissions: @@ -69,7 +70,7 @@ jobs: PY - name: Exercise boundaries and unchanged security contracts run: >- - python -m pytest tests/test_session_boundary.py tests/test_foundation.py + python -m pytest tests/test_session_boundary.py tests/test_metadata_session_boundary.py tests/test_foundation.py tests/test_postgres.py tests/test_security_rollback_fences.py tests/test_security_storage_rollback.py tests/test_ledger_accounts.py --junitxml=artifacts/storage-session.xml diff --git a/docs/ISSUE_PROGRESS.md b/docs/ISSUE_PROGRESS.md index 64546fc1..b4ce7e64 100644 --- a/docs/ISSUE_PROGRESS.md +++ b/docs/ISSUE_PROGRESS.md @@ -1,62 +1,40 @@ # Issue 推进笔记 -本轮入口：2026-09-28T20:34Z 用户请求。源码、测试证据、合并及现场验收分开记录；不以局部绿灯关闭总体验收。 +用户入口：2026-09-28T20:34Z。PR #167 是本轮 #156/#157 的唯一组合交付入口；基准 main476e64f04ed1d9c7dcab4a41c4f3cdb72faad6a7，tree40ca207a3c49be347120e058e47ed328ec4e3098。最终提交/CI/合并状态以 #167 最新验收评论为准；本文只登记已取得的证据，不用历史绿灯代替新头。 -## 本轮交付入口 +## 已完成的源码切片 -PR #167 集成 #156（执行器边界）和 #157（存储会话边界）。两组独立红基线与实现历史均保留，只在同一集成头做最终验收，避免分别合并后重复跑全量。 +#156：core.batching、events、packet、schemas 各有一份中立实现，原导入路径为同对象兼容别名；URL秘密拒绝和gzip仍在传输层。executor没有Application/插件反向引用，只接收固定投影和事务通知接口。HTTP/SSH由Application.new_executor统一装配请求上限、秘密响应和恢复隔离。签名、ID、回放、权限、事务和receipt规则保留。 -基准 main：476e64f04ed1d9c7dcab4a41c4f3cdb72faad6a7 / tree40ca207a3c49be347120e058e47ed328ec4e3098。该基准 CI36472507327 和 Recovery safety36472507300 已成功，不代表本次新头通过。 +#157：PG/SQLite独立继承不持有驱动/连接的RelationalSession。SQL、schema、迁移、锁、保存点、补偿和提交信号仍由后端管理。显式QuerySession覆盖真实消费者，MetadataSession补齐path/ancestors/organization/job/save_job；查询结果视图每次读取重验任务/生命周期，不公开connection/commit。没有新增ORM、逐表Repository或任意SQL网络接口。 -已阅读权威设计、有效完成归档、仓库贡献规则与当前源码。最终运行状态与合并结果以 PR #167 最新验收评论为准；下列证据各自绑定明确的历史头，不互相冒用。 +SQLite仍是精简测试后端。政策消费者测试只在隔离fixture复用生产仲裁表DDL，验证真实已存政策及损坏digest拒绝，不修改生产迁移或声称完整市场后端等价。 -## #156 执行器与公共协议 +## 测试先行证据（均已下载并独立核验） -已实现：固定 ResultProjection、TransactionalEventNotifications 接口；Application 同一工厂装配 HTTP/SSH 的请求上限、秘密响应释放和恢复隔离。executor 不再持有整棵 Application，也不导入插件内部函数。 +| 范围 | 测试头 / tree | run / artifact | JUnit | +|---|---|---|---| +| #156 | 453d6f858e19136261278c7042c13598035aef1a / ae0b4f5e31dc1844c1abd0142da898b75b7779d4 | 36481310584 / 10997043729 | 7 failed / 158 passed / 0 errors / 0 skipped | +| #157 | 2daf1e0b81783d8b52685540b041ba9563333282 / e1668ddcbf5879bce9520a31f8bfe911c813370c | 36482022051 / 10998115401 | 8 failed / 35 passed / 0 errors / 0 skipped | -core.batching/core.events/core.packet/core.schemas 各自只有一份逻辑；原公开导入路径重导出同一对象。URL秘密拒绝和gzip规则仍在传输层；签署字节、确定性事件ID、批处理排除项、当前权限、回放、事务和receipt规则保持。 +新增用例准确失败，原有选定用例全部通过。ZIP SHA256分别为c2e1978649b6e251e38703befb0fc674fbb336db55fa075113af0edb07d81fb7、fe037acb748babcb05dcdc7c679bb42986f7ae509af86612ce510307c90ed3e3；source.json匹配head/tree/Python3.15.0rc2/attempt1。 -红基线：453d6f858e19136261278c7042c13598035aef1a / treeae0b4f5e31dc1844c1abd0142da898b75b7779d4。 +实现历史：10f46fd300ef0fa4890ca0ea7aad3d10bca7f7a5（执行器）、4a6b852961b5e9745c1fea8d37117a7a7c1bde58（存储），组合a85b74456be8eb20977ddd144b94483b0b21202f/tree042422943d4f5a9c43e5d531797dcc3d9c92a787。该历史组合存储专项36483874055已success；后续扩展回归头仍须重新验收。 -- 云端36481310584，job109127358396：165 tests，7 failed / 158 passed / 0 errors / 0 skipped。 -- 七个新增回归全部失败，原有158个用例全部通过。 -- artifact10997043729已下载核验，ZIP SHA256 c2e1978649b6e251e38703befb0fc674fbb336db55fa075113af0edb07d81fb7。 -- source.json匹配上述head/tree，Python3.15.0rc2，attempt1。 +## 重叠PR收敛 -源码实现10f46fd300ef0fa4890ca0ea7aad3d10bca7f7a5 / treee9e33a3286f6d4efd19a421834bba813448f6a79；并发与笔记补充20b6d7e3c6c2d22a8f9e1ae7f2f89e1a1812dc27 / tree40e3f60f6689435b2bfc39e6c7c63f115beadd16。独立头的运行结果不可替代组合头验收。 +已对照#166和#169完整改动及用例，在#83评论5878614115、#166评论5878687666、#169评论5878694971协调，不覆盖其源分支。 -## #157 存储会话与真实接口 +保留#166全部16个结构/双后端行为测试，仅将拟议基类名改为唯一RelationalSession；补充#169的独立装配、工厂批处理回放、事件通知失败整事务回滚3个用例。签名/认证/事务和失败不提交断言未降低。#169的missing-decoder配置用例不适用于内置中立codec，不引入第二套decoder/BatchPolicy制造该错误；既有限长、秘密拒绝、投影缺失拒绝均保留。原red证据仍归#166/#169，不冒充本分支新红基线。 -已实现：PostgresSession 和 SqliteSession 同为 RelationalSession 的子类；共用域方法不再持有连接或导入数据库驱动。两后端各自保留连接、SQL转换、schema迁移、隔离、写栅栏、嵌套事务、回滚补偿与提交后信号。 +## 合并门槛与执行边界 -QuerySession明确execute/one/rows/setting/set_setting，MetadataSession继承该接口；补齐真实跨模块使用的path/ancestors/organization/job/save_job。check和run_rollback_effects仍属存储生命周期实现，不作为调用者权限。 +两个专项、完整四分片、精确node-ID全集/互斥、conformance/实际tokenizer、wheel/sdist、Recovery safety均保留。同一来源仓库分支的push/PR共享并发组以减少重复runner，不删测试、放宽断言或降低合并门槛。最终组合通过并合并后才关闭两个已修复issue和重复PR。 -execute返回受会话约束的结果视图，不公开驱动connection/commit。rowcount、fetchone、fetchall与迭代重新检查任务归属和事务生命周期。没有增加ORM、逐表Repository或任意SQL网络接口，也不把进程内Python视作安全沙箱。 +项目执行全部在GitHub Actions；本地仅源文本/Git差异与树检查、下载证据读取。临时精确树发布/源码导出仅在辅助分支，不进入产品树。42个共享方法主体、两后端Store/schema/迁移的静态对照未发现改动，云端运行仍是验收依据。 -红基线：2daf1e0b81783d8b52685540b041ba9563333282 / treee1668ddcbf5879bce9520a31f8bfe911c813370c。 +## 剩余与并行工作 -- 云端36482022051，job109129719949：43 tests，8 failed / 35 passed / 0 errors / 0 skipped。 -- 八个新增回归全部失败，原有35个用例全部通过，包含真实PG和SQLite。 -- artifact10998115401已下载核验，ZIP SHA256 fe037acb748babcb05dcdc7c679bb42986f7ae509af86612ce510307c90ed3e3。 -- source.json匹配上述head/tree，Python3.15.0rc2，attempt1。 +#165工具、#168市场、#158客户端及#155恢复由独立分支推进，不覆盖。#155已知0e4dbe7头完整36480031758及专项36480031762成功，但仍须当前main组合与本机CLI/文件边界验收。#68–83/#85各功能总体验收不能因这两个架构修复整体关闭。 -源码实现4a6b852961b5e9745c1fea8d37117a7a7c1bde58 / tree32e13775b0e2e0e85d240bebccae8fd2d56adfa8；组合提交再补齐五个域接口与更强的政策消费者回归。 - -SQLite是精简测试后端，不是完整市场生产bootstrap。消费者测试的SQLite隔离fixture复用生产_SCHEMA中的arbitration_policies DDL；不修改生产迁移、不声称全市场后端等价。原断言全部保留，另增加真实已存政策读取、损坏digest拒绝和公开域方法完整声明检查，不只走默认空表分支。 - -## 集成验收门槛 - -保留两个专项、完整四分片、精确node-ID全集/互斥校验、conformance、实际tokenizer、wheel/sdist和Recovery safety。CI只把同一来源仓库分支的push/PR归到同一并发组，减少重复runner，不删用例、放宽断言、减少依赖或取消合并门槛。 - -所有项目执行放在GitHub Actions；本地只编辑源文本、检查Git差异/树与读取下载证据。临时源码导出/发布脚本与工作流均只在work/issue-workspace-20260929，不进入产品分支。发布阶段验证脚本SHA256和完整预期Git树，并只写本会话暂存分支。 - -本笔记提交时，最终组合验收尚未完成；不标已合并，也不据此关闭#83/#85。最终证据在PR #167验收评论中逐项登记。 - -## 并行协作与剩余项 - -- #155恢复分支0e4dbe777053e354bcf4845b851a0c9b86ab8310：专项36480031762成功，完整36480031758四分片/conformance/build/node-ID gate成功；需与最新main冲突处理后的组合验收。本会话不覆盖该分支。 -- #160由fix/architecture-boundaries-20260929处理；#159已出现PR #168市场边界回归。本会话不重复写这些分支。 -- #158客户端/服务端依赖、#68–70恢复完整验收、#71–72市场矩阵、#76–82各功能验收、#83配置/Registry/逐条证据、#85其余模型收敛仍按对应issue/PR跟踪，不因本次两个架构修复整体关闭。 -- #64/#65/#84仍缺授权的生产代理/日志、真实旧库快照、容量与恢复材料，CI不能替代。#68/#69/#70独立新鲜检查点、备份退役、真实控制台及promotion必须分别验收。 - -本次不部署、不操作真实Root/PIN、资金、外发、备份销毁或恢复放行。 +#64/#65/#84仍缺授权生产代理/日志、真实旧库快照、容量与恢复材料；#68–70独立新鲜检查点、备份退役、真实控制台及promotion须分别验收，CI不能替代。此次不部署、不操作真实Root/PIN、资金、外发、备份销毁或恢复放行。 diff --git a/tests/test_execution_composition.py b/tests/test_execution_composition.py new file mode 100644 index 00000000..15f143fc --- /dev/null +++ b/tests/test_execution_composition.py @@ -0,0 +1,106 @@ +"""Retain PR #169's useful assembly/rollback cases on the one executor port. + +Source cases: f7f8a890d91e4eda5d3da5f3c2ac4d8988e70848. No alternative +BatchPolicy/decoder/factory is installed solely to satisfy a competing API. +""" +from contextlib import asynccontextmanager +from datetime import timedelta +from types import SimpleNamespace + +from msg.core.codec import wire +from msg.core.errors import Failure +from msg.core.executor import OperationExecutor +from msg.core.models import HandlerOutput, Principal, ResourceRef +from msg.core.requests import request_for +from test_service import NOW, call, register +from test_batch import packet + + +async def test_core_executor_runs_with_explicit_services_and_no_application(): + active = [] + transaction = object() + principal = Principal(actor='u_test', subject='u_test', credential_id='k_test', + method='signature', certificates=(), ceiling=()) + + @asynccontextmanager + async def transaction_scope(*, write): + assert write is False + active.append(transaction) + try: + yield transaction + finally: + active.pop() + + async def authenticate(request, tx, *, entry): + assert tx is transaction and entry == 'network' + return principal + + async def requirements(request, tx): + return () + + async def authorize(context, request, checks, tx): + assert context.principal is principal and tx is transaction + + async def handler(context, request, tx): + return HandlerOutput(resources=(ResourceRef(id='r_test'),), data={'original': True}) + + async def project(context, request, tx, ref, *, fields): + assert active == [transaction] and tx is transaction + assert request.return_fields == fields == ('id',) + return {'id': ref.id} + + spec = SimpleNamespace(name='testing.read', version=1, entries={'network'}, effect='read', + input_schema=None, output_schema=None, requirements=requirements, handler=handler) + registry = SimpleNamespace(operation=lambda *args: spec, validate=lambda *args: None) + executor = OperationExecutor(registry, SimpleNamespace(transaction=transaction_scope), None, + SimpleNamespace(authenticate=authenticate), SimpleNamespace(require=authorize), + lambda: NOW, None, result_projection=project) + assert not hasattr(executor, 'application') + result = await executor.execute(request_for('testing.read', {}, 'https://unit.invalid', + return_fields=('id',))) + assert result.status == 'ok', result + assert result.data == {'original': True, 'projection': [{'id': 'r_test'}]} + assert active == [] + + +async def test_factory_batch_and_projection_work_without_application_backreference(installed): + app, _ = installed + key, subject, _ = await register(app, 'executor-factory') + app.executor = app.new_executor() + child = packet(app, key, subject, 'content.post_create', + {'parent': '/main', 'body': 'boundaries'}, 'factory-child') + result = await call(app, 'batch.atomic', {'requests': [child]}, + key=key, subject=subject, rid='factory-parent') + assert result.status == 'ok', wire(result) + assert result.data['results'][0]['status'] == 'ok' + repeated = await call(app, 'batch.atomic', {'requests': [child]}, + key=key, subject=subject, rid='factory-parent') + assert repeated.replayed + projected = await app.executor.execute(request_for('content.post_create', + {'parent': '/main', 'body': 'project'}, app.settings.service_url, + subject=subject, signer=key, expires_at=NOW + timedelta(seconds=60), + return_fields=('id', 'size'))) + assert projected.status == 'ok', wire(projected) + assert projected.data['projection'][0]['size'] == 7 + + +async def test_event_projection_failure_rolls_back_the_existing_transaction(installed): + app, _ = installed + key, subject, _ = await register(app, 'event-rollback') + app.executor = app.new_executor() + seen = [] + + async def reject(tx, event): + assert tx.one('SELECT id FROM events WHERE id=?', (event.id,)) is not None + seen.append(event.id) + raise Failure('test_event_projection_abort') + + app.executor.event_notifications = reject + result = await call(app, 'content.post_create', {'parent': '/main', 'body': 'not committed'}, + key=key, subject=subject, rid='event-rollback-request') + assert result.error.code == 'test_event_projection_abort', wire(result) + assert len(seen) == 1 + async with app.metadata.transaction(write=False) as tx: + assert tx.one("SELECT COUNT(*) FROM resources WHERE type='post'")[0] == 0 + assert tx.one('SELECT id FROM events WHERE id=?', (seen[0],)) is None + assert tx.one('SELECT request_id FROM results WHERE request_id=?', ('event-rollback-request',)) is None diff --git a/tests/test_metadata_session_boundary.py b/tests/test_metadata_session_boundary.py new file mode 100644 index 00000000..c35cb79b --- /dev/null +++ b/tests/test_metadata_session_boundary.py @@ -0,0 +1,190 @@ +"""PR #166 contracts retained against the single integrated session owner. + +Adapted from 0ef416ec99e20604de2398622e72d7f479a84ceb. Only the proposed +base-class name changes; every structural and behavioral assertion remains. +""" +from __future__ import annotations + +import asyncio +from dataclasses import replace +from datetime import UTC, datetime +import importlib +import inspect +from pathlib import Path +import subprocess +import sys + +import pytest + +from msg.core.contracts import MetadataSession, MetadataStore +from msg.core.errors import Failure +from msg.core.models import AuditEvent, EffectJob, Event, OperationResult, Principal +from test_postgres import item + + +def test_postgres_import_does_not_load_a_sqlite_adapter_or_driver(): + code = ''' +import importlib.abc, sys +class NoSqlite(importlib.abc.MetaPathFinder): + def find_spec(self, fullname, path=None, target=None): + if fullname == 'msg.storage.sqlite' or fullname == 'sqlite3' or fullname.startswith('sqlite3.'): + raise AssertionError('PostgreSQL imported SQLite: ' + fullname) +sys.meta_path.insert(0, NoSqlite()) +import msg.storage.postgres +assert 'msg.storage.sqlite' not in sys.modules +assert 'sqlite3' not in sys.modules +''' + result = subprocess.run([sys.executable, '-c', code], capture_output=True, text=True) + assert result.returncode == 0, result.stderr + + +def test_backends_share_only_the_neutral_session_implementation(): + from msg.storage.postgres import PostgresSession + from msg.storage.sqlite import SqliteSession + common = importlib.import_module('msg.storage.session').RelationalSession + assert not issubclass(PostgresSession, SqliteSession) + assert PostgresSession.__bases__ == SqliteSession.__bases__ == (common,) + for name in ('resource','children','save_result','append_audit','on_rollback','check'): + assert getattr(PostgresSession, name) is getattr(SqliteSession, name) is getattr(common, name) + assert PostgresSession.execute is not SqliteSession.execute + assert 'execute' in common.__abstractmethods__ + + +def test_shared_layer_does_not_own_connections_dialects_schemas_or_commits(): + module = importlib.import_module('msg.storage.session') + text = Path(module.__file__).read_text() + for forbidden in ('sqlite3', 'psycopg', '_connection', 'PRAGMA ', 'CREATE TABLE ', + 'INSERT OR IGNORE', '.commit(', '.rollback(', 'pg_advisory'): + assert forbidden not in text + assert set(module.RelationalSession.__abstractmethods__) == {'execute'} + assert set(inspect.signature(module.RelationalSession).parameters) == {'write'} + + +def test_shared_implementation_covers_the_declared_metadata_contract(): + common = importlib.import_module('msg.storage.session').RelationalSession + for name, member in vars(MetadataSession).items(): + if not name.startswith('_') and callable(member): + assert name in vars(common), name + assert callable(getattr(common, name)), name + assert 'transaction' in vars(MetadataStore) + assert {'one','rows','check','setting','set_setting','job','save_job', + 'organization','path','ancestors','run_rollback_effects'} <= set(vars(common)) + + +@pytest.fixture(params=['postgres','sqlite']) +async def metadata(request, pg_dsn, tmp_path): + from msg.storage.postgres import PostgresMetadataStore + from msg.storage.sqlite import SqliteMetadataStore + store = (PostgresMetadataStore(pg_dsn) if request.param == 'postgres' else + SqliteMetadataStore(tmp_path/'metadata.sqlite')) + try: + yield store + finally: + await store.close() + + +async def test_shared_resource_generation_path_and_paging(metadata): + async with metadata.transaction(write=True) as tx: + await tx.insert(item()) + await tx.insert(item('a','root')) + await tx.insert(item('b','root')) + assert await tx.resolve('/a') == 'a' + assert await tx.path('a') == '/a' + assert [r.id for r in await tx.ancestors('a')] == ['root'] + first = await tx.children('root', limit=1) + assert [r.id for r in first.items] == ['a'] and first.next_cursor == 'a' + assert [r.id for r in (await tx.children('root', cursor=first.next_cursor)).items] == ['b'] + old = await tx.resource('a') + await tx.replace(replace(old, generation=1, mode=0o700), 0) + assert tx.setting('authorization_epoch') == 1 + with pytest.raises(Failure, match='generation_conflict'): + await tx.replace(replace(old, generation=1), 0) + async with metadata.transaction(write=False) as tx: + assert (await tx.resource('a')).generation == 1 + assert (await tx.resource('a')).mode == 0o700 + + +async def test_shared_result_event_audit_and_job_roundtrips(metadata): + now = datetime.now(UTC) + principal = Principal(actor='owner', subject='owner', credential_id=None, + method='local', certificates=(), ceiling=()) + result = OperationResult(request_id='once', operation='test', status='ok', actor=None, subject=None) + event = Event(id='event-one', type='test', time=now, request_id='once', + actor='owner', subject='owner', resources=(), data={}) + audit = AuditEvent(event=event, authority=(), before_digest=None, after_digest=None, + previous_digest=None, entry_digest='', result='ok') + job = EffectJob(id='job-one', event_id=event.id, kind='mail', dedupe_key='unique-job', + principal=principal, operation='mail.send', arguments={}, state='pending', + attempts=0, next_attempt_at=now, lease_until=None) + async with metadata.transaction(write=True) as tx: + await tx.save_result('owner','digest-one',result) + await tx.append_event(event) + await tx.append_audit(audit) + await tx.append_audit(replace(audit, event=replace(event,id='event-two'))) + await tx.enqueue(job) + async with metadata.transaction(write=False) as tx: + assert await tx.request_result('owner','once','digest-one') == result + with pytest.raises(Failure, match='idempotency_conflict'): + await tx.request_result('owner','once','another-digest') + rows = tx.rows('SELECT digest,previous FROM audit ORDER BY seq') + assert rows[0][1] is None and rows[1][1] == rows[0][0] + assert await tx.job(job.id) == job + assert tx.one('SELECT id FROM events WHERE id=?',(event.id,))[0] == event.id + + +async def test_shared_savepoints_and_compensation_keep_their_scope(metadata): + effects = [] + async def compensate(value): + effects.append(value) + async with metadata.transaction(write=True) as outer: + outer.set_setting('outer', True) + outer.on_rollback(lambda: compensate('outer')) + with pytest.raises(ValueError, match='inner'): + async with metadata.transaction(write=True) as inner: + assert inner is outer + inner.set_setting('inner', True) + inner.on_rollback(lambda: compensate('inner')) + raise ValueError('inner') + assert effects == ['inner'] and outer.setting('inner') is None + assert effects == ['inner'] + with pytest.raises(ValueError, match='whole'): + async with metadata.transaction(write=True) as tx: + tx.set_setting('whole', True) + tx.on_rollback(lambda: compensate('whole')) + raise ValueError('whole') + async with metadata.transaction(write=False) as tx: + assert tx.setting('outer') is True and tx.setting('whole') is None + assert effects == ['inner','whole'] + + +async def test_shared_read_only_closed_and_cross_task_guards(metadata): + async with metadata.transaction(write=False) as tx: + with pytest.raises(Failure, match='read_only_transaction'): + tx.set_setting('forbidden', True) + with pytest.raises(Failure, match='read_only_transaction'): + async with metadata.transaction(write=True): + pytest.fail('read-only transaction was upgraded') + async def other_task(): + tx.setting('anything') + with pytest.raises(Failure, match='transaction_cross_task'): + await asyncio.create_task(other_task()) + with pytest.raises(Failure, match='transaction_closed'): + tx.setting('anything') + + +async def test_each_driver_translates_unique_errors_and_rolls_back(metadata): + async with metadata.transaction(write=True) as tx: + await tx.insert(item()) + with pytest.raises(Failure, match='constraint_conflict'): + async with metadata.transaction(write=True) as tx: + await tx.insert(item('child','root')) + await tx.insert(item('child','root')) + async with metadata.transaction(write=False) as tx: + assert tx.one('SELECT COUNT(*) FROM resources')[0] == 1 + assert tx.one('SELECT id FROM resources WHERE id=?',('child',)) is None + + +async def test_each_driver_preserves_literal_percent_and_question_mark(metadata): + async with metadata.transaction(write=False) as tx: + row = tx.one("SELECT '?', ?, '50%'", ('literal?%',)) + assert tuple(row) == ('?', 'literal?%', '50%')