From 612fc67d32028827a66e762325a3a58879dece76 Mon Sep 17 00:00:00 2001 From: Philip Z Date: Tue, 27 Oct 2026 10:22:43 +0800 Subject: [PATCH 01/75] Start Python reference value and runtime capability gate Refs teaql/teaql-python#66 and teaql/teaql-conformance#241 --- src/teaql/core/__init__.py | 3 +- src/teaql/core/blob_ref.py | 100 +++++++++++++++++++++ src/teaql/runtime/__init__.py | 2 + src/teaql/runtime/capability.py | 48 ++++++++++ src/teaql/runtime/context.py | 5 ++ src/teaql/runtime/module.py | 13 +++ test-vectors/blob-value-cases.json | 45 ++++++++++ tests/runtime/test_blob_reference_value.py | 78 ++++++++++++++++ 8 files changed, 293 insertions(+), 1 deletion(-) create mode 100644 src/teaql/core/blob_ref.py create mode 100644 src/teaql/runtime/capability.py create mode 100644 test-vectors/blob-value-cases.json create mode 100644 tests/runtime/test_blob_reference_value.py diff --git a/src/teaql/core/__init__.py b/src/teaql/core/__init__.py index 688c9db7..dabf482a 100644 --- a/src/teaql/core/__init__.py +++ b/src/teaql/core/__init__.py @@ -1,4 +1,5 @@ from .value import Value, DataType, Timestamp +from .blob_ref import BlobRef, BlobFieldSpec from .entity import BaseEntityData, EntityKey, EntityChangeSet, EntityRoot from .expr import ( Expr, ExprBuilder, BinaryOp, ExprFunction, @@ -46,7 +47,7 @@ __all__ = [ "QueryIntent", "MutationIntent", "RequestIntentError", "QueryOperation", - "Value", "DataType", "Timestamp", + "Value", "DataType", "Timestamp", "BlobRef", "BlobFieldSpec", "BaseEntityData", "EntityKey", "EntityChangeSet", "EntityRoot", "Expr", "ExprBuilder", "BinaryOp", "ExprFunction", "SelectQuery", "SortDirection", "OrderBy", "Aggregate", "AggregateFunction", diff --git a/src/teaql/core/blob_ref.py b/src/teaql/core/blob_ref.py new file mode 100644 index 00000000..8026f39e --- /dev/null +++ b/src/teaql/core/blob_ref.py @@ -0,0 +1,100 @@ +"""Stable Blob catalog values shared with the Java/Rust reference contract.""" + +from __future__ import annotations + +import re + +_KEY = re.compile(r"blob_[A-Za-z0-9_-]{1,123}", re.ASCII) +_MIME = { + "jpg": "image/jpeg", "jpeg": "image/jpeg", "png": "image/png", + "mp4": "video/mp4", "mp3": "audio/mpeg", + "doc": "application/msword", + "docx": "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + "xls": "application/vnd.ms-excel", + "xlsx": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + "pdf": "application/pdf", +} + + +class BlobRef: + """A Key identifies catalog state; it never conveys read or bind authority.""" + + __slots__ = ("_key",) + + def __init__(self, key: str) -> None: + if not isinstance(key, str) or _KEY.fullmatch(key) is None: + raise ValueError("BLOB_REFERENCE_INVALID") + object.__setattr__(self, "_key", key) + + def __setattr__(self, name: str, value: object) -> None: + raise AttributeError("BlobRef is immutable") + + @property + def key(self) -> str: + return self._key + + @classmethod + def from_json(cls, value: object) -> "BlobRef": + if not isinstance(value, str): + raise ValueError("BLOB_REFERENCE_INVALID") + return cls(value) + + @classmethod + def from_sql(cls, value: object) -> "BlobRef": + if not isinstance(value, str): + raise ValueError("BLOB_REFERENCE_INVALID") + return cls(value) + + def to_sql(self) -> str: + return self._key + + def __repr__(self) -> str: + return "BlobRef[masked]" + + __str__ = __repr__ + + def __eq__(self, other: object) -> bool: + return isinstance(other, BlobRef) and self._key == other._key + + def __hash__(self) -> int: + return hash(self._key) + + +class BlobFieldSpec: + """Trusted model restriction; server-selected policy may narrow formats.""" + + __slots__ = ("entity", "field", "nullable", "policy_profile", "allowed_extensions", + "allowed_content_types") + CAPABILITY = "blob.v1" + + def __init__(self, entity: str, field: str, extensions: list[str] | tuple[str, ...], + nullable: bool, policy_profile: str) -> None: + if (not isinstance(entity, str) or not entity or entity != entity.strip() + or not isinstance(field, str) or not field or field != field.strip() + or not isinstance(policy_profile, str) or not policy_profile + or policy_profile != policy_profile.strip()): + raise ValueError("BLOB_POLICY_INVALID") + try: + normalized = tuple(sorted(set(extensions))) + except (TypeError, ValueError): + raise ValueError("BLOB_FORMAT_NOT_ALLOWED") from None + if not normalized or any(extension not in _MIME for extension in normalized): + raise ValueError("BLOB_FORMAT_NOT_ALLOWED") + object.__setattr__(self, "entity", entity) + object.__setattr__(self, "field", field) + object.__setattr__(self, "nullable", nullable) + object.__setattr__(self, "policy_profile", policy_profile) + object.__setattr__(self, "allowed_extensions", normalized) + object.__setattr__(self, "allowed_content_types", + tuple(sorted({_MIME[extension] for extension in normalized}))) + + def __setattr__(self, name: str, value: object) -> None: + raise AttributeError("BlobFieldSpec is immutable") + + @property + def protocol_version(self) -> str: + return self.CAPABILITY + + @property + def max_files(self) -> int: + return 1 diff --git a/src/teaql/runtime/__init__.py b/src/teaql/runtime/__init__.py index e8146ed4..7886207c 100644 --- a/src/teaql/runtime/__init__.py +++ b/src/teaql/runtime/__init__.py @@ -8,6 +8,7 @@ Tools, ToolToken, ToolUnavailableError, ) from .module import RuntimeModule, DefaultEntityDataServiceBehavior +from .capability import install_runtime_capability, validate_capabilities from .graph_session import GraphMutationSession, GraphCommittedError from .store import DataStore from .audit import RawAuditEvent, SafeAuditEvent, MutationAuditKind @@ -66,6 +67,7 @@ __all__ = ["WireFieldMetadata", "WireEntityMetadata", "NormalizedWireInput", "WireInputError", "create_wire_entity_metadata", "normalize_wire_input", "encode_wire_output", "retain_submitted_paths", "EntityKey", "EntityChangeSet", "EntityRoot", "ContextEntityRef", "ContextRootError", "CheckException", "CheckResult", "I18nCatalog", "JsonFieldNamingProfile", "Locale", "ObjectLocation", "UnsupportedLocaleError", "UserContext", "TeaqlRuntime", "SqlLogEntry", "SqlLogOperation", "DiagnosticSqlLogSink", "TextDiagnosticSqlLogSink", "ServiceRuntimeFromEnv", "RuntimeModule", "DataStore", "RawAuditEvent", "SafeAuditEvent", "MutationAuditKind", "BusinessClock", "FixedBusinessClock", "SystemBusinessClock", "InternalIdAllocator", "AeadEntityReferenceCodec", "EntityReferenceClaims", "EntityReferenceCodec", "EntityReferenceTokenError", "ENTITY_REFERENCE_AAD", "UNSAFE_RAW_ENTITY_REFERENCES_ACKNOWLEDGEMENT", "UNSAFE_RAW_ENTITY_REFERENCES_ENVIRONMENT", "BUSINESS_ID_PERMUTATION_V1_ALPHABET", "BUSINESS_ID_PERMUTATION_V1_DOMAIN_SIZE", "BUSINESS_ID_PERMUTATION_V1_MAX_SEQUENCE", "BUSINESS_ID_PERMUTATION_V1_WIDTH", "DefaultBusinessIdProfileFactory", "DefaultBusinessIdService", "InMemoryBusinessIdAllocator", "PermutedDailyBusinessIdProfile", "StaticBusinessIdKeyProvider", "encode_business_id_permutation_v1", "ContextTools", "ExecutableHttpTool", "HTTP_TOOL", "HttpIntentPhase", "HttpTool", "HttpToolProvider", "ToolDeniedError", "ToolError", "ToolPolicy", "ToolRisk", "Tools", "ToolToken", "ToolUnavailableError"] __all__ += [ + "install_runtime_capability", "validate_capabilities", "Checker", "CheckerRegistry", "DiagnosticTelemetryAdapter", "RequestPolicy", "RequestPolicyInput", "RuntimeTelemetryAdapter", "RuntimeTelemetryInput", "SafeAuditEventSink", "SchemaCapability", diff --git a/src/teaql/runtime/capability.py b/src/teaql/runtime/capability.py new file mode 100644 index 00000000..3b833a52 --- /dev/null +++ b/src/teaql/runtime/capability.py @@ -0,0 +1,48 @@ +"""Passive optional-module resource slots and explicit readiness checks.""" + +from __future__ import annotations + +from inspect import isawaitable +from typing import Protocol, TYPE_CHECKING + +if TYPE_CHECKING: + from .context import UserContext + + +class RuntimeCapabilityResource(Protocol): + @property + def capability(self) -> str: ... + + def validate_ready(self, context: "UserContext") -> object: ... + + +def _slot(name: str) -> str: + return f"teaql.runtime.capability:{name}" + + +def install_runtime_capability(context: "UserContext", resource: RuntimeCapabilityResource) -> None: + name = getattr(resource, "capability", None) + if not isinstance(name, str) or not name or name != name.strip() \ + or not callable(getattr(resource, "validate_ready", None)): + raise ValueError("RUNTIME_CAPABILITY_INVALID") + if context.get_resource(_slot(name)) is not None: + raise ValueError("RUNTIME_CAPABILITY_DUPLICATE") + context.insert_resource(_slot(name), resource) + + +async def validate_capabilities(context: "UserContext") -> None: + for name in sorted(context._required_capabilities): + resource = context.get_resource(_slot(name)) + if resource is None or getattr(resource, "capability", None) != name: + raise RuntimeError(_missing(name)) + try: + result = resource.validate_ready(context) + if isawaitable(result): + await result + except Exception: + raise RuntimeError(_missing(name)) from None + + +def _missing(name: str) -> str: + return {"blob.v1": "BLOB_CAPABILITY_MISSING", + "secure_document.v1": "DOCUMENT_CONFIGURATION"}.get(name, "RUNTIME_CAPABILITY_MISSING") diff --git a/src/teaql/runtime/context.py b/src/teaql/runtime/context.py index f816b70a..c2f14dab 100644 --- a/src/teaql/runtime/context.py +++ b/src/teaql/runtime/context.py @@ -101,6 +101,7 @@ def invalidate(self, query_key: str) -> None: class UserContext: def __init__(self): self._resources: Dict[str, Any] = {} + self._required_capabilities: frozenset[str] = frozenset() self._resources["sql_log_options"] = SqlLogOptions.all() self._resources["diagnostic_sql_log_sink"] = TextDiagnosticSqlLogSink() self._metadata: Optional[Any] = None @@ -237,6 +238,8 @@ async def execute_graph_save(self, work, *, comment=None): lambda: self._execute_graph_transaction(work, intent, GraphMutationSession)) async def _execute_graph_transaction(self, work, intent, session_type): + from .capability import validate_capabilities + await validate_capabilities(self) provider = self.require_resource("dataService") begin = getattr(provider, "begin", None) if not callable(begin): @@ -618,6 +621,8 @@ def set_schema_provider(self, provider: SchemaCapability): self.with_schema_provider(provider) async def ensure_schema(self): + from .capability import validate_capabilities + await validate_capabilities(self) provider = self.get_resource("schema_provider") if provider: from ._schema_capability import SCHEMA_CAPABILITY diff --git a/src/teaql/runtime/module.py b/src/teaql/runtime/module.py index 14cbe9d9..ca012a7e 100644 --- a/src/teaql/runtime/module.py +++ b/src/teaql/runtime/module.py @@ -11,6 +11,7 @@ def __init__(self): self._generated_bootstraps: List[Any] = [] self._schema_entities: List[Any] = [] self._wire_metadata: Dict[str, Any] = {} + self._required_capabilities: set[str] = set() @classmethod def new(cls) -> 'RuntimeModule': @@ -60,6 +61,16 @@ def wire_metadata(self, entity: str, metadata: Any) -> 'RuntimeModule': self._wire_metadata[entity] = metadata return self + def require_capability(self, name: str) -> 'RuntimeModule': + if not isinstance(name, str) or not name or name != name.strip() or any(ch.isspace() for ch in name): + raise ValueError("invalid runtime capability") + self._required_capabilities.add(name) + return self + + @property + def required_capabilities(self) -> tuple[str, ...]: + return tuple(sorted(self._required_capabilities)) + def schema_entity(self, descriptor: Any) -> 'RuntimeModule': """Register generated storage metadata without embedding a provider.""" self._schema_entities.append(descriptor) @@ -78,6 +89,7 @@ def and_module(self, other: 'RuntimeModule') -> 'RuntimeModule': ] combined._schema_entities = [*self._schema_entities, *other._schema_entities] combined._wire_metadata = {**self._wire_metadata, **other._wire_metadata} + combined._required_capabilities = self._required_capabilities | other._required_capabilities combined._initial_graphs = [ *getattr(self, '_initial_graphs', []), *getattr(other, '_initial_graphs', []), @@ -89,6 +101,7 @@ def and_module(self, other: 'RuntimeModule') -> 'RuntimeModule': return combined def apply_to(self, context: UserContext): + context._required_capabilities = frozenset(self._required_capabilities) for name, dep in self._dependencies.items(): context.insert_resource(name, dep) installed_entities = self._schema_entities or self._entities diff --git a/test-vectors/blob-value-cases.json b/test-vectors/blob-value-cases.json new file mode 100644 index 00000000..e5775309 --- /dev/null +++ b/test-vectors/blob-value-cases.json @@ -0,0 +1,45 @@ +{ + "schema": "teaql.blob-value-cases/v1", + "valid": [ + "blob_a", + "blob_AZaz09_-", + "blob_xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" + ], + "invalid": [ + "", + "blob_", + "BLOB_a", + "blob_xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", + " blob_a", + "blob_a ", + "blob_a\n", + "blob_汉", + "blob_a/b", + "blob_a.b", + "blob_a?sig=1", + "/tmp/file.pdf", + "https://example.invalid/file.pdf", + "application.docx" + ], + "invalidJson": [ + 0, + true, + {}, + [], + { + "key": "blob_a" + } + ], + "formats": { + "jpg": "image/jpeg", + "jpeg": "image/jpeg", + "png": "image/png", + "mp4": "video/mp4", + "mp3": "audio/mpeg", + "doc": "application/msword", + "docx": "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + "xls": "application/vnd.ms-excel", + "xlsx": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + "pdf": "application/pdf" + } +} diff --git a/tests/runtime/test_blob_reference_value.py b/tests/runtime/test_blob_reference_value.py new file mode 100644 index 00000000..902ccc43 --- /dev/null +++ b/tests/runtime/test_blob_reference_value.py @@ -0,0 +1,78 @@ +import json +from pathlib import Path + +import pytest + +from teaql.core import BlobFieldSpec, BlobRef +from teaql.runtime import RuntimeModule, UserContext, install_runtime_capability, validate_capabilities + + +VECTORS = json.loads((Path(__file__).resolve().parents[2] / "test-vectors" / + "blob-value-cases.json").read_text()) + + +def test_shared_blob_ref_and_format_vectors(): + for key in VECTORS["valid"]: + reference = BlobRef.from_json(key) + assert reference == BlobRef.from_sql(key) + assert reference.to_sql() == key + assert repr(reference) == "BlobRef[masked]" + with pytest.raises(TypeError): + json.dumps(reference) + with pytest.raises(AttributeError): + reference._key = "blob_other" + for value in [*VECTORS["invalid"], *VECTORS["invalidJson"]]: + with pytest.raises(ValueError, match="BLOB_REFERENCE_INVALID"): + BlobRef.from_json(value) + + spec = BlobFieldSpec("Asset", "document", list(reversed(VECTORS["formats"])), + False, "default") + assert spec.allowed_extensions == tuple(sorted(VECTORS["formats"])) + assert spec.allowed_content_types == tuple(sorted(set(VECTORS["formats"].values()))) + with pytest.raises(ValueError, match="BLOB_FORMAT_NOT_ALLOWED"): + BlobFieldSpec("Asset", "document", ["application.docx"], False, "default") + + +class ReadyResource: + def __init__(self, capability: str, failure: bool = False): + self.capability = capability + self.failure = failure + + async def validate_ready(self, context): + if self.failure: + raise RuntimeError("private provider diagnostic") + + +@pytest.mark.asyncio +async def test_module_capabilities_fail_before_graph_transaction_and_schema(): + blob = RuntimeModule.new().require_capability("blob.v1") + secure = RuntimeModule.new().require_capability("secure_document.v1").require_capability("blob.v1") + both = blob.and_module(secure) + assert both.required_capabilities == ("blob.v1", "secure_document.v1") + assert blob.required_capabilities == ("blob.v1",) + context = both.into_context() + entered = False + + async def work(_session): + nonlocal entered + entered = True + + with pytest.raises(RuntimeError, match="BLOB_CAPABILITY_MISSING"): + await context.execute_graph_save(work, comment="save document") + assert not entered + with pytest.raises(RuntimeError, match="BLOB_CAPABILITY_MISSING"): + await context.ensure_schema() + + install_runtime_capability(context, ReadyResource("blob.v1")) + with pytest.raises(RuntimeError, match="DOCUMENT_CONFIGURATION"): + await validate_capabilities(context) + install_runtime_capability(context, ReadyResource("secure_document.v1")) + await validate_capabilities(context) + with pytest.raises(ValueError, match="RUNTIME_CAPABILITY_DUPLICATE"): + install_runtime_capability(context, ReadyResource("blob.v1")) + + failing = RuntimeModule.new().require_capability("blob.v1").into_context() + install_runtime_capability(failing, ReadyResource("blob.v1", failure=True)) + with pytest.raises(RuntimeError, match="BLOB_CAPABILITY_MISSING") as error: + await validate_capabilities(failing) + assert "private provider diagnostic" not in str(error.value) From f30858cef4d9692ef741a9109916b06e07662e0b Mon Sep 17 00:00:00 2001 From: Philip Z Date: Tue, 27 Oct 2026 10:44:44 +0800 Subject: [PATCH 02/75] Add Python detached view preparation and Blob projection boundary Refs teaql/teaql-python#66 teaql/teaql-conformance#241 --- src/teaql/runtime/__init__.py | 10 ++ src/teaql/runtime/view_preparation.py | 234 ++++++++++++++++++++++++++ tests/test_view_preparation_blob.py | 106 ++++++++++++ 3 files changed, 350 insertions(+) create mode 100644 src/teaql/runtime/view_preparation.py create mode 100644 tests/test_view_preparation_blob.py diff --git a/src/teaql/runtime/__init__.py b/src/teaql/runtime/__init__.py index 7886207c..df17f0e9 100644 --- a/src/teaql/runtime/__init__.py +++ b/src/teaql/runtime/__init__.py @@ -10,6 +10,12 @@ from .module import RuntimeModule, DefaultEntityDataServiceBehavior from .capability import install_runtime_capability, validate_capabilities from .graph_session import GraphMutationSession, GraphCommittedError +from .view_preparation import ( + AuthorizedBlobSlot, AuthorizedResponseGraph, BlobResponseContributor, + PreparedSecureDocument, PreparedView, ProjectedBlobURL, + ResponseGraphAuthorizer, SecureDocumentContributor, ViewOptions, + ViewPreparationError, ViewPreparationService, +) from .store import DataStore from .audit import RawAuditEvent, SafeAuditEvent, MutationAuditKind from .business_clock import BusinessClock, FixedBusinessClock, SystemBusinessClock @@ -67,6 +73,10 @@ __all__ = ["WireFieldMetadata", "WireEntityMetadata", "NormalizedWireInput", "WireInputError", "create_wire_entity_metadata", "normalize_wire_input", "encode_wire_output", "retain_submitted_paths", "EntityKey", "EntityChangeSet", "EntityRoot", "ContextEntityRef", "ContextRootError", "CheckException", "CheckResult", "I18nCatalog", "JsonFieldNamingProfile", "Locale", "ObjectLocation", "UnsupportedLocaleError", "UserContext", "TeaqlRuntime", "SqlLogEntry", "SqlLogOperation", "DiagnosticSqlLogSink", "TextDiagnosticSqlLogSink", "ServiceRuntimeFromEnv", "RuntimeModule", "DataStore", "RawAuditEvent", "SafeAuditEvent", "MutationAuditKind", "BusinessClock", "FixedBusinessClock", "SystemBusinessClock", "InternalIdAllocator", "AeadEntityReferenceCodec", "EntityReferenceClaims", "EntityReferenceCodec", "EntityReferenceTokenError", "ENTITY_REFERENCE_AAD", "UNSAFE_RAW_ENTITY_REFERENCES_ACKNOWLEDGEMENT", "UNSAFE_RAW_ENTITY_REFERENCES_ENVIRONMENT", "BUSINESS_ID_PERMUTATION_V1_ALPHABET", "BUSINESS_ID_PERMUTATION_V1_DOMAIN_SIZE", "BUSINESS_ID_PERMUTATION_V1_MAX_SEQUENCE", "BUSINESS_ID_PERMUTATION_V1_WIDTH", "DefaultBusinessIdProfileFactory", "DefaultBusinessIdService", "InMemoryBusinessIdAllocator", "PermutedDailyBusinessIdProfile", "StaticBusinessIdKeyProvider", "encode_business_id_permutation_v1", "ContextTools", "ExecutableHttpTool", "HTTP_TOOL", "HttpIntentPhase", "HttpTool", "HttpToolProvider", "ToolDeniedError", "ToolError", "ToolPolicy", "ToolRisk", "Tools", "ToolToken", "ToolUnavailableError"] __all__ += [ + "AuthorizedBlobSlot", "AuthorizedResponseGraph", "BlobResponseContributor", + "PreparedSecureDocument", "PreparedView", "ProjectedBlobURL", + "ResponseGraphAuthorizer", "SecureDocumentContributor", "ViewOptions", + "ViewPreparationError", "ViewPreparationService", "install_runtime_capability", "validate_capabilities", "Checker", "CheckerRegistry", "DiagnosticTelemetryAdapter", "RequestPolicy", "RequestPolicyInput", "RuntimeTelemetryAdapter", "RuntimeTelemetryInput", diff --git a/src/teaql/runtime/view_preparation.py b/src/teaql/runtime/view_preparation.py new file mode 100644 index 00000000..c3588021 --- /dev/null +++ b/src/teaql/runtime/view_preparation.py @@ -0,0 +1,234 @@ +"""Detached, context-authorized view preparation after business completion.""" + +from __future__ import annotations + +from dataclasses import dataclass +from datetime import datetime, timezone +from typing import Any, Protocol +from urllib.parse import urlsplit + +from teaql.core.blob_ref import BlobRef + +from .context import UserContext + + +@dataclass(frozen=True) +class ViewOptions: + mode: str # ordinary or governed; selected by the trusted host + business_outcome: str # read_only or committed + expires_at: datetime | None = None + + +@dataclass(frozen=True) +class AuthorizedBlobSlot: + path: tuple[str | int, ...] + entity: str + entity_id: str + field: str + scope: str + binding_version: int + reference: BlobRef + + +@dataclass(frozen=True) +class AuthorizedResponseGraph: + presentation: Any + blob_slots: tuple[AuthorizedBlobSlot, ...] + + +class ResponseGraphAuthorizer(Protocol): + async def authorize(self, context: UserContext, result: Any, + options: ViewOptions) -> AuthorizedResponseGraph: ... + + +@dataclass(frozen=True) +class PreparedSecureDocument: + graph: AuthorizedResponseGraph + envelope: Any + expires_at: datetime + + +class SecureDocumentContributor(Protocol): + async def prepare(self, context: UserContext, graph: AuthorizedResponseGraph, + options: ViewOptions) -> PreparedSecureDocument: ... + + +@dataclass(frozen=True) +class ProjectedBlobURL: + url: str + expires_at: datetime + + +class BlobResponseContributor(Protocol): + async def project_batch(self, context: UserContext, graph: AuthorizedResponseGraph, + expires_at: datetime | None) -> tuple[ProjectedBlobURL, ...]: ... + + +@dataclass(frozen=True) +class PreparedView: + presentation: Any + document: Any + expires_at: datetime | None + cache_control: str = "private, no-store" + + +_SAFE_CODES = frozenset({ + "VIEW_OPTIONS_INVALID", "VIEW_AUTHORIZER_MISSING", "VIEW_GRAPH_INVALID", + "DOCUMENT_CONFIGURATION", "BLOB_CAPABILITY_MISSING", "BLOB_SIGN_FAILED", + "BLOB_READ_DENIED", "BLOB_REFERENCE_EXPIRED", "BLOB_SCHEMA_MISSING", + "BLOB_CATALOG_FAILED", +}) + + +class ViewPreparationError(Exception): + def __init__(self, code: str, business_state: str = "read_only") -> None: + self.code = code + self.business_state = business_state + super().__init__(code) + + @property + def retry(self) -> str: + return "read" if self.business_state == "committed" else "prepare" + + +def _fail(code: str) -> None: + raise ViewPreparationError(code) + + +def _detached(value: Any, depth: int = 0, budget: list[int] | None = None) -> Any: + if budget is None: + budget = [100_000] + budget[0] -= 1 + if budget[0] < 0 or depth > 64: + _fail("VIEW_GRAPH_INVALID") + if value is None or type(value) in (bool, int, float, str, BlobRef, datetime): + return value + if type(value) is dict: + if len(value) > budget[0] or any(type(key) is not str for key in value): + _fail("VIEW_GRAPH_INVALID") + return {key: _detached(child, depth + 1, budget) for key, child in value.items()} + if type(value) is list: + if len(value) > budget[0]: + _fail("VIEW_GRAPH_INVALID") + return [_detached(child, depth + 1, budget) for child in value] + _fail("VIEW_GRAPH_INVALID") + + +def _path_key(path: tuple[str | int, ...]) -> tuple[tuple[str, str | int], ...]: + if not path or len(path) > 64: + _fail("VIEW_GRAPH_INVALID") + key = [] + for part in path: + if type(part) is str and part: + key.append(("s", part)) + elif type(part) is int and part >= 0: + key.append(("i", part)) + else: + _fail("VIEW_GRAPH_INVALID") + return tuple(key) + + +def _at_path(root: Any, path: tuple[str | int, ...]) -> Any: + value = root + for part in path: + if type(part) is str and type(value) is dict and part in value: + value = value[part] + elif type(part) is int and type(value) is list and 0 <= part < len(value): + value = value[part] + else: + _fail("VIEW_GRAPH_INVALID") + return value + + +def _replace(root: Any, path: tuple[str | int, ...], value: str) -> None: + _at_path(root, path) + parent = _at_path(root, path[:-1]) + parent[path[-1]] = value + + +def _contains_blob(value: Any, depth: int = 0, budget: list[int] | None = None) -> bool: + if budget is None: + budget = [100_000] + budget[0] -= 1 + if budget[0] < 0 or depth > 64: + return True + if type(value) is BlobRef: + return True + if type(value) is dict: + return any(_contains_blob(child, depth + 1, budget) for child in value.values()) + if type(value) is list: + return any(_contains_blob(child, depth + 1, budget) for child in value) + return False + + +def _earlier(left: datetime | None, right: datetime) -> datetime: + if left is None: + return right + return min(left, right) + + +def _future(value: datetime) -> bool: + return value.tzinfo is not None and value > datetime.now(timezone.utc) + + +class ViewPreparationService: + async def prepare(self, context: UserContext, result: Any, + options: ViewOptions) -> PreparedView: + state = "committed" if getattr(options, "business_outcome", None) == "committed" else "read_only" + try: + if (context is None or not isinstance(options, ViewOptions) + or options.mode not in ("ordinary", "governed") + or options.business_outcome not in ("read_only", "committed") + or (options.expires_at is not None and not _future(options.expires_at))): + _fail("VIEW_OPTIONS_INVALID") + authorizer = context.get_resource("response_graph_authorizer") + if authorizer is None: + _fail("VIEW_AUTHORIZER_MISSING") + graph = await authorizer.authorize(context, result, options) + if type(graph.blob_slots) not in (tuple, list) or len(graph.blob_slots) > 10_000: + _fail("VIEW_GRAPH_INVALID") + graph = AuthorizedResponseGraph(_detached(graph.presentation), + tuple(graph.blob_slots)) + expiry = options.expires_at + document = None + if options.mode == "governed": + secure = context.get_resource("secure_document_contributor") + if secure is None: + _fail("DOCUMENT_CONFIGURATION") + prepared = await secure.prepare(context, graph, options) + if not _future(prepared.expires_at): + _fail("DOCUMENT_CONFIGURATION") + graph, document = prepared.graph, prepared.envelope + if type(graph.blob_slots) not in (tuple, list) or len(graph.blob_slots) > 10_000: + _fail("VIEW_GRAPH_INVALID") + expiry = _earlier(expiry, prepared.expires_at) + presentation = _detached(graph.presentation) + paths: set[tuple[tuple[str, str | int], ...]] = set() + for slot in graph.blob_slots: + path = _path_key(slot.path) + if path in paths or _at_path(presentation, slot.path) != slot.reference or \ + type(_at_path(presentation, slot.path)) is not BlobRef: + _fail("VIEW_GRAPH_INVALID") + paths.add(path) + if graph.blob_slots: + contributor = context.get_resource("blob_response_contributor") + if contributor is None: + _fail("BLOB_CAPABILITY_MISSING") + urls = await contributor.project_batch(context, graph, expiry) + if len(urls) != len(graph.blob_slots): + _fail("BLOB_SIGN_FAILED") + for slot, signed in zip(graph.blob_slots, urls): + parsed = urlsplit(signed.url) + if (parsed.scheme not in ("http", "https") or not parsed.hostname + or not _future(signed.expires_at) + or (expiry is not None and signed.expires_at > expiry)): + _fail("BLOB_SIGN_FAILED") + _replace(presentation, slot.path, signed.url) + expiry = _earlier(expiry, signed.expires_at) + if _contains_blob(presentation): + _fail("BLOB_CAPABILITY_MISSING") + return PreparedView(presentation, document, expiry) + except Exception as exc: + code = exc.code if isinstance(exc, ViewPreparationError) and exc.code in _SAFE_CODES \ + else "VIEW_PREPARATION_FAILED" + raise ViewPreparationError(code, state) from None diff --git a/tests/test_view_preparation_blob.py b/tests/test_view_preparation_blob.py new file mode 100644 index 00000000..8d4ddba3 --- /dev/null +++ b/tests/test_view_preparation_blob.py @@ -0,0 +1,106 @@ +from datetime import datetime, timedelta, timezone + +import pytest + +from teaql.core.blob_ref import BlobRef +from teaql.runtime.context import UserContext +from teaql.runtime.view_preparation import ( + AuthorizedBlobSlot, AuthorizedResponseGraph, PreparedSecureDocument, + ProjectedBlobURL, ViewOptions, ViewPreparationError, ViewPreparationService, +) + + +@pytest.mark.asyncio +async def test_authorized_view_is_detached_and_governed_document_precedes_blob_url(): + reference = BlobRef("blob_asset_1") + source = {"document": reference, "nested": [{"title": "untouched"}]} + stages = [] + ceiling = datetime.now(timezone.utc) + timedelta(minutes=5) + + class Authorizer: + async def authorize(self, _context, _result, _options): + stages.append("authorize") + return AuthorizedResponseGraph(source, (AuthorizedBlobSlot( + path=("document",), entity="Asset", entity_id="1", field="document", + scope="tenant-a", binding_version=4, reference=reference),)) + + class Secure: + async def prepare(self, _context, graph, _options): + stages.append("secure") + graph.presentation["governed"] = True + return PreparedSecureDocument(graph, "sealed-document", ceiling) + + class Blob: + async def project_batch(self, _context, graph, expiry): + stages.append("blob") + assert expiry == ceiling + assert graph.blob_slots[0].reference == reference + return (ProjectedBlobURL("https://files.example.invalid/read/asset", + ceiling - timedelta(minutes=1)),) + + context = UserContext() + context.insert_resource("response_graph_authorizer", Authorizer()) + context.insert_resource("secure_document_contributor", Secure()) + context.insert_resource("blob_response_contributor", Blob()) + view = await ViewPreparationService().prepare(context, source, + ViewOptions(mode="governed", business_outcome="committed")) + assert stages == ["authorize", "secure", "blob"] + assert source == {"document": reference, "nested": [{"title": "untouched"}]} + assert view.presentation == { + "document": "https://files.example.invalid/read/asset", + "nested": [{"title": "untouched"}], "governed": True, + } + assert view.document == "sealed-document" + assert view.expires_at == ceiling - timedelta(minutes=1) + assert view.cache_control == "private, no-store" + + +@pytest.mark.asyncio +async def test_unlisted_key_and_provider_failure_fail_closed_after_commit(): + reference = BlobRef("blob_private_canary") + + class Authorizer: + def __init__(self): + self.listed = False + + async def authorize(self, _context, _result, _options): + slots = (AuthorizedBlobSlot(("document",), "Asset", "1", "document", + "tenant-a", 4, reference),) if self.listed else () + return AuthorizedResponseGraph({"document": reference}, slots) + + class FailingBlob: + async def project_batch(self, _context, _graph, _expiry): + raise RuntimeError("private URL https://secret.invalid/read?sig=canary") + + authorizer = Authorizer() + context = UserContext().insert_resource("response_graph_authorizer", authorizer) + service = ViewPreparationService() + options = ViewOptions(mode="ordinary", business_outcome="committed") + with pytest.raises(ViewPreparationError) as first: + await service.prepare(context, None, options) + assert first.value.code == "BLOB_CAPABILITY_MISSING" + assert first.value.business_state == "committed" + assert first.value.retry == "read" + assert reference.key not in str(first.value) + + authorizer.listed = True + context.insert_resource("blob_response_contributor", FailingBlob()) + with pytest.raises(ViewPreparationError) as second: + await service.prepare(context, None, options) + assert second.value.code == "VIEW_PREPARATION_FAILED" + assert "secret.invalid" not in str(second.value) + + +@pytest.mark.asyncio +async def test_invalid_or_expired_view_options_are_rejected_before_authorization(): + context = UserContext() + service = ViewPreparationService() + for options in ( + None, + ViewOptions(mode="downgrade", business_outcome="read_only"), + ViewOptions(mode="ordinary", business_outcome="committed", + expires_at=datetime.now(timezone.utc) - timedelta(seconds=1)), + ): + with pytest.raises(ViewPreparationError) as failure: + await service.prepare(context, {}, options) + assert failure.value.code == "VIEW_OPTIONS_INVALID" From fd266a8b5921bc402077e7131376f09e7c501cf0 Mon Sep 17 00:00:00 2001 From: Philip Z Date: Tue, 27 Oct 2026 10:58:09 +0800 Subject: [PATCH 03/75] Validate independent Blob URL lifetimes in prepared views --- src/teaql/runtime/view_preparation.py | 5 ++-- tests/test_view_preparation_blob.py | 33 +++++++++++++++++++++++++++ 2 files changed, 36 insertions(+), 2 deletions(-) diff --git a/src/teaql/runtime/view_preparation.py b/src/teaql/runtime/view_preparation.py index c3588021..7b40c417 100644 --- a/src/teaql/runtime/view_preparation.py +++ b/src/teaql/runtime/view_preparation.py @@ -214,14 +214,15 @@ async def prepare(self, context: UserContext, result: Any, contributor = context.get_resource("blob_response_contributor") if contributor is None: _fail("BLOB_CAPABILITY_MISSING") - urls = await contributor.project_batch(context, graph, expiry) + ceiling = expiry + urls = await contributor.project_batch(context, graph, ceiling) if len(urls) != len(graph.blob_slots): _fail("BLOB_SIGN_FAILED") for slot, signed in zip(graph.blob_slots, urls): parsed = urlsplit(signed.url) if (parsed.scheme not in ("http", "https") or not parsed.hostname or not _future(signed.expires_at) - or (expiry is not None and signed.expires_at > expiry)): + or (ceiling is not None and signed.expires_at > ceiling)): _fail("BLOB_SIGN_FAILED") _replace(presentation, slot.path, signed.url) expiry = _earlier(expiry, signed.expires_at) diff --git a/tests/test_view_preparation_blob.py b/tests/test_view_preparation_blob.py index 8d4ddba3..da89eeb9 100644 --- a/tests/test_view_preparation_blob.py +++ b/tests/test_view_preparation_blob.py @@ -55,6 +55,39 @@ async def project_batch(self, _context, graph, expiry): assert view.cache_control == "private, no-store" +@pytest.mark.asyncio +async def test_independent_url_lifetimes_use_original_ceiling_and_minimum(): + first = BlobRef("blob_asset_1") + second = BlobRef("blob_asset_2") + ceiling = datetime.now(timezone.utc) + timedelta(seconds=90) + first_expiry = datetime.now(timezone.utc) + timedelta(seconds=20) + second_expiry = datetime.now(timezone.utc) + timedelta(seconds=40) + + class Authorizer: + async def authorize(self, _context, _result, _options): + return AuthorizedResponseGraph({"first": first, "second": second}, ( + AuthorizedBlobSlot(("first",), "Asset", "1", "first", "tenant-a", 4, first), + AuthorizedBlobSlot(("second",), "Asset", "1", "second", "tenant-a", 5, second), + )) + + class Blob: + async def project_batch(self, _context, _graph, expiry): + assert expiry == ceiling + return (ProjectedBlobURL("https://files.example.invalid/first", first_expiry), + ProjectedBlobURL("https://files.example.invalid/second", second_expiry)) + + context = UserContext() + context.insert_resource("response_graph_authorizer", Authorizer()) + context.insert_resource("blob_response_contributor", Blob()) + view = await ViewPreparationService().prepare(context, None, + ViewOptions(mode="ordinary", business_outcome="read_only", expires_at=ceiling)) + assert view.expires_at == first_expiry + assert view.presentation == { + "first": "https://files.example.invalid/first", + "second": "https://files.example.invalid/second", + } + + @pytest.mark.asyncio async def test_unlisted_key_and_provider_failure_fail_closed_after_commit(): reference = BlobRef("blob_private_canary") From 9ecf29a210100599ab576c22eafa7e5eef67c2c1 Mon Sep 17 00:00:00 2001 From: Philip Z Date: Tue, 27 Oct 2026 11:21:55 +0800 Subject: [PATCH 04/75] Add graph-scoped physical SQL lease for optional modules --- src/teaql/runtime/graph_session.py | 8 ++ src/teaql/sql/executor.py | 53 ++++++++ .../sqlite/test_graph_physical_sql.py | 118 ++++++++++++++++++ 3 files changed, 179 insertions(+) create mode 100644 tests/provider/sqlite/test_graph_physical_sql.py diff --git a/src/teaql/runtime/graph_session.py b/src/teaql/runtime/graph_session.py index af050bd3..127e534e 100644 --- a/src/teaql/runtime/graph_session.py +++ b/src/teaql/runtime/graph_session.py @@ -49,6 +49,14 @@ def _require_active(self): if not self._active or self._committed: raise RuntimeError('graph mutation session is no longer writable') + async def with_sql_transport(self, work): + """Borrow this graph's physical SQL transaction for an optional module.""" + self._require_active() + owner = getattr(self.transaction, 'with_physical_sql', None) + if not callable(owner): + raise RuntimeError('BLOB_TRANSACTION_REQUIRED') + return await owner(work) + def scope(self, key, parent=None, local_reason=None): self._require_active() if parent is None: diff --git a/src/teaql/sql/executor.py b/src/teaql/sql/executor.py index 28f571cd..4393ecf0 100644 --- a/src/teaql/sql/executor.py +++ b/src/teaql/sql/executor.py @@ -1390,6 +1390,27 @@ def __init__(self, dialect: SqlDialect, transport: SqlTransactionTransportTx, sc self._audit_journal = None self._audit_owner = None self._completed = False + self._rollback_only = False + self._physical_sql_active = False + + async def with_physical_sql(self, work): + """Lend this physical SQL owner to an explicit graph module callback.""" + if self._completed or self._rollback_only or self._physical_sql_active or not callable(work): + raise RuntimeError('BLOB_TRANSACTION_REQUIRED') + lease = _PhysicalSQLLease(self.transport) + self._physical_sql_active = True + try: + result = await work(lease) + if lease.failed: + self._rollback_only = True + raise RuntimeError('GRAPH_TRANSACTION_ROLLBACK_ONLY') + return result + except BaseException: + self._rollback_only = True + raise + finally: + lease.close() + self._physical_sql_active = False def capabilities(self) -> DataServiceCapabilities: return DataServiceCapabilities( @@ -1410,6 +1431,8 @@ async def mutate(self, context: 'UserContext', request: MutationRequest) -> Muta request.validate() if self._completed: raise RuntimeError('SQL transaction is already completed') + if self._rollback_only: + raise RuntimeError('GRAPH_TRANSACTION_ROLLBACK_ONLY') if context is not None: context._require_mutation_invocation() if context._graph_session is None: @@ -1436,6 +1459,8 @@ async def ensure_id_floor(self, entity: str, floor: int) -> None: async def commit(self, context: 'UserContext') -> None: if self._completed: raise RuntimeError('SQL transaction is already completed') + if self._rollback_only: + raise RuntimeError('GRAPH_TRANSACTION_ROLLBACK_ONLY') await self.transport.commit_sql() self._completed = True if self._audit_journal: @@ -1446,3 +1471,31 @@ async def rollback(self, context: 'UserContext') -> None: self._audit_journal.discard() self._completed = True await self.transport.rollback_sql() + + +class _PhysicalSQLLease(SqlTransport): + def __init__(self, transport: SqlTransactionTransportTx): + self._transport = transport + self._active = True + self.failed = False + + def close(self): + self._active = False + + async def fetch_all_sql(self, query: CompiledQuery): + if not self._active: + raise RuntimeError('BLOB_TRANSACTION_REQUIRED') + try: + return await self._transport.fetch_all_sql(query) + except BaseException: + self.failed = True + raise + + async def execute_sql(self, query: CompiledQuery): + if not self._active: + raise RuntimeError('BLOB_TRANSACTION_REQUIRED') + try: + return await self._transport.execute_sql(query) + except BaseException: + self.failed = True + raise diff --git a/tests/provider/sqlite/test_graph_physical_sql.py b/tests/provider/sqlite/test_graph_physical_sql.py new file mode 100644 index 00000000..4b517d2d --- /dev/null +++ b/tests/provider/sqlite/test_graph_physical_sql.py @@ -0,0 +1,118 @@ +"""The optional-module SQL lease shares the native graph's physical owner.""" + +import aiosqlite +import pytest + +from teaql.core.meta import EntityDescriptor, PropertyDescriptor +from teaql.core.entity import EntityKey +from teaql.core.mutation import InsertCommand +from teaql.core.value import DataType, Value +from teaql.provider.sqlite import SimpleSchemaProvider +from teaql.provider.sqlite.dialect import SqliteDialect +from teaql.provider.sqlite.transport import SqliteTransport +from teaql.runtime import RuntimeModule +from teaql.sql.executor import SqlDataServiceExecutor +from teaql.sql.types import CompiledQuery + + +async def fixture(tmp_path): + path = str(tmp_path / "graph-owner.db") + descriptor = EntityDescriptor("Asset").table_name("asset") + descriptor.property(PropertyDescriptor("id", DataType.I64).is_id()) + descriptor.property(PropertyDescriptor("version", DataType.I64).is_version()) + descriptor.property(PropertyDescriptor("name", DataType.Text)) + schema = SimpleSchemaProvider() + schema.register_entity(descriptor) + context = RuntimeModule.new().entity(descriptor).into_context() + context.insert_resource("dataService", SqlDataServiceExecutor(SqliteDialect(), SqliteTransport(path), schema)) + async with aiosqlite.connect(path) as db: + await db.execute("CREATE TABLE asset (id INTEGER PRIMARY KEY, version INTEGER, name TEXT)") + await db.execute("CREATE TABLE module_binding (id INTEGER PRIMARY KEY, asset_id INTEGER NOT NULL)") + await db.commit() + return context, path + + +async def rows(path, table): + async with aiosqlite.connect(path) as db: + async with db.execute(f"SELECT * FROM {table}") as cursor: + return await cursor.fetchall() + + +@pytest.mark.asyncio +async def test_native_graph_and_module_rows_commit_or_rollback_together(tmp_path): + context, path = await fixture(tmp_path) + retained = [] + + async def save(graph, asset_id, *, fail=False): + command = (InsertCommand.new("Asset").value("id", asset_id) + .value("version", 1).value("name", "document")) + await graph.transaction.mutate(graph.context, graph.request(command, graph.scope(EntityKey("Asset", asset_id)))) + + async def bind(sql): + retained.append(sql) + assert not hasattr(sql, "commit_sql") + await sql.execute_sql(CompiledQuery( + "INSERT INTO module_binding(id, asset_id) VALUES (?, ?)", + [Value.I64(asset_id), Value.I64(asset_id)])) + assert len(await sql.fetch_all_sql(CompiledQuery( + "SELECT id FROM asset WHERE id = ?", [Value.I64(asset_id)]))) == 1 + + await graph.with_sql_transport(bind) + if fail: + raise RuntimeError("abort graph") + + await context.execute_graph_save(lambda graph: save(graph, 1), comment="save asset") + assert len(await rows(path, "asset")) == len(await rows(path, "module_binding")) == 1 + with pytest.raises(RuntimeError, match="abort graph"): + await context.execute_graph_save(lambda graph: save(graph, 2, fail=True), comment="save asset") + assert len(await rows(path, "asset")) == len(await rows(path, "module_binding")) == 1 + with pytest.raises(RuntimeError, match="BLOB_TRANSACTION_REQUIRED"): + await retained[0].execute_sql(CompiledQuery("DELETE FROM module_binding", [])) + + +@pytest.mark.asyncio +async def test_caught_callback_or_sql_error_still_blocks_graph_commit(tmp_path): + context, path = await fixture(tmp_path) + + async def save(graph, fail_in_sql): + command = (InsertCommand.new("Asset").value("id", 1) + .value("version", 1).value("name", "document")) + await graph.transaction.mutate(graph.context, graph.request(command, graph.scope(EntityKey("Asset", 1)))) + + async def bind(sql): + await sql.execute_sql(CompiledQuery( + "INSERT INTO module_binding(id, asset_id) VALUES (1, 1)", [])) + if fail_in_sql: + try: + await sql.execute_sql(CompiledQuery( + "INSERT INTO module_binding(id, asset_id) VALUES (1, 1)", [])) + except Exception: + pass + else: + raise ValueError("module rejected binding") + + try: + await graph.with_sql_transport(bind) + except Exception: + pass + + for fail_in_sql in (False, True): + with pytest.raises(RuntimeError, match="GRAPH_TRANSACTION_ROLLBACK_ONLY"): + await context.execute_graph_save( + lambda graph: save(graph, fail_in_sql), comment="save asset") + assert await rows(path, "asset") == await rows(path, "module_binding") == [] + + +@pytest.mark.asyncio +async def test_module_sql_requires_live_graph_and_transaction(tmp_path): + context, _ = await fixture(tmp_path) + saved = [] + + async def save(graph): + saved.append(graph) + await graph.with_sql_transport(lambda sql: sql.fetch_all_sql(CompiledQuery("SELECT 1", []))) + + await context.execute_graph_save(save, comment="inspect graph") + with pytest.raises(RuntimeError, match="no longer writable"): + await saved[0].with_sql_transport(lambda sql: sql.fetch_all_sql(CompiledQuery("SELECT 1", []))) + From 284a969850c3dddc0a28860d3ca3962df59253dd Mon Sep 17 00:00:00 2001 From: Philip Z Date: Tue, 27 Oct 2026 11:47:50 +0800 Subject: [PATCH 05/75] Add SQLite Blob catalog binding under native graph owner --- src/teaql_blob_reference/__init__.py | 5 + src/teaql_blob_reference/sqlite_catalog.py | 188 ++++++++++++++++++ .../sqlite/test_blob_catalog_graph.py | 137 +++++++++++++ 3 files changed, 330 insertions(+) create mode 100644 src/teaql_blob_reference/__init__.py create mode 100644 src/teaql_blob_reference/sqlite_catalog.py create mode 100644 tests/provider/sqlite/test_blob_catalog_graph.py diff --git a/src/teaql_blob_reference/__init__.py b/src/teaql_blob_reference/__init__.py new file mode 100644 index 00000000..4c71bd35 --- /dev/null +++ b/src/teaql_blob_reference/__init__.py @@ -0,0 +1,5 @@ +"""Optional Blob reference services; the base teaql runtime does not import them.""" + +from .sqlite_catalog import BlobBindingChange, BlobBindingTarget, SqliteBlobCatalog + +__all__ = ["BlobBindingChange", "BlobBindingTarget", "SqliteBlobCatalog"] diff --git a/src/teaql_blob_reference/sqlite_catalog.py b/src/teaql_blob_reference/sqlite_catalog.py new file mode 100644 index 00000000..0d78755c --- /dev/null +++ b/src/teaql_blob_reference/sqlite_catalog.py @@ -0,0 +1,188 @@ +"""Authoritative SQLite binding component under an explicit native graph owner.""" + +from dataclasses import dataclass +from typing import Protocol +import unicodedata + +from teaql.core.blob_ref import BlobRef +from teaql.core.value import Value +from teaql.sql.types import CompiledQuery + + +@dataclass(frozen=True) +class BlobBindingTarget: + entity: str + entity_id: str + field: str + + +@dataclass(frozen=True) +class BlobBindingChange: + target: BlobBindingTarget + next: BlobRef | None + expected_version: int + next_version: int + action: str + + +class BlobBindingAuthorization(Protocol): + def scope(self, context) -> str: ... + + async def authorize(self, context, target: BlobBindingTarget, + previous: BlobRef | None, next_ref: BlobRef | None, + action: str) -> bool: ... + + +def _name(value): + return (isinstance(value, str) and value and value == value.strip() + and len(value) <= 128 and not any(unicodedata.category(ch) == "Cc" for ch in value)) + + +def _scope(value): + if not isinstance(value, str) or not value or value != value.strip(): + return False + try: + return (len(value.encode("utf-8")) <= 256 + and not any(unicodedata.category(ch) == "Cc" for ch in value)) + except UnicodeEncodeError: + return False + + +def _entity_id(value): + return (isinstance(value, str) and 1 <= len(value) <= 20 and value.isascii() + and value.isdecimal() and 0 < int(value) <= 2**64 - 1 + and str(int(value)) == value) + + +class SqliteBlobCatalog: + """SQLite-only catalog; install and schema migration are separate host actions.""" + + def __init__(self, transport): + if (transport is None or not callable(getattr(transport, "execute_sql", None)) + or not callable(getattr(transport, "fetch_all_sql", None))): + raise RuntimeError("BLOB_CAPABILITY_MISSING") + self.transport = transport + + async def ensure_schema(self): + statements = ( + """CREATE TABLE IF NOT EXISTS teaql_blob_catalog ( + key TEXT PRIMARY KEY, scope TEXT NOT NULL, object_key TEXT NOT NULL, + content_version TEXT NOT NULL, file_name TEXT NOT NULL, + content_type TEXT NOT NULL, size_bytes BIGINT NOT NULL, + status TEXT NOT NULL CHECK (status IN + ('initiated','uploaded_unbound','available','deleting','deleted'))) + """, + """CREATE TABLE IF NOT EXISTS teaql_blob_binding ( + scope TEXT NOT NULL, entity TEXT NOT NULL, entity_id TEXT NOT NULL, + field TEXT NOT NULL, key TEXT, version BIGINT NOT NULL, + PRIMARY KEY (scope,entity,entity_id,field), + FOREIGN KEY (key) REFERENCES teaql_blob_catalog(key)) + """, + "CREATE INDEX IF NOT EXISTS teaql_blob_binding_key ON teaql_blob_binding(key)", + ) + for sql in statements: + try: + await self.transport.execute_sql(CompiledQuery(sql, [])) + except Exception: + raise RuntimeError("BLOB_CATALOG_FAILED") from None + + async def validate_schema(self): + for sql in ( + "SELECT key, scope, object_key, content_version, file_name, content_type, size_bytes, status " + "FROM teaql_blob_catalog WHERE 1 = 0", + "SELECT scope, entity, entity_id, field, key, version " + "FROM teaql_blob_binding WHERE 1 = 0", + ): + try: + await self.transport.fetch_all_sql(CompiledQuery(sql, [])) + except Exception: + raise RuntimeError("BLOB_SCHEMA_MISSING") from None + try: + columns = await self.transport.fetch_all_sql(CompiledQuery("PRAGMA table_info(teaql_blob_binding)", [])) + names = {row["name"]: row for row in columns} + if names["key"]["notnull"] != 0 or names["version"]["notnull"] != 1: + raise RuntimeError("BLOB_SCHEMA_MISSING") + except Exception: + raise RuntimeError("BLOB_SCHEMA_MISSING") from None + + async def bind_in_graph(self, graph, actor, change: BlobBindingChange, + authorization: BlobBindingAuthorization) -> int: + if graph is None or not callable(getattr(graph, "with_sql_transport", None)): + raise RuntimeError("BLOB_TRANSACTION_REQUIRED") + + async def bind(sql): + if actor is not graph.context: + raise RuntimeError("BLOB_BIND_DENIED") + if (not isinstance(change, BlobBindingChange) or not isinstance(change.target, BlobBindingTarget) + or not _name(change.target.entity) or not _entity_id(change.target.entity_id) + or not _name(change.target.field) or not isinstance(change.expected_version, int) + or isinstance(change.expected_version, bool) or change.expected_version < 0 + or not isinstance(change.next_version, int) or isinstance(change.next_version, bool) + or change.next_version <= change.expected_version + or change.action not in ("Create", "Update", "Delete") + or change.action == "Create" and change.expected_version != 0 + or change.action == "Delete" and change.next is not None + or change.next is not None and not isinstance(change.next, BlobRef) + or authorization is None): + raise RuntimeError("BLOB_REFERENCE_INVALID") + try: + scope = authorization.scope(actor) + except Exception: + raise RuntimeError("BLOB_BIND_DENIED") from None + if not _scope(scope): + raise RuntimeError("BLOB_BIND_DENIED") + target = change.target + identity = [Value.Text(scope), Value.Text(target.entity), + Value.Text(target.entity_id), Value.Text(target.field)] + try: + rows = await sql.fetch_all_sql(CompiledQuery( + "SELECT key, version FROM teaql_blob_binding WHERE scope = ? AND entity = ? " + "AND entity_id = ? AND field = ?", identity)) + except Exception: + raise RuntimeError("BLOB_CATALOG_FAILED") from None + if len(rows) > 1: + raise RuntimeError("BLOB_CATALOG_FAILED") + row = rows[0] if rows else None + version = row["version"] if row else 0 + if not isinstance(version, int) or version < 0: + raise RuntimeError("BLOB_CATALOG_FAILED") + if version != change.expected_version: + raise RuntimeError("BLOB_BIND_DENIED") + try: + previous = BlobRef.from_sql(row["key"]) if row and row["key"] is not None else None + except ValueError: + raise RuntimeError("BLOB_CATALOG_FAILED") from None + try: + allowed = await authorization.authorize(actor, target, previous, change.next, change.action) + except Exception: + raise RuntimeError("BLOB_BIND_DENIED") from None + if not allowed: + raise RuntimeError("BLOB_BIND_DENIED") + if change.next: + try: + available = await sql.fetch_all_sql(CompiledQuery( + "SELECT key FROM teaql_blob_catalog WHERE key = ? AND scope = ? AND status = 'available'", + [Value.Text(change.next.key), Value.Text(scope)])) + except Exception: + raise RuntimeError("BLOB_CATALOG_FAILED") from None + if len(available) != 1: + raise RuntimeError("BLOB_BIND_DENIED") + key = Value.Text(change.next.key) if change.next else Value.Null() + try: + if row: + affected, _ = await sql.execute_sql(CompiledQuery( + "UPDATE teaql_blob_binding SET key = ?, version = ? WHERE scope = ? " + "AND entity = ? AND entity_id = ? AND field = ? AND version = ?", + [key, Value.I64(change.next_version), *identity, Value.I64(version)])) + else: + affected, _ = await sql.execute_sql(CompiledQuery( + "INSERT INTO teaql_blob_binding (scope, entity, entity_id, field, key, version) " + "VALUES (?, ?, ?, ?, ?, ?)", + [*identity, key, Value.I64(change.next_version)])) + except Exception: + raise RuntimeError("BLOB_CATALOG_FAILED") from None + if affected != 1: + raise RuntimeError("BLOB_BIND_DENIED") + return change.next_version + + return await graph.with_sql_transport(bind) diff --git a/tests/provider/sqlite/test_blob_catalog_graph.py b/tests/provider/sqlite/test_blob_catalog_graph.py new file mode 100644 index 00000000..81d5c69a --- /dev/null +++ b/tests/provider/sqlite/test_blob_catalog_graph.py @@ -0,0 +1,137 @@ +"""Real SQLite catalog binding belongs to the native business graph owner.""" + +import aiosqlite +import pytest + +from teaql.core.blob_ref import BlobRef +from teaql.core.entity import EntityKey +from teaql.core.meta import EntityDescriptor, PropertyDescriptor +from teaql.core.mutation import InsertCommand +from teaql.core.value import DataType +from teaql.provider.sqlite import SimpleSchemaProvider +from teaql.provider.sqlite.dialect import SqliteDialect +from teaql.provider.sqlite.transport import SqliteTransport +from teaql.runtime import RuntimeModule, UserContext +from teaql.sql.executor import SqlDataServiceExecutor +from teaql_blob_reference import BlobBindingChange, BlobBindingTarget, SqliteBlobCatalog + + +class Authorization: + def __init__(self): + self.current_scope = "tenant-a" + self.allowed = True + + def scope(self, _context): + return self.current_scope + + async def authorize(self, _context, _target, _previous, _next, _action): + return self.allowed + + +async def fixture(tmp_path): + path = str(tmp_path / "blob-catalog.db") + descriptor = EntityDescriptor("Asset").table_name("asset") + descriptor.property(PropertyDescriptor("id", DataType.I64).is_id()) + descriptor.property(PropertyDescriptor("version", DataType.I64).is_version()) + descriptor.property(PropertyDescriptor("name", DataType.Text)) + schema = SimpleSchemaProvider() + schema.register_entity(descriptor) + context = RuntimeModule.new().entity(descriptor).into_context() + context.insert_resource("dataService", SqlDataServiceExecutor( + SqliteDialect(), SqliteTransport(path), schema)) + async with aiosqlite.connect(path) as db: + await db.execute("CREATE TABLE asset (id INTEGER PRIMARY KEY, version INTEGER, name TEXT)") + await db.commit() + return context, path, SqliteBlobCatalog(SqliteTransport(path)) + + +async def all_rows(path, table): + async with aiosqlite.connect(path) as db: + async with db.execute(f"SELECT * FROM {table}") as cursor: + return await cursor.fetchall() + + +@pytest.mark.asyncio +async def test_catalog_binding_same_graph_rollback_authority_tombstone_and_rebind(tmp_path): + context, path, catalog = await fixture(tmp_path) + with pytest.raises(RuntimeError, match="BLOB_SCHEMA_MISSING"): + await catalog.validate_schema() + await catalog.ensure_schema() + await catalog.validate_schema() + reference = BlobRef("blob_python_catalog_1") + async with aiosqlite.connect(path) as db: + await db.execute("INSERT INTO teaql_blob_catalog " + "(key,scope,object_key,content_version,file_name,content_type,size_bytes,status) " + "VALUES (?,?,?,?,?,?,?,?)", + (reference.key, "tenant-a", "objects/1", "v1", "asset.pdf", + "application/pdf", 7, "available")) + await db.commit() + auth = Authorization() + + async def save(graph, asset_id, *, abort=False, catch_denial=False, foreign_actor=False): + command = (InsertCommand.new("Asset").value("id", asset_id) + .value("version", 1).value("name", "document")) + await graph.transaction.mutate(graph.context, + graph.request(command, graph.scope(EntityKey("Asset", asset_id)))) + change = BlobBindingChange(BlobBindingTarget("Asset", str(asset_id), "document"), + reference, 0, 1, "Create") + if catch_denial: + try: + await catalog.bind_in_graph(graph, UserContext.new() if foreign_actor else graph.context, + change, auth) + except RuntimeError: + return + else: + assert await catalog.bind_in_graph(graph, graph.context, change, auth) == 1 + if abort: + raise RuntimeError("abort graph") + + await context.execute_graph_save(lambda graph: save(graph, 1), comment="save asset") + assert len(await all_rows(path, "asset")) == len(await all_rows(path, "teaql_blob_binding")) == 1 + with pytest.raises(RuntimeError, match="abort graph"): + await context.execute_graph_save(lambda graph: save(graph, 2, abort=True), comment="save asset") + assert len(await all_rows(path, "asset")) == len(await all_rows(path, "teaql_blob_binding")) == 1 + + auth.allowed = False + with pytest.raises(RuntimeError, match="GRAPH_TRANSACTION_ROLLBACK_ONLY"): + await context.execute_graph_save( + lambda graph: save(graph, 3, catch_denial=True), comment="save asset") + auth.allowed = True + with pytest.raises(RuntimeError, match="GRAPH_TRANSACTION_ROLLBACK_ONLY"): + await context.execute_graph_save( + lambda graph: save(graph, 4, catch_denial=True, foreign_actor=True), comment="save asset") + assert len(await all_rows(path, "asset")) == len(await all_rows(path, "teaql_blob_binding")) == 1 + + stale = BlobBindingChange(BlobBindingTarget("Asset", "1", "document"), reference, 0, 2, "Update") + with pytest.raises(RuntimeError, match="BLOB_BIND_DENIED"): + await context.execute_graph_save( + lambda graph: catalog.bind_in_graph(graph, graph.context, stale, auth), + comment="reject stale bind") + auth.current_scope = "tenant-b" + with pytest.raises(RuntimeError, match="BLOB_BIND_DENIED"): + await context.execute_graph_save( + lambda graph: catalog.bind_in_graph(graph, graph.context, + BlobBindingChange(BlobBindingTarget("Asset", "2", "document"), reference, 0, 1, "Create"), auth), + comment="reject wrong scope") + auth.current_scope = "tenant-a" + clear = BlobBindingChange(BlobBindingTarget("Asset", "1", "document"), None, 1, 2, "Update") + await context.execute_graph_save( + lambda graph: catalog.bind_in_graph(graph, graph.context, clear, auth), comment="clear binding") + assert (await all_rows(path, "teaql_blob_binding"))[0][-2:] == (None, 2) + rebind = BlobBindingChange(BlobBindingTarget("Asset", "1", "document"), reference, 2, 3, "Update") + await context.execute_graph_save( + lambda graph: catalog.bind_in_graph(graph, graph.context, rebind, auth), comment="rebind") + assert (await all_rows(path, "teaql_blob_binding"))[0][-2:] == (reference.key, 3) + + +@pytest.mark.asyncio +async def test_catalog_rejects_old_not_null_binding_schema(tmp_path): + _, path, catalog = await fixture(tmp_path) + async with aiosqlite.connect(path) as db: + await db.execute("CREATE TABLE teaql_blob_binding (scope TEXT NOT NULL, entity TEXT NOT NULL, " + "entity_id TEXT NOT NULL, field TEXT NOT NULL, key TEXT NOT NULL, " + "version BIGINT NOT NULL, PRIMARY KEY (scope,entity,entity_id,field))") + await db.commit() + await catalog.ensure_schema() + with pytest.raises(RuntimeError, match="BLOB_SCHEMA_MISSING"): + await catalog.validate_schema() From 0218aba28f4823c8bd78a70a2b7440750769e7c5 Mon Sep 17 00:00:00 2001 From: Philip Z Date: Tue, 27 Oct 2026 12:09:01 +0800 Subject: [PATCH 06/75] Recheck current Blob bindings before Python URL projection --- src/teaql_blob_reference/__init__.py | 6 +- src/teaql_blob_reference/read_service.py | 62 +++++++++++++ src/teaql_blob_reference/sqlite_catalog.py | 38 ++++++++ .../sqlite/test_blob_catalog_graph.py | 88 ++++++++++++++++++- 4 files changed, 191 insertions(+), 3 deletions(-) create mode 100644 src/teaql_blob_reference/read_service.py diff --git a/src/teaql_blob_reference/__init__.py b/src/teaql_blob_reference/__init__.py index 4c71bd35..9e215b70 100644 --- a/src/teaql_blob_reference/__init__.py +++ b/src/teaql_blob_reference/__init__.py @@ -1,5 +1,7 @@ """Optional Blob reference services; the base teaql runtime does not import them.""" -from .sqlite_catalog import BlobBindingChange, BlobBindingTarget, SqliteBlobCatalog +from .sqlite_catalog import BlobBindingChange, BlobBindingTarget, BlobCatalogRead, SqliteBlobCatalog +from .read_service import SqliteBlobResponseContributor -__all__ = ["BlobBindingChange", "BlobBindingTarget", "SqliteBlobCatalog"] +__all__ = ["BlobBindingChange", "BlobBindingTarget", "BlobCatalogRead", + "SqliteBlobCatalog", "SqliteBlobResponseContributor"] diff --git a/src/teaql_blob_reference/read_service.py b/src/teaql_blob_reference/read_service.py new file mode 100644 index 00000000..a4cf8ef8 --- /dev/null +++ b/src/teaql_blob_reference/read_service.py @@ -0,0 +1,62 @@ +"""Current-authorized Blob URL projection backed by the optional SQLite catalog.""" + +import asyncio +from datetime import datetime, timedelta, timezone +from typing import Protocol + +from teaql.core.object_storage import ObjectKey, ReadUrlOptions +from teaql.runtime.view_preparation import ( + AuthorizedBlobSlot, AuthorizedResponseGraph, ProjectedBlobURL, ViewPreparationError, +) + +from .sqlite_catalog import SqliteBlobCatalog + + +class BlobReadAuthorization(Protocol): + async def authorize(self, context, slot: AuthorizedBlobSlot) -> bool: ... + + +class SqliteBlobResponseContributor: + """Reauthorizes and rereads before requesting complete temporary storage URLs.""" + + def __init__(self, catalog: SqliteBlobCatalog, + authorization: BlobReadAuthorization, maximum_url_seconds: int = 600): + if catalog is None or authorization is None or type(maximum_url_seconds) is not int \ + or not 1 <= maximum_url_seconds <= 3600: + raise RuntimeError("BLOB_POLICY_INVALID") + self.catalog = catalog + self.authorization = authorization + self.maximum_url_seconds = maximum_url_seconds + + async def project_batch(self, context, graph: AuthorizedResponseGraph, + expires_at: datetime | None) -> tuple[ProjectedBlobURL, ...]: + results = [] + for slot in graph.blob_slots: + try: + allowed = await self.authorization.authorize(context, slot) + except Exception: + raise ViewPreparationError("BLOB_READ_DENIED") from None + if not allowed: + raise ViewPreparationError("BLOB_READ_DENIED") + try: + record = await self.catalog.current_read(slot) + except RuntimeError as error: + code = str(error) if str(error) in ("BLOB_READ_DENIED", "BLOB_CATALOG_FAILED") \ + else "BLOB_CATALOG_FAILED" + raise ViewPreparationError(code) from None + seconds = self.maximum_url_seconds + if expires_at is not None: + seconds = min(seconds, int((expires_at - datetime.now(timezone.utc)).total_seconds()) - 1) + if seconds < 1: + raise ViewPreparationError("BLOB_REFERENCE_EXPIRED") + disposition = "inline" if record.content_type.startswith(("image/", "audio/", "video/")) \ + or record.content_type == "application/pdf" else "attachment" + try: + signed = await asyncio.to_thread( + context.object_storage("prepare authorized Blob view").get_read_url, + ObjectKey(record.object_key), + ReadUrlOptions(timedelta(seconds=seconds), disposition, record.file_name)) + except Exception: + raise ViewPreparationError("BLOB_SIGN_FAILED") from None + results.append(ProjectedBlobURL(signed.url, signed.expires_at)) + return tuple(results) diff --git a/src/teaql_blob_reference/sqlite_catalog.py b/src/teaql_blob_reference/sqlite_catalog.py index 0d78755c..793a6bb0 100644 --- a/src/teaql_blob_reference/sqlite_catalog.py +++ b/src/teaql_blob_reference/sqlite_catalog.py @@ -7,6 +7,7 @@ from teaql.core.blob_ref import BlobRef from teaql.core.value import Value from teaql.sql.types import CompiledQuery +from teaql.runtime.view_preparation import AuthorizedBlobSlot @dataclass(frozen=True) @@ -25,6 +26,15 @@ class BlobBindingChange: action: str +@dataclass(frozen=True) +class BlobCatalogRead: + object_key: str + content_version: str + file_name: str + content_type: str + size_bytes: int + + class BlobBindingAuthorization(Protocol): def scope(self, context) -> str: ... @@ -105,6 +115,34 @@ async def validate_schema(self): except Exception: raise RuntimeError("BLOB_SCHEMA_MISSING") from None + async def current_read(self, slot: AuthorizedBlobSlot) -> BlobCatalogRead: + """Reread the exact current binding; possession of the stable Key grants no read.""" + if (not isinstance(slot, AuthorizedBlobSlot) or not _scope(slot.scope) + or not _name(slot.entity) or not _entity_id(slot.entity_id) + or not _name(slot.field) or type(slot.binding_version) is not int + or slot.binding_version < 1 or not isinstance(slot.reference, BlobRef)): + raise RuntimeError("BLOB_READ_DENIED") + try: + rows = await self.transport.fetch_all_sql(CompiledQuery( + "SELECT c.object_key,c.content_version,c.file_name,c.content_type,c.size_bytes " + "FROM teaql_blob_binding b JOIN teaql_blob_catalog c ON c.key = b.key " + "WHERE b.scope = ? AND b.entity = ? AND b.entity_id = ? AND b.field = ? " + "AND b.key = ? AND b.version = ? AND c.scope = b.scope AND c.status = 'available'", + [Value.Text(slot.scope), Value.Text(slot.entity), Value.Text(slot.entity_id), + Value.Text(slot.field), Value.Text(slot.reference.key), Value.I64(slot.binding_version)])) + except Exception: + raise RuntimeError("BLOB_CATALOG_FAILED") from None + if len(rows) != 1: + raise RuntimeError("BLOB_READ_DENIED") + row = rows[0] + values = (row.get("object_key"), row.get("content_version"), + row.get("file_name"), row.get("content_type")) + size = row.get("size_bytes") + if (any(not isinstance(value, str) or not value for value in values) + or type(size) is not int or size < 0): + raise RuntimeError("BLOB_CATALOG_FAILED") + return BlobCatalogRead(*values, size) + async def bind_in_graph(self, graph, actor, change: BlobBindingChange, authorization: BlobBindingAuthorization) -> int: if graph is None or not callable(getattr(graph, "with_sql_transport", None)): diff --git a/tests/provider/sqlite/test_blob_catalog_graph.py b/tests/provider/sqlite/test_blob_catalog_graph.py index 81d5c69a..7b1baf84 100644 --- a/tests/provider/sqlite/test_blob_catalog_graph.py +++ b/tests/provider/sqlite/test_blob_catalog_graph.py @@ -2,8 +2,10 @@ import aiosqlite import pytest +from datetime import datetime, timedelta, timezone from teaql.core.blob_ref import BlobRef +from teaql.core.object_storage import ObjectReadUrl from teaql.core.entity import EntityKey from teaql.core.meta import EntityDescriptor, PropertyDescriptor from teaql.core.mutation import InsertCommand @@ -12,8 +14,14 @@ from teaql.provider.sqlite.dialect import SqliteDialect from teaql.provider.sqlite.transport import SqliteTransport from teaql.runtime import RuntimeModule, UserContext +from teaql.runtime.view_preparation import ( + AuthorizedBlobSlot, AuthorizedResponseGraph, ViewOptions, + ViewPreparationError, ViewPreparationService, +) from teaql.sql.executor import SqlDataServiceExecutor -from teaql_blob_reference import BlobBindingChange, BlobBindingTarget, SqliteBlobCatalog +from teaql_blob_reference import ( + BlobBindingChange, BlobBindingTarget, SqliteBlobCatalog, SqliteBlobResponseContributor, +) class Authorization: @@ -135,3 +143,81 @@ async def test_catalog_rejects_old_not_null_binding_schema(tmp_path): await catalog.ensure_schema() with pytest.raises(RuntimeError, match="BLOB_SCHEMA_MISSING"): await catalog.validate_schema() + + +@pytest.mark.asyncio +async def test_current_sqlite_binding_is_reread_before_view_url_signing(tmp_path): + context, path, catalog = await fixture(tmp_path) + await catalog.ensure_schema() + reference = BlobRef("blob_python_read_1") + async with aiosqlite.connect(path) as db: + await db.execute("INSERT INTO teaql_blob_catalog " + "(key,scope,object_key,content_version,file_name,content_type,size_bytes,status) " + "VALUES (?,?,?,?,?,?,?,?)", + (reference.key, "tenant-a", "objects/read-1", "v1", "report.pdf", + "application/pdf", 7, "available")) + await db.commit() + auth = Authorization() + change = BlobBindingChange(BlobBindingTarget("Asset", "1", "document"), reference, 0, 1, "Create") + await context.execute_graph_save( + lambda graph: catalog.bind_in_graph(graph, graph.context, change, auth), comment="bind read fixture") + slot = AuthorizedBlobSlot(("document",), "Asset", "1", "document", "tenant-a", 1, reference) + record = await catalog.current_read(slot) + assert (record.object_key, record.content_version, record.size_bytes) == ("objects/read-1", "v1", 7) + for invalid in ( + AuthorizedBlobSlot(slot.path, slot.entity, slot.entity_id, slot.field, + slot.scope, 2, slot.reference), + AuthorizedBlobSlot(slot.path, slot.entity, slot.entity_id, slot.field, + "tenant-b", 1, slot.reference), + AuthorizedBlobSlot(slot.path, slot.entity, slot.entity_id, slot.field, + slot.scope, 1, BlobRef("blob_old")), + ): + with pytest.raises(RuntimeError, match="BLOB_READ_DENIED"): + await catalog.current_read(invalid) + + class ReadAuthorization: + allowed = True + + async def authorize(self, _context, _slot): + return self.allowed + + class Authorizer: + async def authorize(self, _context, _result, _options): + return AuthorizedResponseGraph({"document": reference}, (slot,)) + + class SigningStore: + calls = 0 + key = None + header = None + + def get_read_url(self, _invocation, key, options): + self.calls += 1 + self.key = key.value + self.header = options.content_disposition() + return ObjectReadUrl("https://example.invalid/opaque-ticket", + datetime.now(timezone.utc) + timedelta(seconds=30)) + + read_authorization = ReadAuthorization() + store = SigningStore() + context.with_object_storage(store) + context.insert_resource("response_graph_authorizer", Authorizer()) + context.insert_resource("blob_response_contributor", + SqliteBlobResponseContributor(catalog, read_authorization)) + view = await ViewPreparationService().prepare(context, None, + ViewOptions("ordinary", "read_only", datetime.now(timezone.utc) + timedelta(seconds=60))) + assert view.presentation == {"document": "https://example.invalid/opaque-ticket"} + assert (store.calls, store.key, store.header) == \ + (1, "objects/read-1", 'inline; filename="report.pdf"') + + read_authorization.allowed = False + with pytest.raises(ViewPreparationError) as denied: + await ViewPreparationService().prepare(context, None, + ViewOptions("ordinary", "read_only", datetime.now(timezone.utc) + timedelta(seconds=60))) + assert denied.value.code == "BLOB_READ_DENIED" + assert store.calls == 1 + + async with aiosqlite.connect(path) as db: + await db.execute("UPDATE teaql_blob_catalog SET status = 'deleted' WHERE key = ?", (reference.key,)) + await db.commit() + with pytest.raises(RuntimeError, match="BLOB_READ_DENIED"): + await catalog.current_read(slot) From dd56c39047a27d377d0a5ce209ad2e1c71e791e7 Mon Sep 17 00:00:00 2001 From: Philip Z Date: Tue, 27 Oct 2026 12:13:52 +0800 Subject: [PATCH 07/75] Package Python Blob reference as an independent optional wheel --- .github/workflows/example-gate.yml | 1 + .github/workflows/optional-blob-wheel.yml | 39 +++++++++++++++++++ .github/workflows/release.yml | 1 + README.md | 2 +- packages/teaql-blob-reference/README.md | 3 ++ packages/teaql-blob-reference/pyproject.toml | 17 ++++++++ .../src}/teaql_blob_reference/__init__.py | 0 .../src}/teaql_blob_reference/read_service.py | 0 .../teaql_blob_reference/sqlite_catalog.py | 0 9 files changed, 62 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/optional-blob-wheel.yml create mode 100644 packages/teaql-blob-reference/README.md create mode 100644 packages/teaql-blob-reference/pyproject.toml rename {src => packages/teaql-blob-reference/src}/teaql_blob_reference/__init__.py (100%) rename {src => packages/teaql-blob-reference/src}/teaql_blob_reference/read_service.py (100%) rename {src => packages/teaql-blob-reference/src}/teaql_blob_reference/sqlite_catalog.py (100%) diff --git a/.github/workflows/example-gate.yml b/.github/workflows/example-gate.yml index c44661b1..7c5cd67a 100644 --- a/.github/workflows/example-gate.yml +++ b/.github/workflows/example-gate.yml @@ -21,6 +21,7 @@ jobs: python-version: '3.10' cache: pip - run: pip install -e ".[test,telemetry,object-storage-s3]" + - run: pip install --no-deps -e ./packages/teaql-blob-reference - name: Start dedicated S3 example fixture run: | docker run -d --name teaql-python-example-s3 \ diff --git a/.github/workflows/optional-blob-wheel.yml b/.github/workflows/optional-blob-wheel.yml new file mode 100644 index 00000000..214c8252 --- /dev/null +++ b/.github/workflows/optional-blob-wheel.yml @@ -0,0 +1,39 @@ +name: Optional Blob wheel +on: + pull_request: + push: + branches: [main] + workflow_dispatch: +jobs: + optional-wheel: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.10' + cache: pip + - run: python -m pip install --upgrade build + - run: python -m pip install -e ".[test,object-storage-s3]" + - run: python -m pip install --no-deps -e ./packages/teaql-blob-reference + - run: python -m pytest -q tests/provider/sqlite/test_blob_catalog_graph.py + - run: python -m build --wheel --no-isolation --outdir .artifacts/base . + - run: python -m build --wheel --no-isolation --outdir .artifacts/blob ./packages/teaql-blob-reference + - name: Check wheel isolation + run: | + python - <<'PY' + from pathlib import Path + from zipfile import ZipFile + + base = list(Path('.artifacts/base').glob('*.whl')) + blob = list(Path('.artifacts/blob').glob('*.whl')) + assert len(base) == len(blob) == 1, (base, blob) + with ZipFile(base[0]) as wheel: + entries = wheel.namelist() + assert any(name.startswith('teaql/') for name in entries) + assert not any(name.startswith('teaql_blob_reference/') for name in entries) + with ZipFile(blob[0]) as wheel: + entries = wheel.namelist() + assert any(name.startswith('teaql_blob_reference/') for name in entries) + assert not any(name.startswith('teaql/') for name in entries) + PY diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index d90b5522..eea7a2c6 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -15,6 +15,7 @@ jobs: python-version: "3.12" - run: python -m pip install --upgrade build - run: python -m pip install --editable ".[test,telemetry]" + - run: python -m pip install --no-deps --editable ./packages/teaql-blob-reference - name: Verify tag matches package version run: test "${GITHUB_REF_NAME#python-v}" = "$(python -c 'import importlib.metadata; print(importlib.metadata.version("teaql"))')" - run: python -m pytest diff --git a/README.md b/README.md index c7547a9c..44766e34 100644 --- a/README.md +++ b/README.md @@ -424,7 +424,7 @@ distinguish implemented source, executed examples, artifact verification and rel ```bash python -m pip install -e '.[test]' -PYTHONDONTWRITEBYTECODE=1 PYTHONPATH=src python -m pytest -q +PYTHONDONTWRITEBYTECODE=1 PYTHONPATH=src:packages/teaql-blob-reference/src python -m pytest -q PYTHONDONTWRITEBYTECODE=1 bash scripts/verify-examples.sh ``` diff --git a/packages/teaql-blob-reference/README.md b/packages/teaql-blob-reference/README.md new file mode 100644 index 00000000..18e3048b --- /dev/null +++ b/packages/teaql-blob-reference/README.md @@ -0,0 +1,3 @@ +# teaql-blob-reference + +Optional Blob catalog and current-read services for TeaQL Python. Install this wheel alongside the matching `teaql` version. Catalog migration and runtime service installation are explicit host actions; this package does not install schema or expose routes on import. diff --git a/packages/teaql-blob-reference/pyproject.toml b/packages/teaql-blob-reference/pyproject.toml new file mode 100644 index 00000000..190d7644 --- /dev/null +++ b/packages/teaql-blob-reference/pyproject.toml @@ -0,0 +1,17 @@ +[build-system] +requires = ["setuptools>=61.0"] +build-backend = "setuptools.build_meta" + +[project] +name = "teaql-blob-reference" +version = "0.3.1" +description = "Optional TeaQL Blob reference module" +readme = "README.md" +requires-python = ">=3.10" +license = "Apache-2.0" +authors = [{ name = "TeaQL" }] +dependencies = ["teaql==0.3.1"] + +[tool.setuptools.packages.find] +where = ["src"] +include = ["teaql_blob_reference*"] diff --git a/src/teaql_blob_reference/__init__.py b/packages/teaql-blob-reference/src/teaql_blob_reference/__init__.py similarity index 100% rename from src/teaql_blob_reference/__init__.py rename to packages/teaql-blob-reference/src/teaql_blob_reference/__init__.py diff --git a/src/teaql_blob_reference/read_service.py b/packages/teaql-blob-reference/src/teaql_blob_reference/read_service.py similarity index 100% rename from src/teaql_blob_reference/read_service.py rename to packages/teaql-blob-reference/src/teaql_blob_reference/read_service.py diff --git a/src/teaql_blob_reference/sqlite_catalog.py b/packages/teaql-blob-reference/src/teaql_blob_reference/sqlite_catalog.py similarity index 100% rename from src/teaql_blob_reference/sqlite_catalog.py rename to packages/teaql-blob-reference/src/teaql_blob_reference/sqlite_catalog.py From 1f540e92f0122c5001ed56aa1fc5cf109d7af453 Mon Sep 17 00:00:00 2001 From: Philip Z Date: Tue, 27 Oct 2026 12:35:28 +0800 Subject: [PATCH 08/75] Add optional Python file storage and local authenticated HTTP reads --- .github/workflows/optional-blob-wheel.yml | 2 +- .../src/teaql_blob_reference/__init__.py | 3 +- .../src/teaql_blob_reference/file_storage.py | 371 ++++++++++++++++++ .../provider/sqlite/test_file_blob_storage.py | 82 ++++ 4 files changed, 456 insertions(+), 2 deletions(-) create mode 100644 packages/teaql-blob-reference/src/teaql_blob_reference/file_storage.py create mode 100644 tests/provider/sqlite/test_file_blob_storage.py diff --git a/.github/workflows/optional-blob-wheel.yml b/.github/workflows/optional-blob-wheel.yml index 214c8252..0abd59c7 100644 --- a/.github/workflows/optional-blob-wheel.yml +++ b/.github/workflows/optional-blob-wheel.yml @@ -16,7 +16,7 @@ jobs: - run: python -m pip install --upgrade build - run: python -m pip install -e ".[test,object-storage-s3]" - run: python -m pip install --no-deps -e ./packages/teaql-blob-reference - - run: python -m pytest -q tests/provider/sqlite/test_blob_catalog_graph.py + - run: python -m pytest -q tests/provider/sqlite/test_blob_catalog_graph.py tests/provider/sqlite/test_file_blob_storage.py - run: python -m build --wheel --no-isolation --outdir .artifacts/base . - run: python -m build --wheel --no-isolation --outdir .artifacts/blob ./packages/teaql-blob-reference - name: Check wheel isolation diff --git a/packages/teaql-blob-reference/src/teaql_blob_reference/__init__.py b/packages/teaql-blob-reference/src/teaql_blob_reference/__init__.py index 9e215b70..187bf799 100644 --- a/packages/teaql-blob-reference/src/teaql_blob_reference/__init__.py +++ b/packages/teaql-blob-reference/src/teaql_blob_reference/__init__.py @@ -2,6 +2,7 @@ from .sqlite_catalog import BlobBindingChange, BlobBindingTarget, BlobCatalogRead, SqliteBlobCatalog from .read_service import SqliteBlobResponseContributor +from .file_storage import FileObjectStorage __all__ = ["BlobBindingChange", "BlobBindingTarget", "BlobCatalogRead", - "SqliteBlobCatalog", "SqliteBlobResponseContributor"] + "SqliteBlobCatalog", "SqliteBlobResponseContributor", "FileObjectStorage"] diff --git a/packages/teaql-blob-reference/src/teaql_blob_reference/file_storage.py b/packages/teaql-blob-reference/src/teaql_blob_reference/file_storage.py new file mode 100644 index 00000000..d5fa1448 --- /dev/null +++ b/packages/teaql-blob-reference/src/teaql_blob_reference/file_storage.py @@ -0,0 +1,371 @@ +"""Optional private file store with an explicitly started authenticated local HTTP gateway.""" +from datetime import datetime, timedelta, timezone +from hashlib import sha256 +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path +from threading import RLock, Thread +from urllib.parse import urlsplit +import base64 +import json +import os +import re +import secrets +import stat +import time + +from cryptography.hazmat.primitives.ciphers.aead import AESGCM + +from teaql.core.object_storage import ( + ObjectContent, ObjectMetadata, ObjectReadUrl, StorageError, StorageErrorCode, WriteMode, +) + + +def _error(code): + return StorageError(code) + + +def _b64(data): + return base64.urlsafe_b64encode(data).rstrip(b"=").decode("ascii") + + +def _unb64(value): + if not isinstance(value, str) or len(value) > 4096 or not re.fullmatch(r"[A-Za-z0-9_-]+", value): + return None + try: + decoded = base64.urlsafe_b64decode(value + "=" * (-len(value) % 4)) + except (ValueError, base64.binascii.Error): + return None + return decoded if _b64(decoded) == value else None + + +class FileObjectStorage: + """The host must protect root and its parent; old generations require explicit GC.""" + + def __init__(self, root, signing_key: bytes): + try: + path = Path(root) + except TypeError: + raise _error(StorageErrorCode.INVALID_INPUT) from None + if not path.is_absolute() or not isinstance(signing_key, bytes) or len(signing_key) != 32: + raise _error(StorageErrorCode.INVALID_INPUT) + self._root = path + self._cipher = AESGCM(signing_key) + self._lock = RLock() + self._server = None + self._thread = None + self._base_url = None + + def _private_root(self, create=False): + if create: + try: + self._root.mkdir(mode=0o700, parents=True, exist_ok=True) + except OSError: + raise _error(StorageErrorCode.IO) from None + try: + info = self._root.lstat() + except OSError: + raise _error(StorageErrorCode.UNAVAILABLE) from None + if not stat.S_ISDIR(info.st_mode) or info.st_mode & 0o077: + raise _error(StorageErrorCode.UNAVAILABLE) + + def _directory(self, key_hash): + return self._root / key_hash[:2] / key_hash + + @staticmethod + def _hash(key): + return sha256(key.value.encode("utf-8")).hexdigest() + + def _manifest(self, key_hash): + try: + raw = (self._directory(key_hash) / "manifest.json").read_bytes() + except FileNotFoundError: + return None + except OSError: + raise _error(StorageErrorCode.UNAVAILABLE) from None + if len(raw) > 16384: + raise _error(StorageErrorCode.UNAVAILABLE) + try: + value = json.loads(raw) + version = value["version"] + if not isinstance(version, str) or not re.fullmatch(r"[A-Za-z0-9_-]{32}", version) or \ + type(value["size"]) is not int or not 0 <= value["size"] <= 5 * 1024**3 or \ + not isinstance(value["content_type"], str) or not value["content_type"] or \ + not all(32 <= ord(c) <= 126 for c in value["content_type"]) or \ + not isinstance(value["sha256"], str) or not re.fullmatch(r"[0-9a-f]{64}", value["sha256"]) or \ + not isinstance(value["modified"], str) or not isinstance(value["metadata"], dict): + raise ValueError("invalid manifest") + datetime.fromisoformat(value["modified"]) + except (ValueError, TypeError, KeyError): + raise _error(StorageErrorCode.UNAVAILABLE) from None + return value + + def _metadata(self, key, manifest): + return ObjectMetadata(key, manifest["size"], manifest["content_type"], manifest["sha256"], + datetime.fromisoformat(manifest["modified"]), manifest["metadata"]) + + def _data_path(self, key_hash, manifest): + return self._directory(key_hash) / ("data-" + manifest["version"]) + + def _check_data(self, key_hash, manifest): + try: + info = self._data_path(key_hash, manifest).stat() + except OSError: + raise _error(StorageErrorCode.UNAVAILABLE) from None + if not stat.S_ISREG(info.st_mode) or info.st_size != manifest["size"]: + raise _error(StorageErrorCode.UNAVAILABLE) + + def put(self, invocation, key, content, options): + self._private_root(create=True) + key_hash = self._hash(key) + directory = self._directory(key_hash) + try: + directory.mkdir(mode=0o700, parents=True, exist_ok=True) + except OSError: + raise _error(StorageErrorCode.IO) from None + version = _b64(secrets.token_bytes(24)) + data_path = directory / ("data-" + version) + temp_path = directory / ("manifest-" + version) + published = False + try: + digest = sha256() + size = 0 + with data_path.open("xb") as output: + while True: + try: + block = content.read(65536) + except Exception: + raise _error(StorageErrorCode.IO) from None + if not isinstance(block, bytes) or len(block) > options.size - size: + raise _error(StorageErrorCode.INVALID_INPUT) + if not block: + break + output.write(block) + digest.update(block) + size += len(block) + if size != options.size: + raise _error(StorageErrorCode.INVALID_INPUT) + output.flush() + os.fsync(output.fileno()) + manifest = dict(version=version, size=size, content_type=options.content_type, + sha256=digest.hexdigest(), modified=datetime.now(timezone.utc).isoformat(), + metadata=dict(options.metadata)) + with temp_path.open("xb") as output: + output.write(json.dumps(manifest, separators=(",", ":")).encode("utf-8")) + output.flush() + os.fsync(output.fileno()) + destination = directory / "manifest.json" + if options.mode == WriteMode.CREATE_ONLY: + try: + os.link(temp_path, destination) + except FileExistsError: + raise _error(StorageErrorCode.CONFLICT) from None + published = True + try: + temp_path.unlink(missing_ok=True) + except OSError: + pass + else: + os.replace(temp_path, destination) + published = True + return self._metadata(key, manifest) + except StorageError: + raise + except OSError: + raise _error(StorageErrorCode.IO) from None + finally: + if not published: + for path in (data_path, temp_path): + try: + path.unlink(missing_ok=True) + except OSError: + pass + + def get(self, invocation, key): + self._private_root() + key_hash = self._hash(key) + manifest = self._manifest(key_hash) + if manifest is None: + raise _error(StorageErrorCode.NOT_FOUND) + try: + stream = self._data_path(key_hash, manifest).open("rb") + info = os.fstat(stream.fileno()) + if not stat.S_ISREG(info.st_mode) or info.st_size != manifest["size"]: + stream.close() + raise _error(StorageErrorCode.UNAVAILABLE) + except OSError: + raise _error(StorageErrorCode.UNAVAILABLE) from None + return ObjectContent(self._metadata(key, manifest), stream) + + def head(self, invocation, key): + self._private_root() + key_hash = self._hash(key) + manifest = self._manifest(key_hash) + if manifest is None: + return None + self._check_data(key_hash, manifest) + return self._metadata(key, manifest) + + def delete(self, invocation, key): + self._private_root() + try: + (self._directory(self._hash(key)) / "manifest.json").unlink(missing_ok=True) + except OSError: + raise _error(StorageErrorCode.IO) from None + + def _seal(self, ticket): + nonce = secrets.token_bytes(12) + payload = json.dumps(ticket, separators=(",", ":")).encode("utf-8") + return _b64(nonce + self._cipher.encrypt(nonce, payload, None)) + + def _open_ticket(self, token): + raw = _unb64(token) + if raw is None or len(raw) < 29: + return None + try: + value = json.loads(self._cipher.decrypt(raw[:12], raw[12:], None)) + if not isinstance(value, dict) or not isinstance(value.get("hash"), str) or \ + not re.fullmatch(r"[0-9a-f]{64}", value["hash"]) or \ + not isinstance(value.get("version"), str) or \ + not re.fullmatch(r"[A-Za-z0-9_-]{32}", value["version"]) or \ + type(value.get("expires_ms")) is not int or \ + value["expires_ms"] <= time.time_ns() // 1_000_000 or \ + not (value.get("disposition") is None or isinstance(value["disposition"], str) and + all(32 <= ord(c) <= 126 for c in value["disposition"])): + return None + except Exception: + return None + return value + + def get_read_url(self, invocation, key, options): + with self._lock: + base = self._base_url + if base is None: + raise _error(StorageErrorCode.NOT_CONFIGURED) + self._private_root() + key_hash = self._hash(key) + manifest = self._manifest(key_hash) + if manifest is None: + raise _error(StorageErrorCode.NOT_FOUND) + self._check_data(key_hash, manifest) + expires_at = datetime.now(timezone.utc) + options.lifetime + ticket = self._seal(dict(hash=key_hash, version=manifest["version"], + expires_ms=int(expires_at.timestamp() * 1000), + disposition=options.content_disposition())) + return ObjectReadUrl(base + "/objects/" + ticket, expires_at) + + def start(self): + with self._lock: + if self._server is not None: + raise _error(StorageErrorCode.CONFLICT) + self._private_root(create=True) + owner = self + + class Handler(BaseHTTPRequestHandler): + def do_GET(self): + owner._serve(self, head=False) + + def do_HEAD(self): + owner._serve(self, head=True) + + def do_POST(self): + self.send_error(405) + + do_PUT = do_POST + do_DELETE = do_POST + + def log_message(self, *_): + pass # The path contains a private read ticket. + + try: + server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + except OSError: + raise _error(StorageErrorCode.UNAVAILABLE) from None + server.daemon_threads = True + thread = Thread(target=server.serve_forever, daemon=True) + self._server, self._thread = server, thread + self._base_url = f"http://127.0.0.1:{server.server_port}" + thread.start() + return self._base_url + + def close(self): + with self._lock: + server, thread = self._server, self._thread + self._server = self._thread = self._base_url = None + if server is not None: + server.shutdown() + server.server_close() + thread.join(timeout=5) + + def _serve(self, handler, head): + path = urlsplit(handler.path).path + token = path.removeprefix("/objects/") if path.startswith("/objects/") else "" + ticket = self._open_ticket(token) + if ticket is None: + handler.send_error(404) + return + try: + manifest = self._manifest(ticket["hash"]) + if manifest is None or manifest["version"] != ticket["version"]: + handler.send_error(404) + return + data_path = self._data_path(ticket["hash"], manifest) + with data_path.open("rb") as source: + info = os.fstat(source.fileno()) + if not stat.S_ISREG(info.st_mode) or info.st_size != manifest["size"]: + handler.send_error(404) + return + size, start, end = manifest["size"], 0, manifest["size"] - 1 + range_header = handler.headers.get("Range") + partial = range_header is not None + if partial: + match = re.fullmatch(r"bytes=(\d*)-(\d*)", range_header) + if not match or size == 0 or not any(match.groups()) or \ + any(len(part) > 20 for part in match.groups()): + handler.send_response(416) + handler.send_header("Content-Range", f"bytes */{size}") + handler.end_headers() + return + first, last = match.groups() + if first: + start = int(first) + if last: + end = int(last) + else: + suffix = int(last) + start = max(0, size - suffix) + if start >= size or end < start or not first and int(last) == 0: + handler.send_response(416) + handler.send_header("Content-Range", f"bytes */{size}") + handler.end_headers() + return + end = min(end, size - 1) + handler.send_response(206 if partial else 200) + handler.send_header("Content-Type", manifest["content_type"]) + handler.send_header("Content-Length", str(end - start + 1 if partial else size)) + handler.send_header("Accept-Ranges", "bytes") + handler.send_header("Cache-Control", "private, no-store") + handler.send_header("X-Content-Type-Options", "nosniff") + handler.send_header("ETag", '"' + manifest["sha256"] + '"') + if ticket["disposition"]: + handler.send_header("Content-Disposition", ticket["disposition"]) + if partial: + handler.send_header("Content-Range", f"bytes {start}-{end}/{size}") + handler.end_headers() + if not head and size: + source.seek(start) + remaining = end - start + 1 + while remaining: + block = source.read(min(65536, remaining)) + if not block: + break + handler.wfile.write(block) + remaining -= len(block) + except (OSError, StorageError): + if not handler.wfile.closed: + try: + handler.send_error(404) + except OSError: + pass + + def __repr__(self): + return "FileObjectStorage[configuration=masked]" diff --git a/tests/provider/sqlite/test_file_blob_storage.py b/tests/provider/sqlite/test_file_blob_storage.py new file mode 100644 index 00000000..b687b163 --- /dev/null +++ b/tests/provider/sqlite/test_file_blob_storage.py @@ -0,0 +1,82 @@ +"""Real local File/HTTP behavior for the independent optional wheel.""" +from datetime import timedelta +from urllib.error import HTTPError +from urllib.request import Request, urlopen +import hashlib +import os +import time + +import pytest + +from teaql_blob_reference import FileObjectStorage +from teaql.core.object_storage import ( + ObjectKey, PutOptions, ReadUrlOptions, StorageError, StorageErrorCode, WriteMode, +) +from teaql.runtime import UserContext + + +def _status(url, *, method="GET", headers=None): + try: + with urlopen(Request(url, method=method, headers=headers or {}), timeout=5) as response: + return response.status, dict(response.headers), response.read() + except HTTPError as error: + return error.code, dict(error.headers), error.read() + + +def test_file_storage_exact_bytes_and_authenticated_http(tmp_path): + provider = FileObjectStorage(tmp_path / "objects", os.urandom(32)) + context = UserContext().with_object_storage(provider).with_object_storage_observer(lambda event: None) + store = context.object_storage("verify real optional file store") + key = ObjectKey("objects/PRIVATE-PYTHON-CANARY") + content = b"%PDF-1.7\nTeaQL document\n%%EOF" + try: + with pytest.raises(StorageError) as unstarted: + store.get_read_url(key) + assert unstarted.value.code == StorageErrorCode.NOT_CONFIGURED + assert provider.start().startswith("http://127.0.0.1:") + options = PutOptions(len(content), WriteMode.CREATE_ONLY, "application/pdf") + metadata = store.put_bytes(key, content, options) + assert metadata.etag == hashlib.sha256(content).hexdigest() + assert store.head(key).size == len(content) + assert store.get_bytes(key, len(content)) == content + with pytest.raises(StorageError) as conflict: + store.put_bytes(key, content, options) + assert conflict.value.code == StorageErrorCode.CONFLICT + with pytest.raises(StorageError): + store.put_bytes(key, content, PutOptions(len(content) + 1, WriteMode.REPLACE)) + assert store.get_bytes(key, len(content)) == content + + signed = store.get_read_url(key, ReadUrlOptions(timedelta(seconds=2), "attachment", "asset.pdf")) + assert key.value not in signed.url and "PRIVATE-PYTHON-CANARY" not in repr(signed) + code, headers, body = _status(signed.url) + assert code == 200 and body == content + assert headers["Content-Type"] == "application/pdf" + assert headers["Content-Disposition"] == 'attachment; filename="asset.pdf"' + assert headers["Cache-Control"] == "private, no-store" + code, headers, body = _status(signed.url, method="HEAD") + assert code == 200 and body == b"" and headers["Content-Length"] == str(len(content)) + code, headers, body = _status(signed.url, headers={"Range": "bytes=0-3"}) + assert code == 206 and body == content[:4] + assert headers["Content-Range"] == f"bytes 0-3/{len(content)}" + code, _, body = _status(signed.url, headers={"Range": "bytes=-3"}) + assert code == 206 and body == content[-3:] + assert _status(signed.url, headers={"Range": "bytes=999-"})[0] == 416 + + tampered = signed.url.replace("/objects/", "/objects/A", 1) + assert _status(tampered)[0] == 404 + replacement = b"next generation" + store.put_bytes(key, replacement, PutOptions(len(replacement), WriteMode.REPLACE)) + assert _status(signed.url)[0] == 404 + assert store.get_bytes(key, len(replacement)) == replacement + expiring = store.get_read_url(key, ReadUrlOptions(timedelta(seconds=1))) + time.sleep(1.1) + assert _status(expiring.url)[0] == 404 + current = provider._manifest(provider._hash(key)) + provider._data_path(provider._hash(key), current).unlink() + with pytest.raises(StorageError) as missing: + store.head(key) + assert missing.value.code == StorageErrorCode.UNAVAILABLE + store.delete(key) + assert not store.exists(key) + finally: + provider.close() From 319969f473d32f9525aad789037305b9b6c538d9 Mon Sep 17 00:00:00 2001 From: Philip Z Date: Tue, 27 Oct 2026 12:39:17 +0800 Subject: [PATCH 09/75] Track Python Blob availability across SQLite bindings --- .../teaql_blob_reference/sqlite_catalog.py | 22 ++++++++++++++++++- .../sqlite/test_blob_catalog_graph.py | 14 +++++++++++- 2 files changed, 34 insertions(+), 2 deletions(-) diff --git a/packages/teaql-blob-reference/src/teaql_blob_reference/sqlite_catalog.py b/packages/teaql-blob-reference/src/teaql_blob_reference/sqlite_catalog.py index 793a6bb0..fea718e0 100644 --- a/packages/teaql-blob-reference/src/teaql_blob_reference/sqlite_catalog.py +++ b/packages/teaql-blob-reference/src/teaql_blob_reference/sqlite_catalog.py @@ -199,7 +199,8 @@ async def bind(sql): if change.next: try: available = await sql.fetch_all_sql(CompiledQuery( - "SELECT key FROM teaql_blob_catalog WHERE key = ? AND scope = ? AND status = 'available'", + "SELECT key FROM teaql_blob_catalog WHERE key = ? AND scope = ? " + "AND status IN ('uploaded_unbound','available')", [Value.Text(change.next.key), Value.Text(scope)])) except Exception: raise RuntimeError("BLOB_CATALOG_FAILED") from None @@ -221,6 +222,25 @@ async def bind(sql): raise RuntimeError("BLOB_CATALOG_FAILED") from None if affected != 1: raise RuntimeError("BLOB_BIND_DENIED") + if change.next: + try: + promoted, _ = await sql.execute_sql(CompiledQuery( + "UPDATE teaql_blob_catalog SET status = 'available' WHERE key = ? AND scope = ? " + "AND status IN ('uploaded_unbound','available')", + [Value.Text(change.next.key), Value.Text(scope)])) + except Exception: + raise RuntimeError("BLOB_CATALOG_FAILED") from None + if promoted != 1: + raise RuntimeError("BLOB_BIND_DENIED") + if previous and previous != change.next: + try: + await sql.execute_sql(CompiledQuery( + "UPDATE teaql_blob_catalog SET status = 'uploaded_unbound' " + "WHERE key = ? AND scope = ? AND status = 'available' " + "AND NOT EXISTS (SELECT 1 FROM teaql_blob_binding WHERE key = ?)", + [Value.Text(previous.key), Value.Text(scope), Value.Text(previous.key)])) + except Exception: + raise RuntimeError("BLOB_CATALOG_FAILED") from None return change.next_version return await graph.with_sql_transport(bind) diff --git a/tests/provider/sqlite/test_blob_catalog_graph.py b/tests/provider/sqlite/test_blob_catalog_graph.py index 7b1baf84..0043c86c 100644 --- a/tests/provider/sqlite/test_blob_catalog_graph.py +++ b/tests/provider/sqlite/test_blob_catalog_graph.py @@ -72,7 +72,7 @@ async def test_catalog_binding_same_graph_rollback_authority_tombstone_and_rebin "(key,scope,object_key,content_version,file_name,content_type,size_bytes,status) " "VALUES (?,?,?,?,?,?,?,?)", (reference.key, "tenant-a", "objects/1", "v1", "asset.pdf", - "application/pdf", 7, "available")) + "application/pdf", 7, "uploaded_unbound")) await db.commit() auth = Authorization() @@ -96,6 +96,7 @@ async def save(graph, asset_id, *, abort=False, catch_denial=False, foreign_acto await context.execute_graph_save(lambda graph: save(graph, 1), comment="save asset") assert len(await all_rows(path, "asset")) == len(await all_rows(path, "teaql_blob_binding")) == 1 + assert (await all_rows(path, "teaql_blob_catalog"))[0][-1] == "available" with pytest.raises(RuntimeError, match="abort graph"): await context.execute_graph_save(lambda graph: save(graph, 2, abort=True), comment="save asset") assert len(await all_rows(path, "asset")) == len(await all_rows(path, "teaql_blob_binding")) == 1 @@ -126,10 +127,21 @@ async def save(graph, asset_id, *, abort=False, catch_denial=False, foreign_acto await context.execute_graph_save( lambda graph: catalog.bind_in_graph(graph, graph.context, clear, auth), comment="clear binding") assert (await all_rows(path, "teaql_blob_binding"))[0][-2:] == (None, 2) + assert (await all_rows(path, "teaql_blob_catalog"))[0][-1] == "uploaded_unbound" rebind = BlobBindingChange(BlobBindingTarget("Asset", "1", "document"), reference, 2, 3, "Update") await context.execute_graph_save( lambda graph: catalog.bind_in_graph(graph, graph.context, rebind, auth), comment="rebind") assert (await all_rows(path, "teaql_blob_binding"))[0][-2:] == (reference.key, 3) + assert (await all_rows(path, "teaql_blob_catalog"))[0][-1] == "available" + await context.execute_graph_save(lambda graph: save(graph, 2), comment="bind second asset") + clear = BlobBindingChange(BlobBindingTarget("Asset", "1", "document"), None, 3, 4, "Update") + await context.execute_graph_save( + lambda graph: catalog.bind_in_graph(graph, graph.context, clear, auth), comment="clear first binding") + assert (await all_rows(path, "teaql_blob_catalog"))[0][-1] == "available" + clear = BlobBindingChange(BlobBindingTarget("Asset", "2", "document"), None, 1, 2, "Update") + await context.execute_graph_save( + lambda graph: catalog.bind_in_graph(graph, graph.context, clear, auth), comment="clear final binding") + assert (await all_rows(path, "teaql_blob_catalog"))[0][-1] == "uploaded_unbound" @pytest.mark.asyncio From 4e9ef208b4d4bc5b38ba0134eb7a5ad29b4da4de Mon Sep 17 00:00:00 2001 From: Philip Z Date: Tue, 27 Oct 2026 13:03:09 +0800 Subject: [PATCH 10/75] Verify authorized Python Blob view against real File HTTP storage --- .../sqlite/test_blob_catalog_graph.py | 66 ++++++++++++++++++- 1 file changed, 64 insertions(+), 2 deletions(-) diff --git a/tests/provider/sqlite/test_blob_catalog_graph.py b/tests/provider/sqlite/test_blob_catalog_graph.py index 0043c86c..6224a660 100644 --- a/tests/provider/sqlite/test_blob_catalog_graph.py +++ b/tests/provider/sqlite/test_blob_catalog_graph.py @@ -1,11 +1,13 @@ """Real SQLite catalog binding belongs to the native business graph owner.""" import aiosqlite +import os import pytest from datetime import datetime, timedelta, timezone +from urllib.request import urlopen from teaql.core.blob_ref import BlobRef -from teaql.core.object_storage import ObjectReadUrl +from teaql.core.object_storage import ObjectKey, ObjectReadUrl, PutOptions, WriteMode from teaql.core.entity import EntityKey from teaql.core.meta import EntityDescriptor, PropertyDescriptor from teaql.core.mutation import InsertCommand @@ -20,7 +22,8 @@ ) from teaql.sql.executor import SqlDataServiceExecutor from teaql_blob_reference import ( - BlobBindingChange, BlobBindingTarget, SqliteBlobCatalog, SqliteBlobResponseContributor, + BlobBindingChange, BlobBindingTarget, FileObjectStorage, SqliteBlobCatalog, + SqliteBlobResponseContributor, ) @@ -233,3 +236,62 @@ def get_read_url(self, _invocation, key, options): await db.commit() with pytest.raises(RuntimeError, match="BLOB_READ_DENIED"): await catalog.current_read(slot) + + +@pytest.mark.asyncio +async def test_authorized_sqlite_view_projects_real_file_http_bytes_and_denies_revoked_refresh(tmp_path): + context, path, catalog = await fixture(tmp_path) + await catalog.ensure_schema() + provider = FileObjectStorage(tmp_path / "private", os.urandom(32)) + try: + assert provider.start().startswith("http://127.0.0.1:") + context.with_object_storage(provider) + key = ObjectKey("objects/read-real") + data = b"%PDF-1x" + context.object_storage("publish current Blob").put_bytes( + key, data, PutOptions(len(data), WriteMode.CREATE_ONLY, "application/pdf")) + reference = BlobRef("blob_python_real_1") + async with aiosqlite.connect(path) as db: + await db.execute("INSERT INTO teaql_blob_catalog " + "(key,scope,object_key,content_version,file_name,content_type,size_bytes,status) " + "VALUES (?,?,?,?,?,?,?,?)", + (reference.key, "tenant-a", key.value, "v1", "report.pdf", + "application/pdf", len(data), "uploaded_unbound")) + await db.commit() + change = BlobBindingChange(BlobBindingTarget("Asset", "1", "document"), + reference, 0, 1, "Create") + await context.execute_graph_save( + lambda graph: catalog.bind_in_graph(graph, graph.context, change, Authorization()), + comment="bind real file") + slot = AuthorizedBlobSlot(("document",), "Asset", "1", "document", "tenant-a", 1, reference) + + class ReadAuthorization: + allowed = True + + async def authorize(self, _context, _slot): + return self.allowed + + class Authorizer: + async def authorize(self, _context, _result, _options): + return AuthorizedResponseGraph({"document": reference}, (slot,)) + + read_permission = ReadAuthorization() + context.insert_resource("response_graph_authorizer", Authorizer()) + context.insert_resource("blob_response_contributor", + SqliteBlobResponseContributor(catalog, read_permission)) + options = ViewOptions("ordinary", "read_only", + datetime.now(timezone.utc) + timedelta(seconds=60)) + view = await ViewPreparationService().prepare(context, None, options) + link = view.presentation["document"] + assert link.startswith("http://127.0.0.1:") and key.value not in link + with urlopen(link, timeout=5) as response: + assert response.status == 200 + assert response.read() == data + assert response.headers["Content-Type"] == "application/pdf" + assert response.headers["Content-Disposition"] == 'inline; filename="report.pdf"' + read_permission.allowed = False + with pytest.raises(ViewPreparationError) as denied: + await ViewPreparationService().prepare(context, None, options) + assert denied.value.code == "BLOB_READ_DENIED" + finally: + provider.close() From 7eabe9dac66eb45c521b66da2a5834932ee758b9 Mon Sep 17 00:00:00 2001 From: Philip Z Date: Tue, 27 Oct 2026 13:10:03 +0800 Subject: [PATCH 11/75] Bind File ETags to immutable generations --- .../src/teaql_blob_reference/file_storage.py | 5 +++-- tests/provider/sqlite/test_file_blob_storage.py | 5 ++++- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/packages/teaql-blob-reference/src/teaql_blob_reference/file_storage.py b/packages/teaql-blob-reference/src/teaql_blob_reference/file_storage.py index d5fa1448..4044cc8f 100644 --- a/packages/teaql-blob-reference/src/teaql_blob_reference/file_storage.py +++ b/packages/teaql-blob-reference/src/teaql_blob_reference/file_storage.py @@ -100,7 +100,8 @@ def _manifest(self, key_hash): return value def _metadata(self, key, manifest): - return ObjectMetadata(key, manifest["size"], manifest["content_type"], manifest["sha256"], + return ObjectMetadata(key, manifest["size"], manifest["content_type"], + manifest["sha256"] + "." + manifest["version"], datetime.fromisoformat(manifest["modified"]), manifest["metadata"]) def _data_path(self, key_hash, manifest): @@ -345,7 +346,7 @@ def _serve(self, handler, head): handler.send_header("Accept-Ranges", "bytes") handler.send_header("Cache-Control", "private, no-store") handler.send_header("X-Content-Type-Options", "nosniff") - handler.send_header("ETag", '"' + manifest["sha256"] + '"') + handler.send_header("ETag", '"' + manifest["sha256"] + "." + manifest["version"] + '"') if ticket["disposition"]: handler.send_header("Content-Disposition", ticket["disposition"]) if partial: diff --git a/tests/provider/sqlite/test_file_blob_storage.py b/tests/provider/sqlite/test_file_blob_storage.py index b687b163..83c46459 100644 --- a/tests/provider/sqlite/test_file_blob_storage.py +++ b/tests/provider/sqlite/test_file_blob_storage.py @@ -36,7 +36,10 @@ def test_file_storage_exact_bytes_and_authenticated_http(tmp_path): assert provider.start().startswith("http://127.0.0.1:") options = PutOptions(len(content), WriteMode.CREATE_ONLY, "application/pdf") metadata = store.put_bytes(key, content, options) - assert metadata.etag == hashlib.sha256(content).hexdigest() + assert metadata.etag.startswith(hashlib.sha256(content).hexdigest() + ".") + same = store.put_bytes(key, content, PutOptions(len(content), WriteMode.REPLACE, "application/pdf")) + assert same.etag != metadata.etag + assert same.etag.startswith(hashlib.sha256(content).hexdigest() + ".") assert store.head(key).size == len(content) assert store.get_bytes(key, len(content)) == content with pytest.raises(StorageError) as conflict: From e05d892e656eb00516c03304a2337b9147a88cc7 Mon Sep 17 00:00:00 2001 From: Philip Z Date: Tue, 27 Oct 2026 13:25:58 +0800 Subject: [PATCH 12/75] Add scoped SQLite Blob upload verification to optional wheel --- .github/workflows/optional-blob-wheel.yml | 6 +- packages/teaql-blob-reference/README.md | 2 + .../src/teaql_blob_reference/__init__.py | 6 +- .../teaql_blob_reference/sqlite_catalog.py | 13 + .../teaql_blob_reference/upload_service.py | 401 ++++++++++++++++++ src/teaql/core/blob_ref.py | 5 + tests/provider/sqlite/test_blob_upload.py | 158 +++++++ 7 files changed, 589 insertions(+), 2 deletions(-) create mode 100644 packages/teaql-blob-reference/src/teaql_blob_reference/upload_service.py create mode 100644 tests/provider/sqlite/test_blob_upload.py diff --git a/.github/workflows/optional-blob-wheel.yml b/.github/workflows/optional-blob-wheel.yml index 0abd59c7..10b2179e 100644 --- a/.github/workflows/optional-blob-wheel.yml +++ b/.github/workflows/optional-blob-wheel.yml @@ -16,7 +16,7 @@ jobs: - run: python -m pip install --upgrade build - run: python -m pip install -e ".[test,object-storage-s3]" - run: python -m pip install --no-deps -e ./packages/teaql-blob-reference - - run: python -m pytest -q tests/provider/sqlite/test_blob_catalog_graph.py tests/provider/sqlite/test_file_blob_storage.py + - run: python -m pytest -q tests/provider/sqlite/test_blob_catalog_graph.py tests/provider/sqlite/test_file_blob_storage.py tests/provider/sqlite/test_blob_upload.py - run: python -m build --wheel --no-isolation --outdir .artifacts/base . - run: python -m build --wheel --no-isolation --outdir .artifacts/blob ./packages/teaql-blob-reference - name: Check wheel isolation @@ -36,4 +36,8 @@ jobs: entries = wheel.namelist() assert any(name.startswith('teaql_blob_reference/') for name in entries) assert not any(name.startswith('teaql/') for name in entries) + for module in ('file_storage.py', 'sqlite_catalog.py', 'upload_service.py'): + name = f'teaql_blob_reference/{module}' + assert name in entries + assert wheel.read(name) == (Path('packages/teaql-blob-reference/src') / name).read_bytes() PY diff --git a/packages/teaql-blob-reference/README.md b/packages/teaql-blob-reference/README.md index 18e3048b..9eed105f 100644 --- a/packages/teaql-blob-reference/README.md +++ b/packages/teaql-blob-reference/README.md @@ -1,3 +1,5 @@ # teaql-blob-reference Optional Blob catalog and current-read services for TeaQL Python. Install this wheel alongside the matching `teaql` version. Catalog migration and runtime service installation are explicit host actions; this package does not install schema or expose routes on import. + +`SqliteBlobUploadService` offers a host-invoked, scope-bound upload journal. The host must provide current model/action authorization and a trusted parser that consumes every stored byte and returns actual format, size, and SHA256. The service writes through the context's object-storage provider, rechecks the stored generation before confirmation, and promotes the row to `uploaded_unbound` only after full verification. The included PDF verifier in tests is a fixture, not a production format parser. Failure recovery, retention/GC, other SQL backends, and production ten-format parsing remain open. diff --git a/packages/teaql-blob-reference/src/teaql_blob_reference/__init__.py b/packages/teaql-blob-reference/src/teaql_blob_reference/__init__.py index 187bf799..c9b09a8f 100644 --- a/packages/teaql-blob-reference/src/teaql_blob_reference/__init__.py +++ b/packages/teaql-blob-reference/src/teaql_blob_reference/__init__.py @@ -3,6 +3,10 @@ from .sqlite_catalog import BlobBindingChange, BlobBindingTarget, BlobCatalogRead, SqliteBlobCatalog from .read_service import SqliteBlobResponseContributor from .file_storage import FileObjectStorage +from .upload_service import (BlobUploadInput, BlobUploadSession, BlobUploadTarget, + BlobVerifiedContent, SqliteBlobUploadService) __all__ = ["BlobBindingChange", "BlobBindingTarget", "BlobCatalogRead", - "SqliteBlobCatalog", "SqliteBlobResponseContributor", "FileObjectStorage"] + "SqliteBlobCatalog", "SqliteBlobResponseContributor", "FileObjectStorage", + "BlobUploadInput", "BlobUploadSession", "BlobUploadTarget", + "BlobVerifiedContent", "SqliteBlobUploadService"] diff --git a/packages/teaql-blob-reference/src/teaql_blob_reference/sqlite_catalog.py b/packages/teaql-blob-reference/src/teaql_blob_reference/sqlite_catalog.py index fea718e0..0a5f1bb8 100644 --- a/packages/teaql-blob-reference/src/teaql_blob_reference/sqlite_catalog.py +++ b/packages/teaql-blob-reference/src/teaql_blob_reference/sqlite_catalog.py @@ -89,6 +89,16 @@ async def ensure_schema(self): FOREIGN KEY (key) REFERENCES teaql_blob_catalog(key)) """, "CREATE INDEX IF NOT EXISTS teaql_blob_binding_key ON teaql_blob_binding(key)", + """CREATE TABLE IF NOT EXISTS teaql_blob_upload ( + upload_id TEXT PRIMARY KEY, scope TEXT NOT NULL, replay_key TEXT NOT NULL, + request_hash TEXT NOT NULL, key TEXT NOT NULL, entity TEXT NOT NULL, + entity_id TEXT NOT NULL, field TEXT NOT NULL, file_name TEXT NOT NULL, + extension TEXT NOT NULL, declared_size BIGINT NOT NULL, + expires_ms BIGINT NOT NULL, state TEXT NOT NULL CHECK + (state IN ('initiated','written','completed')), + written_sha256 TEXT, written_etag TEXT, + UNIQUE(scope,replay_key), FOREIGN KEY(key) REFERENCES teaql_blob_catalog(key)) + """, ) for sql in statements: try: @@ -102,6 +112,9 @@ async def validate_schema(self): "FROM teaql_blob_catalog WHERE 1 = 0", "SELECT scope, entity, entity_id, field, key, version " "FROM teaql_blob_binding WHERE 1 = 0", + "SELECT upload_id, scope, replay_key, request_hash, key, entity, entity_id, " + "field, file_name, extension, declared_size, expires_ms, state, " + "written_sha256, written_etag FROM teaql_blob_upload WHERE 1 = 0", ): try: await self.transport.fetch_all_sql(CompiledQuery(sql, [])) diff --git a/packages/teaql-blob-reference/src/teaql_blob_reference/upload_service.py b/packages/teaql-blob-reference/src/teaql_blob_reference/upload_service.py new file mode 100644 index 00000000..dd5114d9 --- /dev/null +++ b/packages/teaql-blob-reference/src/teaql_blob_reference/upload_service.py @@ -0,0 +1,401 @@ +"""Scoped SQLite Blob upload journal and full-byte confirmation. + +The host supplies current authorization and a trusted format parser. This module +does not treat a client MIME hint or storage metadata as format verification. +""" + +from __future__ import annotations + +from contextlib import suppress +from dataclasses import dataclass +from datetime import datetime, timezone +from hashlib import sha256 +from typing import BinaryIO, Callable, Protocol +import asyncio +import base64 +import json +import re +import secrets +import time + +from teaql.core.blob_ref import BlobFieldSpec, BlobRef, blob_mime_for_extension +from teaql.core.object_storage import MAX_PUT_SIZE, ObjectKey, PutOptions, WriteMode +from teaql.core.value import Value +from teaql.sql.executor import SqlTransactionTransport +from teaql.sql.types import CompiledQuery + +from .sqlite_catalog import SqliteBlobCatalog, _entity_id, _name, _scope + + +@dataclass(frozen=True) +class BlobUploadTarget: + entity: str + entity_id: str + field: str + + +@dataclass(frozen=True) +class BlobUploadInput: + file_name: str + declared_size: int + declared_content_type: str = "" + + +@dataclass(frozen=True) +class BlobVerifiedContent: + size_bytes: int + sha256: str + content_type: str + + +@dataclass(frozen=True, repr=False) +class BlobUploadSession: + upload_id: str + reference: BlobRef + expires_at: datetime + + def __repr__(self): + return "BlobUploadSession[masked]" + + +class BlobUploadAuthorization(Protocol): + def scope(self, context) -> str: ... + def field(self, context, target: BlobUploadTarget) -> BlobFieldSpec: ... + async def authorize(self, context, target: BlobUploadTarget, action: str, + input: BlobUploadInput) -> bool: ... + + +class BlobContentVerifier(Protocol): + def verify(self, stream: BinaryIO, extension: str, maximum_bytes: int) -> BlobVerifiedContent: ... + + +def _token(): + return base64.urlsafe_b64encode(secrets.token_bytes(24)).rstrip(b"=").decode("ascii") + + +def _text(value, maximum): + if not isinstance(value, str) or not value or value != value.strip(): + return False + try: + return len(value.encode("utf-8")) <= maximum and not any( + ord(c) < 32 or 127 <= ord(c) <= 159 for c in value) + except UnicodeEncodeError: + return False + + +def _query(sql, *params): + return CompiledQuery(sql, list(params)) + + +class _CountingReader: + def __init__(self, stream: BinaryIO, maximum: int): + self.stream = stream + self.maximum = maximum + self.digest = sha256() + self.count = 0 + self.eof = False + + def read(self, size=-1): + block = self.stream.read(size) + if not isinstance(block, bytes): + raise RuntimeError("BLOB_UPLOAD_INCOMPLETE") + self.count += len(block) + if self.count > self.maximum: + raise RuntimeError("BLOB_SIZE_EXCEEDED") + self.digest.update(block) + if not block and size != 0: + self.eof = True + return block + + def close(self): + self.stream.close() + + +class SqliteBlobUploadService: + """Host-invoked upload path; schema migration remains explicit.""" + + def __init__(self, catalog: SqliteBlobCatalog, authorization: BlobUploadAuthorization, + verifier: BlobContentVerifier, maximum_bytes: int, lease_seconds: int = 900, + now_ms: Callable[[], int] | None = None): + if (not isinstance(catalog, SqliteBlobCatalog) + or not isinstance(catalog.transport, SqlTransactionTransport) + or authorization is None or verifier is None + or not callable(getattr(verifier, "verify", None)) + or type(maximum_bytes) is not int or not 1 <= maximum_bytes <= MAX_PUT_SIZE + or type(lease_seconds) is not int or not 1 <= lease_seconds <= 3600): + raise RuntimeError("BLOB_POLICY_INVALID") + self.catalog = catalog + self.authorization = authorization + self.verifier = verifier + self.maximum_bytes = maximum_bytes + self.lease_seconds = lease_seconds + self.now_ms = now_ms or (lambda: time.time_ns() // 1_000_000) + + def _normalize(self, input: BlobUploadInput): + if (not isinstance(input, BlobUploadInput) or type(input.declared_size) is not int + or not 0 <= input.declared_size <= self.maximum_bytes): + raise RuntimeError("BLOB_SIZE_EXCEEDED") + if not _text(input.file_name, 1024): + raise RuntimeError("BLOB_FORMAT_NOT_ALLOWED") + name = input.file_name.replace("\\", "/").rsplit("/", 1)[-1] + if not _text(name, 512): + raise RuntimeError("BLOB_FORMAT_NOT_ALLOWED") + extension = name.rsplit(".", 1)[-1].lower() + if blob_mime_for_extension(extension) is None: + raise RuntimeError("BLOB_FORMAT_NOT_ALLOWED") + declared = input.declared_content_type.strip().lower() if isinstance( + input.declared_content_type, str) else None + if declared is None or declared and not _text(declared, 128): + raise RuntimeError("BLOB_FORMAT_NOT_ALLOWED") + return BlobUploadInput(name, input.declared_size, declared), extension + + async def _current(self, actor, target, action, input, expected_scope=None): + if (actor is None or not isinstance(target, BlobUploadTarget) + or not _name(target.entity) or not _name(target.field) + or target.entity_id != "" and not _entity_id(target.entity_id)): + raise RuntimeError("BLOB_REFERENCE_INVALID") + try: + scope = self.authorization.scope(actor) + field = self.authorization.field(actor, target) + allowed = await self.authorization.authorize(actor, target, action, input) + except Exception: + raise RuntimeError("BLOB_UPLOAD_DENIED") from None + if (not _scope(scope) or expected_scope is not None and scope != expected_scope + or not isinstance(field, BlobFieldSpec) or field.entity != target.entity + or field.field != target.field or not allowed): + raise RuntimeError("BLOB_UPLOAD_DENIED") + extension = input.file_name.rsplit(".", 1)[-1].lower() + if extension not in field.allowed_extensions: + raise RuntimeError("BLOB_FORMAT_NOT_ALLOWED") + return scope + + async def _transaction(self, work): + try: + tx = await self.catalog.transport.begin_sql() + except Exception: + raise RuntimeError("BLOB_CATALOG_FAILED") from None + committed = False + try: + result = await work(tx) + try: + await tx.commit_sql() + except Exception: + raise RuntimeError("BLOB_CATALOG_FAILED") from None + committed = True + return result + finally: + if not committed: + with suppress(Exception): + await tx.rollback_sql() + + async def begin(self, actor, target: BlobUploadTarget, input: BlobUploadInput, + replay_key: str) -> BlobUploadSession: + normalized, extension = self._normalize(input) + if not _text(replay_key, 256): + raise RuntimeError("BLOB_REFERENCE_INVALID") + scope = await self._current(actor, target, "Begin", normalized) + request = json.dumps([target.entity, target.entity_id, target.field, normalized.file_name, + normalized.declared_size, normalized.declared_content_type], + ensure_ascii=False, separators=(",", ":")).encode("utf-8") + request_hash = sha256(request).hexdigest() + + async def work(tx): + try: + rows = await tx.fetch_all_sql(_query( + "SELECT upload_id,key,request_hash,expires_ms FROM teaql_blob_upload " + "WHERE scope = ? AND replay_key = ?", Value.Text(scope), Value.Text(replay_key))) + except Exception: + raise RuntimeError("BLOB_CATALOG_FAILED") from None + if len(rows) > 1: + raise RuntimeError("BLOB_CATALOG_FAILED") + if rows: + row = rows[0] + if row.get("request_hash") != request_hash: + raise RuntimeError("BLOB_IDEMPOTENCY_CONFLICT") + await self._current(actor, target, "Begin", normalized, scope) + try: + return BlobUploadSession(row["upload_id"], BlobRef(row["key"]), + datetime.fromtimestamp(row["expires_ms"] / 1000, timezone.utc)) + except (KeyError, TypeError, ValueError, OverflowError): + raise RuntimeError("BLOB_CATALOG_FAILED") from None + reference = BlobRef("blob_" + _token()) + upload_id = "bu_" + _token() + object_key = "blob/" + _token() + expires_ms = self.now_ms() + self.lease_seconds * 1000 + await self._current(actor, target, "Begin", normalized, scope) + try: + await tx.execute_sql(_query( + "INSERT INTO teaql_blob_catalog " + "(key,scope,object_key,content_version,file_name,content_type,size_bytes,status) " + "VALUES (?,?,?,?,?,?,?,?)", Value.Text(reference.key), Value.Text(scope), + Value.Text(object_key), Value.Text(""), Value.Text(normalized.file_name), + Value.Text(""), Value.I64(0), Value.Text("initiated"))) + await tx.execute_sql(_query( + "INSERT INTO teaql_blob_upload " + "(upload_id,scope,replay_key,request_hash,key,entity,entity_id,field,file_name," + "extension,declared_size,expires_ms,state) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?)", + Value.Text(upload_id), Value.Text(scope), Value.Text(replay_key), + Value.Text(request_hash), Value.Text(reference.key), Value.Text(target.entity), + Value.Text(target.entity_id), Value.Text(target.field), + Value.Text(normalized.file_name), Value.Text(extension), + Value.I64(normalized.declared_size), Value.I64(expires_ms), + Value.Text("initiated"))) + except Exception: + raise RuntimeError("BLOB_CATALOG_FAILED") from None + return BlobUploadSession(upload_id, reference, + datetime.fromtimestamp(expires_ms / 1000, timezone.utc)) + + return await self._transaction(work) + + async def _row(self, actor, upload_id: str, action: str): + if not isinstance(upload_id, str) or re.fullmatch(r"bu_[A-Za-z0-9_-]{32}", upload_id) is None: + raise RuntimeError("BLOB_UPLOAD_DENIED") + try: + scope = self.authorization.scope(actor) + except Exception: + raise RuntimeError("BLOB_UPLOAD_DENIED") from None + if not _scope(scope): + raise RuntimeError("BLOB_UPLOAD_DENIED") + try: + rows = await self.catalog.transport.fetch_all_sql(_query( + "SELECT upload_id,scope,key,entity,entity_id,field,file_name,extension," + "declared_size,expires_ms,state,written_sha256,written_etag " + "FROM teaql_blob_upload WHERE scope = ? AND upload_id = ?", + Value.Text(scope), Value.Text(upload_id))) + except Exception: + raise RuntimeError("BLOB_CATALOG_FAILED") from None + if len(rows) != 1: + raise RuntimeError("BLOB_UPLOAD_DENIED") + row = rows[0] + try: + target = BlobUploadTarget(row["entity"], row["entity_id"], row["field"]) + input = BlobUploadInput(row["file_name"], row["declared_size"]) + await self._current(actor, target, action, input, scope) + if type(row["expires_ms"]) is not int or self.now_ms() >= row["expires_ms"]: + raise RuntimeError("BLOB_UPLOAD_EXPIRED") + if type(row["declared_size"]) is not int or not 0 <= row["declared_size"] <= self.maximum_bytes: + raise RuntimeError("BLOB_SIZE_EXCEEDED") + if row["extension"] != row["file_name"].rsplit(".", 1)[-1].lower(): + raise RuntimeError("BLOB_CATALOG_FAILED") + except (KeyError, TypeError, AttributeError): + raise RuntimeError("BLOB_CATALOG_FAILED") from None + return row + + async def _object_key(self, actor, row): + try: + rows = await self.catalog.transport.fetch_all_sql(_query( + "SELECT object_key FROM teaql_blob_catalog WHERE key = ? AND scope = ?", + Value.Text(row["key"]), Value.Text(row["scope"]))) + if len(rows) == 1: + return ObjectKey(rows[0]["object_key"]) + except Exception: + pass + raise RuntimeError("BLOB_CATALOG_FAILED") + + async def write(self, actor, upload_id: str, source: BinaryIO) -> int: + if source is None or not callable(getattr(source, "read", None)): + raise RuntimeError("BLOB_UPLOAD_INCOMPLETE") + try: + row = await self._row(actor, upload_id, "Write") + if row["state"] != "initiated": + raise RuntimeError("BLOB_IDEMPOTENCY_CONFLICT") + key = await self._object_key(actor, row) + mime = blob_mime_for_extension(row["extension"]) + reader = _CountingReader(source, row["declared_size"]) + try: + metadata = await asyncio.to_thread( + actor.object_storage("write Blob upload").put, key, reader, + PutOptions(row["declared_size"], WriteMode.CREATE_ONLY, mime)) + except Exception: + raise RuntimeError("BLOB_STORE_FAILED") from None + target = BlobUploadTarget(row["entity"], row["entity_id"], row["field"]) + await self._current(actor, target, "Write", + BlobUploadInput(row["file_name"], row["declared_size"]), row["scope"]) + if self.now_ms() >= row["expires_ms"]: + raise RuntimeError("BLOB_UPLOAD_EXPIRED") + if (not reader.eof or reader.count != row["declared_size"] or + metadata.size != reader.count or not metadata.etag): + raise RuntimeError("BLOB_UPLOAD_INCOMPLETE") + digest = reader.digest.hexdigest() + + async def work(tx): + try: + affected, _ = await tx.execute_sql(_query( + "UPDATE teaql_blob_upload SET state='written', written_sha256=?, written_etag=? " + "WHERE upload_id=? AND scope=? AND state='initiated'", + Value.Text(digest), Value.Text(metadata.etag), + Value.Text(upload_id), Value.Text(row["scope"]))) + except Exception: + raise RuntimeError("BLOB_CATALOG_FAILED") from None + if affected != 1: + raise RuntimeError("BLOB_IDEMPOTENCY_CONFLICT") + await self._transaction(work) + return reader.count + finally: + with suppress(Exception): + source.close() + + async def complete(self, actor, upload_id: str) -> BlobRef: + row = await self._row(actor, upload_id, "Confirm") + if row["state"] != "written" or not row["written_sha256"] or not row["written_etag"]: + raise RuntimeError("BLOB_UPLOAD_INCOMPLETE") + key = await self._object_key(actor, row) + + def verify_stored(): + try: + store = actor.object_storage("verify Blob upload") + before = store.head(key) + if (before is None or before.size != row["declared_size"] + or before.etag != row["written_etag"]): + raise RuntimeError("BLOB_UPLOAD_INCOMPLETE") + with store.get(key) as content: + if (content.metadata.size != before.size or + content.metadata.etag != before.etag): + raise RuntimeError("BLOB_UPLOAD_INCOMPLETE") + reader = _CountingReader(content.stream, self.maximum_bytes) + verified = self.verifier.verify(reader, row["extension"], self.maximum_bytes) + digest = reader.digest.hexdigest() + mime = blob_mime_for_extension(row["extension"]) + if (not isinstance(verified, BlobVerifiedContent) or not reader.eof + or reader.count != before.size or verified.size_bytes != reader.count + or verified.sha256 != digest or verified.sha256 != row["written_sha256"] + or verified.content_type != mime): + raise RuntimeError("BLOB_UPLOAD_INCOMPLETE") + after = store.head(key) + if after is None or after.size != before.size or after.etag != before.etag: + raise RuntimeError("BLOB_UPLOAD_INCOMPLETE") + return verified + except Exception: + raise RuntimeError("BLOB_UPLOAD_INCOMPLETE") from None + + verified = await asyncio.to_thread(verify_stored) + target = BlobUploadTarget(row["entity"], row["entity_id"], row["field"]) + await self._current(actor, target, "Confirm", + BlobUploadInput(row["file_name"], row["declared_size"], + verified.content_type), row["scope"]) + if self.now_ms() >= row["expires_ms"]: + raise RuntimeError("BLOB_UPLOAD_EXPIRED") + + async def work(tx): + try: + affected, _ = await tx.execute_sql(_query( + "UPDATE teaql_blob_upload SET state='completed' WHERE upload_id=? AND scope=? " + "AND state='written' AND written_sha256=? AND written_etag=?", + Value.Text(upload_id), Value.Text(row["scope"]), + Value.Text(row["written_sha256"]), Value.Text(row["written_etag"]))) + if affected != 1: + raise RuntimeError("BLOB_IDEMPOTENCY_CONFLICT") + affected, _ = await tx.execute_sql(_query( + "UPDATE teaql_blob_catalog SET content_version=?,content_type=?,size_bytes=?, " + "status='uploaded_unbound' WHERE key=? AND scope=? AND status='initiated'", + Value.Text(row["written_etag"]), Value.Text(verified.content_type), + Value.I64(verified.size_bytes), Value.Text(row["key"]), + Value.Text(row["scope"]))) + if affected != 1: + raise RuntimeError("BLOB_IDEMPOTENCY_CONFLICT") + except RuntimeError: + raise + except Exception: + raise RuntimeError("BLOB_CATALOG_FAILED") from None + await self._transaction(work) + return BlobRef(row["key"]) diff --git a/src/teaql/core/blob_ref.py b/src/teaql/core/blob_ref.py index 8026f39e..94b69cbd 100644 --- a/src/teaql/core/blob_ref.py +++ b/src/teaql/core/blob_ref.py @@ -16,6 +16,11 @@ } +def blob_mime_for_extension(extension: str) -> str | None: + """Resolve only the frozen Blob format set; never trust client MIME.""" + return _MIME.get(extension) if isinstance(extension, str) else None + + class BlobRef: """A Key identifies catalog state; it never conveys read or bind authority.""" diff --git a/tests/provider/sqlite/test_blob_upload.py b/tests/provider/sqlite/test_blob_upload.py new file mode 100644 index 00000000..5994af09 --- /dev/null +++ b/tests/provider/sqlite/test_blob_upload.py @@ -0,0 +1,158 @@ +"""Real SQLite/File upload confirmation; parser remains test-only.""" + +from hashlib import sha256 +from io import BytesIO +import os + +import aiosqlite +import pytest + +from teaql.core.blob_ref import BlobFieldSpec +from teaql.core.entity import EntityKey +from teaql.core.meta import EntityDescriptor, PropertyDescriptor +from teaql.core.mutation import InsertCommand +from teaql.core.object_storage import ObjectKey, PutOptions, WriteMode +from teaql.core.value import DataType +from teaql.provider.sqlite import SimpleSchemaProvider +from teaql.provider.sqlite.dialect import SqliteDialect +from teaql.provider.sqlite.transport import SqliteTransport +from teaql.runtime import RuntimeModule +from teaql.sql.executor import SqlDataServiceExecutor +from teaql_blob_reference import ( + BlobBindingChange, BlobBindingTarget, BlobUploadInput, BlobUploadTarget, + BlobVerifiedContent, FileObjectStorage, SqliteBlobCatalog, SqliteBlobUploadService, +) + + +class UploadAuthority: + def __init__(self): + self.current_scope = "tenant-a" + self.allowed = True + + def scope(self, _context): + return self.current_scope + + def field(self, _context, _target): + return BlobFieldSpec("Asset", "document", ["pdf"], True, "default") + + async def authorize(self, _context, target, action, _input): + return (self.allowed and target.entity == "Asset" and target.field == "document" + and action in ("Begin", "Write", "Confirm")) + + +class BindingAuthority: + def scope(self, _context): + return "tenant-a" + + async def authorize(self, _context, target, _previous, _next, action): + return target.entity == "Asset" and target.field == "document" and action == "Create" + + +class FixturePDFVerifier: + def __init__(self, early=False): + self.early = early + self.calls = 0 + + def verify(self, stream, extension, maximum_bytes): + self.calls += 1 + if self.early: + return BlobVerifiedContent(1, "0" * 64, "application/pdf") + data = stream.read(maximum_bytes + 1) + assert len(data) <= maximum_bytes and extension == "pdf" + assert data.startswith(b"%PDF-") and b"%%EOF" in data + assert stream.read(1) == b"" + return BlobVerifiedContent(len(data), sha256(data).hexdigest(), "application/pdf") + + +@pytest.mark.asyncio +async def test_upload_real_sqlite_file_verifier_and_native_bind(tmp_path): + path = str(tmp_path / "upload.db") + descriptor = EntityDescriptor("Asset").table_name("asset") + descriptor.property(PropertyDescriptor("id", DataType.I64).is_id()) + descriptor.property(PropertyDescriptor("version", DataType.I64).is_version()) + descriptor.property(PropertyDescriptor("name", DataType.Text)) + schema = SimpleSchemaProvider() + schema.register_entity(descriptor) + context = RuntimeModule.new().entity(descriptor).into_context() + context.insert_resource("dataService", SqlDataServiceExecutor( + SqliteDialect(), SqliteTransport(path), schema)) + provider = FileObjectStorage(tmp_path / "private", os.urandom(32)) + context.with_object_storage(provider).with_object_storage_observer(lambda event: None) + catalog = SqliteBlobCatalog(SqliteTransport(path)) + async with aiosqlite.connect(path) as db: + await db.execute("CREATE TABLE asset (id INTEGER PRIMARY KEY, version INTEGER, name TEXT)") + await db.commit() + await catalog.ensure_schema() + await catalog.validate_schema() + authority = UploadAuthority() + verifier = FixturePDFVerifier() + service = SqliteBlobUploadService(catalog, authority, verifier, 10_000) + target = BlobUploadTarget("Asset", "1", "document") + data = b"%PDF-1.7\nreal file bytes\n%%EOF" + input = BlobUploadInput("path/report.pdf", len(data), "application/pdf") + + session = await service.begin(context, target, input, "request-1") + replay = await service.begin(context, target, input, "request-1") + assert replay.upload_id == session.upload_id + with pytest.raises(RuntimeError, match="BLOB_IDEMPOTENCY_CONFLICT"): + await service.begin(context, target, BlobUploadInput("other.pdf", len(data)), "request-1") + with pytest.raises(RuntimeError, match="BLOB_UPLOAD_INCOMPLETE"): + await service.complete(context, session.upload_id) + authority.current_scope = "tenant-b" + with pytest.raises(RuntimeError, match="BLOB_UPLOAD_DENIED"): + await service.write(context, session.upload_id, BytesIO(data)) + authority.current_scope = "tenant-a" + assert await service.write(context, session.upload_id, BytesIO(data)) == len(data) + async with aiosqlite.connect(path) as db: + async with db.execute("SELECT status FROM teaql_blob_catalog WHERE key=?", + (session.reference.key,)) as cursor: + assert (await cursor.fetchone())[0] == "initiated" + authority.allowed = False + with pytest.raises(RuntimeError, match="BLOB_UPLOAD_DENIED"): + await service.complete(context, session.upload_id) + authority.allowed = True + reference = await service.complete(context, session.upload_id) + assert reference == session.reference and verifier.calls == 1 + + async def bind(graph): + command = (InsertCommand.new("Asset").value("id", 1) + .value("version", 1).value("name", "document")) + await graph.transaction.mutate(graph.context, + graph.request(command, graph.scope(EntityKey("Asset", 1)))) + await catalog.bind_in_graph(graph, graph.context, + BlobBindingChange(BlobBindingTarget("Asset", "1", "document"), + reference, 0, 1, "Create"), BindingAuthority()) + + await context.execute_graph_save(bind, comment="bind verified upload") + async with aiosqlite.connect(path) as db: + async with db.execute("SELECT status FROM teaql_blob_catalog WHERE key=?", + (reference.key,)) as cursor: + assert (await cursor.fetchone())[0] == "available" + + second = await service.begin(context, target, input, "request-2") + await service.write(context, second.upload_id, BytesIO(data)) + async with aiosqlite.connect(path) as db: + async with db.execute("SELECT object_key FROM teaql_blob_catalog WHERE key=?", + (second.reference.key,)) as cursor: + object_key = (await cursor.fetchone())[0] + context.object_storage("replace generation").put_bytes( + ObjectKey(object_key), data, PutOptions(len(data), WriteMode.REPLACE, "application/pdf")) + with pytest.raises(RuntimeError, match="BLOB_UPLOAD_INCOMPLETE"): + await service.complete(context, second.upload_id) + + third = await service.begin(context, target, input, "request-3") + await service.write(context, third.upload_id, BytesIO(data)) + early = SqliteBlobUploadService(catalog, authority, FixturePDFVerifier(early=True), 10_000) + with pytest.raises(RuntimeError, match="BLOB_UPLOAD_INCOMPLETE"): + await early.complete(context, third.upload_id) + + fourth = await service.begin(context, target, input, "request-4") + with pytest.raises(RuntimeError, match="BLOB_STORE_FAILED"): + await service.write(context, fourth.upload_id, BytesIO(data + b"x")) + with pytest.raises(RuntimeError, match="BLOB_UPLOAD_INCOMPLETE"): + await service.complete(context, fourth.upload_id) + fifth = await service.begin(context, target, input, "request-5") + with pytest.raises(RuntimeError, match="BLOB_STORE_FAILED"): + await service.write(context, fifth.upload_id, BytesIO(data[:-1])) + with pytest.raises(RuntimeError, match="BLOB_UPLOAD_INCOMPLETE"): + await service.complete(context, fifth.upload_id) From c7c303a2d863f2dc274ec7a5931aacbbfa3ac02b Mon Sep 17 00:00:00 2001 From: Philip Z Date: Tue, 27 Oct 2026 14:08:56 +0800 Subject: [PATCH 13/75] Add optional Python secure document wire module --- .github/workflows/optional-blob-wheel.yml | 18 +- .../teaql-secure-document-reference/README.md | 5 + .../pyproject.toml | 17 + .../__init__.py | 23 + .../document_round_trip.py | 599 ++++++++++++++++++ .../round_trip_reference.py | 308 +++++++++ .../tests/test_document_round_trip.py | 159 +++++ .../tests/test_round_trip_reference.py | 76 +++ src/teaql/runtime/view_preparation.py | 5 +- 9 files changed, 1207 insertions(+), 3 deletions(-) create mode 100644 packages/teaql-secure-document-reference/README.md create mode 100644 packages/teaql-secure-document-reference/pyproject.toml create mode 100644 packages/teaql-secure-document-reference/src/teaql_secure_document_reference/__init__.py create mode 100644 packages/teaql-secure-document-reference/src/teaql_secure_document_reference/document_round_trip.py create mode 100644 packages/teaql-secure-document-reference/src/teaql_secure_document_reference/round_trip_reference.py create mode 100644 packages/teaql-secure-document-reference/tests/test_document_round_trip.py create mode 100644 packages/teaql-secure-document-reference/tests/test_round_trip_reference.py diff --git a/.github/workflows/optional-blob-wheel.yml b/.github/workflows/optional-blob-wheel.yml index 10b2179e..69fba5a8 100644 --- a/.github/workflows/optional-blob-wheel.yml +++ b/.github/workflows/optional-blob-wheel.yml @@ -1,4 +1,4 @@ -name: Optional Blob wheel +name: Optional reference wheels on: pull_request: push: @@ -16,9 +16,12 @@ jobs: - run: python -m pip install --upgrade build - run: python -m pip install -e ".[test,object-storage-s3]" - run: python -m pip install --no-deps -e ./packages/teaql-blob-reference + - run: python -m pip install --no-deps -e ./packages/teaql-secure-document-reference - run: python -m pytest -q tests/provider/sqlite/test_blob_catalog_graph.py tests/provider/sqlite/test_file_blob_storage.py tests/provider/sqlite/test_blob_upload.py + - run: python -m pytest -q packages/teaql-secure-document-reference/tests - run: python -m build --wheel --no-isolation --outdir .artifacts/base . - run: python -m build --wheel --no-isolation --outdir .artifacts/blob ./packages/teaql-blob-reference + - run: python -m build --wheel --no-isolation --outdir .artifacts/secure ./packages/teaql-secure-document-reference - name: Check wheel isolation run: | python - <<'PY' @@ -27,17 +30,28 @@ jobs: base = list(Path('.artifacts/base').glob('*.whl')) blob = list(Path('.artifacts/blob').glob('*.whl')) - assert len(base) == len(blob) == 1, (base, blob) + secure = list(Path('.artifacts/secure').glob('*.whl')) + assert len(base) == len(blob) == len(secure) == 1, (base, blob, secure) with ZipFile(base[0]) as wheel: entries = wheel.namelist() assert any(name.startswith('teaql/') for name in entries) assert not any(name.startswith('teaql_blob_reference/') for name in entries) + assert not any(name.startswith('teaql_secure_document_reference/') for name in entries) with ZipFile(blob[0]) as wheel: entries = wheel.namelist() assert any(name.startswith('teaql_blob_reference/') for name in entries) assert not any(name.startswith('teaql/') for name in entries) + assert not any(name.startswith('teaql_secure_document_reference/') for name in entries) for module in ('file_storage.py', 'sqlite_catalog.py', 'upload_service.py'): name = f'teaql_blob_reference/{module}' assert name in entries assert wheel.read(name) == (Path('packages/teaql-blob-reference/src') / name).read_bytes() + with ZipFile(secure[0]) as wheel: + entries = wheel.namelist() + assert any(name.startswith('teaql_secure_document_reference/') for name in entries) + assert not any(name.startswith('teaql/') or name.startswith('teaql_blob_reference/') for name in entries) + for module in ('__init__.py', 'round_trip_reference.py', 'document_round_trip.py'): + name = f'teaql_secure_document_reference/{module}' + assert name in entries + assert wheel.read(name) == (Path('packages/teaql-secure-document-reference/src') / name).read_bytes() PY diff --git a/packages/teaql-secure-document-reference/README.md b/packages/teaql-secure-document-reference/README.md new file mode 100644 index 00000000..525ae098 --- /dev/null +++ b/packages/teaql-secure-document-reference/README.md @@ -0,0 +1,5 @@ +# TeaQL secure document reference + +Optional secure reference protocol and document governance package. The base `teaql` runtime does not import it. Installation, key policy and host authorization remain explicit. + +The current source implements governed tqr1 entity references, tqd1/tqd2 document snapshots, and tqs1 Blob selections with the shared Java/Rust wire vectors. `SecureDocumentReferenceModule.install` passively attaches a trusted snapshot contributor to the runtime's governed view preparation. Full document open/accept/replay, transactional selection consumption, and release gates remain in progress. diff --git a/packages/teaql-secure-document-reference/pyproject.toml b/packages/teaql-secure-document-reference/pyproject.toml new file mode 100644 index 00000000..d2ac86b9 --- /dev/null +++ b/packages/teaql-secure-document-reference/pyproject.toml @@ -0,0 +1,17 @@ +[build-system] +requires = ["setuptools>=61.0"] +build-backend = "setuptools.build_meta" + +[project] +name = "teaql-secure-document-reference" +version = "0.3.1" +description = "Optional TeaQL secure document reference module" +readme = "README.md" +requires-python = ">=3.10" +license = "Apache-2.0" +authors = [{ name = "TeaQL" }] +dependencies = ["teaql==0.3.1"] + +[tool.setuptools.packages.find] +where = ["src"] +include = ["teaql_secure_document_reference*"] diff --git a/packages/teaql-secure-document-reference/src/teaql_secure_document_reference/__init__.py b/packages/teaql-secure-document-reference/src/teaql_secure_document_reference/__init__.py new file mode 100644 index 00000000..64400ed6 --- /dev/null +++ b/packages/teaql-secure-document-reference/src/teaql_secure_document_reference/__init__.py @@ -0,0 +1,23 @@ +"""Optional secure-document reference services; no base runtime reverse dependency.""" + +from .round_trip_reference import ( + ContextBoundReferenceRuntime, ReferenceDocumentScope, ReferenceIdentity, + ReferenceKey, RoundTripReferenceError, StaticReferenceKeyProvider, + TrustedReferencePrincipal, VerifiedEntityReference, +) +from .document_round_trip import ( + BlobSelectionClaim, BlobSelectionTarget, BoundDocumentSnapshot, + DocumentBlobBindingSnapshot, DocumentEntitySnapshot, DocumentRelationSnapshot, + DocumentSnapshot, PreparedBoundDocument, SecureDocumentAssembly, + SecureDocumentReferenceModule, VerifiedDocument, +) + +__all__ = [ + "ContextBoundReferenceRuntime", "ReferenceDocumentScope", "ReferenceIdentity", + "ReferenceKey", "RoundTripReferenceError", "StaticReferenceKeyProvider", + "TrustedReferencePrincipal", "VerifiedEntityReference", + "BlobSelectionClaim", "BlobSelectionTarget", "BoundDocumentSnapshot", + "DocumentBlobBindingSnapshot", "DocumentEntitySnapshot", "DocumentRelationSnapshot", + "DocumentSnapshot", "PreparedBoundDocument", "SecureDocumentAssembly", + "SecureDocumentReferenceModule", "VerifiedDocument", +] diff --git a/packages/teaql-secure-document-reference/src/teaql_secure_document_reference/document_round_trip.py b/packages/teaql-secure-document-reference/src/teaql_secure_document_reference/document_round_trip.py new file mode 100644 index 00000000..3a71d7b7 --- /dev/null +++ b/packages/teaql-secure-document-reference/src/teaql_secure_document_reference/document_round_trip.py @@ -0,0 +1,599 @@ +"""Versioned encrypted document snapshots and document-bound Blob selection.""" + +from __future__ import annotations + +import base64 +import binascii +import hmac +import re +import struct +from dataclasses import dataclass +from datetime import datetime, timezone +from typing import Generic, Protocol, TypeVar + +from cryptography.exceptions import InvalidTag +from cryptography.hazmat.primitives.ciphers.aead import AESGCM +from teaql.runtime.capability import install_runtime_capability +from teaql.runtime.view_preparation import PreparedSecureDocument, ViewPreparationError + +from .round_trip_reference import ( + ContextBoundReferenceRuntime, ReferenceDocumentScope, ReferenceIdentity, + ReferenceKey, RoundTripReferenceError, TrustedReferencePrincipal, + _BASE64URL, _KEY_ID, _i64, _u64, +) + +_MUTATIONS = {"UPDATE": 1, "CREATE_CHILD": 2, "REMOVE_CHILD": 3} +_REVERSE_MUTATIONS = {value: key for key, value in _MUTATIONS.items()} +_MAX_I64 = (1 << 63) - 1 +_BLOB_KEY = re.compile(r"blob_[A-Za-z0-9_-]{1,123}\Z", re.ASCII) + + +def _fail(code="ROUND_TRIP_REFERENCE_INVALID"): + raise RoundTripReferenceError(code) + + +def _text(value: str) -> bytes: + if not isinstance(value, str) or not value.strip(): + _fail() + try: + encoded = value.encode("utf-8", errors="strict") + except UnicodeError: + _fail() + if len(encoded) > 16384: + _fail() + return encoded + + +def _str(value: str) -> bytes: + data = _text(value) + return struct.pack(">I", len(data)) + data + + +def _count(value: int, maximum: int) -> bytes: + if type(value) is not int or value < 0 or value > maximum: + _fail("DOCUMENT_PROJECTION_VIOLATION") + return struct.pack(">I", value) + + +def _flag(value: bool) -> bytes: + if type(value) is not bool: + _fail() + return bytes((int(value),)) + + +def _blob_key(value: str) -> str: + if not isinstance(value, str) or not _BLOB_KEY.fullmatch(value): + _fail() + return value + + +def _scope(scope: ReferenceDocumentScope) -> bytes: + scope.validate() + return b"".join((_str(scope.document_id), _str(scope.purpose), _str(scope.aggregate_type), + _u64(scope.aggregate_id), _i64(scope.aggregate_revision))) + + +def _identity(identity: ReferenceIdentity) -> bytes: + identity.validate() + return b"".join((_str(identity.entity_type), _u64(identity.id), _i64(identity.version))) + + +def _set(values: frozenset[str], maximum: int) -> bytes: + if not isinstance(values, frozenset): + _fail("DOCUMENT_PROJECTION_VIOLATION") + return _count(len(values), maximum) + b"".join(_str(value) for value in sorted(values, key=lambda x: x.encode("utf-8"))) + + +def _mutations(values: frozenset[str]) -> bytes: + if not isinstance(values, frozenset) or any(value not in _MUTATIONS for value in values): + _fail("DOCUMENT_PROJECTION_VIOLATION") + return _count(len(values), 3) + bytes(_MUTATIONS[value] for value in sorted(values, key=_MUTATIONS.get)) + + +@dataclass(frozen=True) +class DocumentEntitySnapshot: + row_key: str + identity: ReferenceIdentity + loaded_fields: frozenset[str] + writable_fields: frozenset[str] + allowed_mutations: frozenset[str] + + +@dataclass(frozen=True) +class DocumentRelationSnapshot: + name: str + row_keys: tuple[str, ...] + completeness: str + writable: bool + allowed_mutations: frozenset[str] + + +@dataclass(frozen=True) +class DocumentSnapshot: + model_fingerprint: str + business_id: str + scope: ReferenceDocumentScope + aggregate: ReferenceIdentity + entities: tuple[DocumentEntitySnapshot, ...] + relations: tuple[DocumentRelationSnapshot, ...] + + +@dataclass(frozen=True) +class DocumentBlobBindingSnapshot: + row_key: str + canonical_field: str + entity_type: str + stable_key: str | None + binding_version: int + nullable: bool + writable: bool + + +@dataclass(frozen=True) +class BoundDocumentSnapshot: + snapshot: DocumentSnapshot + bindings: tuple[DocumentBlobBindingSnapshot, ...] + + +@dataclass(frozen=True) +class BlobSelectionTarget: + scope: ReferenceDocumentScope + target_entity: str + canonical_field: str + kind: str # EXISTING_ROW or NEW_CLIENT_KEY + target_key: str + command_id: str + + +@dataclass(frozen=True) +class BlobSelectionClaim: + target: BlobSelectionTarget + stable_key: str + + +T = TypeVar("T") + + +@dataclass(frozen=True) +class VerifiedDocument(Generic[T]): + document: T + key_id: str + issued_at: datetime + expires_at: datetime + + +@dataclass(frozen=True) +class PreparedBoundDocument: + document: BoundDocumentSnapshot + lifetime_seconds: int + + +class SecureDocumentAssembly(Protocol): + def principal(self, context) -> TrustedReferencePrincipal: ... + + async def snapshot(self, context, graph) -> PreparedBoundDocument: ... + + +class _SecureCapability: + capability = "secure_document.v1" + + def __init__(self, contributor) -> None: + self.contributor = contributor + + def validate_ready(self, context) -> None: + if context.get_resource("secure_document_contributor") is not self.contributor: + raise RuntimeError("DOCUMENT_CONFIGURATION") + + +class _SecureContributor: + def __init__(self, module, context, assembly: SecureDocumentAssembly) -> None: + self.module, self.context, self.assembly = module, context, assembly + + async def prepare(self, context, graph, options) -> PreparedSecureDocument: + if context is not self.context or options.mode != "governed": + raise ViewPreparationError("DOCUMENT_CONFIGURATION") + capability = context.get_resource("teaql.runtime.capability:secure_document.v1") + if not isinstance(capability, _SecureCapability): + raise ViewPreparationError("DOCUMENT_CONFIGURATION") + try: + capability.validate_ready(context) + except Exception: + raise ViewPreparationError("DOCUMENT_CONFIGURATION") from None + principal = self.assembly.principal(context) + prepared = await self.assembly.snapshot(context, graph) + document = prepared.document + try: + for slot in graph.blob_slots: + if not slot.entity_id.isascii() or not slot.entity_id.isdecimal() or \ + slot.entity_id.startswith("0"): + raise ViewPreparationError("DOCUMENT_PROJECTION_VIOLATION") + row = next((entity for entity in document.snapshot.entities + if entity.identity.entity_type == slot.entity and + entity.identity.id == int(slot.entity_id)), None) + binding = next((item for item in document.bindings + if item.row_key == (row.row_key if row else None) and + item.canonical_field == slot.field), None) + if binding is None or binding.stable_key != slot.reference.key or \ + binding.binding_version != slot.binding_version: + raise ViewPreparationError("DOCUMENT_PROJECTION_VIOLATION") + token = self.module.issue_bound_document_snapshot(principal, document, + prepared.lifetime_seconds) + verified = self.module.consume_bound_document_snapshot( + principal, token, document.snapshot.scope.aggregate_type, document.snapshot.scope.purpose) + except RoundTripReferenceError as error: + raise ViewPreparationError(error.code) from None + return PreparedSecureDocument(graph, {"businessId": document.snapshot.business_id, + "documentToken": token}, verified.expires_at) + + +class _Reader: + def __init__(self, data: bytes) -> None: + self.data, self.offset = data, 0 + + def take(self, count: int) -> bytes: + if count < 0 or self.offset + count > len(self.data): + _fail() + value = self.data[self.offset:self.offset + count] + self.offset += count + return value + + def byte(self) -> int: + return self.take(1)[0] + + def flag(self) -> bool: + value = self.byte() + if value not in (0, 1): + _fail() + return value == 1 + + def count(self, maximum: int) -> int: + value = struct.unpack(">I", self.take(4))[0] + if value > maximum: + _fail() + return value + + def text(self) -> str: + value = self.take(self.count(16384)) + try: + result = value.decode("utf-8", errors="strict") + except UnicodeError: + _fail() + _text(result) + return result + + def u64(self) -> int: + return struct.unpack(">Q", self.take(8))[0] + + def i64(self) -> int: + return struct.unpack(">q", self.take(8))[0] + + def end(self) -> None: + if self.offset != len(self.data): + _fail() + + +def _read_scope(reader: _Reader) -> ReferenceDocumentScope: + value = ReferenceDocumentScope(reader.text(), reader.text(), reader.text(), reader.u64(), reader.i64()) + value.validate() + return value + + +def _read_identity(reader: _Reader) -> ReferenceIdentity: + value = ReferenceIdentity(reader.text(), reader.u64(), reader.i64()) + value.validate() + return value + + +def _read_set(reader: _Reader, maximum: int) -> frozenset[str]: + count = reader.count(maximum) + values = [reader.text() for _ in range(count)] + if len(set(values)) != count: + _fail() + return frozenset(values) + + +def _read_mutations(reader: _Reader) -> frozenset[str]: + count = reader.count(3) + values = [] + for _ in range(count): + value = _REVERSE_MUTATIONS.get(reader.byte()) + if value is None or value in values: + _fail() + values.append(value) + return frozenset(values) + + +def _encode_snapshot(snapshot: DocumentSnapshot, version: int, actor: bytes, + issued: int, expires: int) -> bytes: + if snapshot.aggregate.entity_type != snapshot.scope.aggregate_type or \ + snapshot.aggregate.id != snapshot.scope.aggregate_id or \ + snapshot.aggregate.version != snapshot.scope.aggregate_revision: + _fail("DOCUMENT_PROJECTION_VIOLATION") + parts = [bytes((version,)), actor, _i64(issued), _i64(expires), _str(snapshot.model_fingerprint), + _str(snapshot.business_id), _scope(snapshot.scope), _identity(snapshot.aggregate), + _count(len(snapshot.entities), 10000)] + row_keys = set() + for entity in snapshot.entities: + if entity.row_key in row_keys or not entity.writable_fields.issubset(entity.loaded_fields) or \ + not entity.allowed_mutations: + _fail("DOCUMENT_PROJECTION_VIOLATION") + row_keys.add(entity.row_key) + parts.extend((_str(entity.row_key), _identity(entity.identity), + _set(entity.loaded_fields, 1024), _set(entity.writable_fields, 1024), + _mutations(entity.allowed_mutations))) + parts.append(_count(len(snapshot.relations), 1024)) + for relation in snapshot.relations: + if relation.completeness not in ("COMPLETE", "PARTIAL") or \ + (not relation.writable and relation.allowed_mutations) or \ + len(set(relation.row_keys)) != len(relation.row_keys) or \ + any(key not in row_keys for key in relation.row_keys): + _fail("DOCUMENT_PROJECTION_VIOLATION") + parts.extend((_str(relation.name), bytes((1 if relation.completeness == "COMPLETE" else 2,)), + _flag(relation.writable), _mutations(relation.allowed_mutations), + _count(len(relation.row_keys), 10000), + *(_str(key) for key in relation.row_keys))) + return b"".join(parts) + + +def _decode_snapshot(reader: _Reader, version: int): + if reader.byte() != version: + _fail() + actor, issued, expires = reader.take(32), reader.i64(), reader.i64() + model_fingerprint, business_id = reader.text(), reader.text() + scope, aggregate = _read_scope(reader), _read_identity(reader) + entities = [] + for _ in range(reader.count(10000)): + entities.append(DocumentEntitySnapshot(reader.text(), _read_identity(reader), + _read_set(reader, 1024), _read_set(reader, 1024), + _read_mutations(reader))) + relations = [] + for _ in range(reader.count(1024)): + name, completeness = reader.text(), reader.byte() + if completeness not in (1, 2): + _fail() + writable, mutations = reader.flag(), _read_mutations(reader) + rows = tuple(reader.text() for _ in range(reader.count(10000))) + relations.append(DocumentRelationSnapshot(name, rows, + "COMPLETE" if completeness == 1 else "PARTIAL", + writable, mutations)) + snapshot = DocumentSnapshot(model_fingerprint, business_id, scope, aggregate, + tuple(entities), tuple(relations)) + _encode_snapshot(snapshot, version, actor, issued, expires) + return actor, issued, expires, snapshot + + +def _bindings(document: BoundDocumentSnapshot) -> tuple[DocumentBlobBindingSnapshot, ...]: + values = tuple(sorted(document.bindings, key=lambda x: (x.row_key.encode(), x.canonical_field.encode()))) + if len(values) > 10000: + _fail("DOCUMENT_PROJECTION_VIOLATION") + previous = None + for binding in values: + for value in (binding.row_key, binding.canonical_field, binding.entity_type): + _text(value) + if value != value.strip(): + _fail("DOCUMENT_PROJECTION_VIOLATION") + pair = (binding.row_key, binding.canonical_field) + if pair == previous: + _fail("DOCUMENT_PROJECTION_VIOLATION") + previous = pair + entity = next((row for row in document.snapshot.entities if row.row_key == binding.row_key), None) + if entity is None or entity.identity.entity_type != binding.entity_type or \ + binding.canonical_field not in entity.loaded_fields or \ + (binding.writable and binding.canonical_field not in entity.writable_fields): + _fail("DOCUMENT_PROJECTION_VIOLATION") + if binding.stable_key is not None: + _blob_key(binding.stable_key) + _u64(binding.binding_version) + _flag(binding.nullable) + _flag(binding.writable) + return values + + +def _target_allowed(document: BoundDocumentSnapshot, target: BlobSelectionTarget) -> None: + target.scope.validate() + for value in (target.target_entity, target.canonical_field, target.target_key, target.command_id): + _text(value) + if value != value.strip(): + _fail() + if document.snapshot.scope != target.scope: + _fail("ROUND_TRIP_REFERENCE_SCOPE_MISMATCH") + if target.kind == "EXISTING_ROW": + entity = next((row for row in document.snapshot.entities if row.row_key == target.target_key), None) + if entity is None or entity.identity.entity_type != target.target_entity or \ + target.canonical_field not in entity.writable_fields or \ + not any(binding.row_key == target.target_key and + binding.canonical_field == target.canonical_field and binding.writable + for binding in document.bindings): + _fail("DOCUMENT_PROJECTION_VIOLATION") + elif target.kind == "NEW_CLIENT_KEY": + if not any(relation.completeness == "COMPLETE" and relation.writable and + "CREATE_CHILD" in relation.allowed_mutations for relation in document.snapshot.relations): + _fail("DOCUMENT_PROJECTION_VIOLATION") + else: + _fail() + + +class SecureDocumentReferenceModule(ContextBoundReferenceRuntime): + """Protocol implementation. Host document open/accept and SQL replay remain separate gates.""" + + def install(self, context, assembly: SecureDocumentAssembly) -> None: + if context is None or assembly is None or not callable(getattr(assembly, "principal", None)) or \ + not callable(getattr(assembly, "snapshot", None)) or \ + context.get_resource("secure_document_contributor") is not None or \ + context.get_resource("teaql.runtime.capability:secure_document.v1") is not None: + raise RuntimeError("DOCUMENT_CONFIGURATION") + contributor = _SecureContributor(self, context, assembly) + install_runtime_capability(context, _SecureCapability(contributor)) + context.insert_resource("secure_document_contributor", contributor) + + def _seal(self, prefix: str, label: bytes, purpose: str, plaintext: bytes, + key: ReferenceKey) -> str: + nonce = self.nonce() + if not isinstance(nonce, bytes) or len(nonce) != 12: + _fail("ROUND_TRIP_REFERENCE_CONFIGURATION_INVALID") + aad = b"".join(_str(value) for value in (prefix, key.key_id, self.service, self.environment, purpose)) + encrypted = AESGCM(self._derive(key, label)).encrypt(nonce, plaintext, aad) + token = f"{prefix}.{key.key_id}." + base64.urlsafe_b64encode(nonce + encrypted).rstrip(b"=").decode() + if len(token) > 256000: + _fail() + return token + + def _open(self, prefix: str, label: bytes, purpose: str, token: str) -> tuple[ReferenceKey, bytes]: + if not isinstance(token, str) or len(token) > 256000: + _fail() + parts = token.split(".") + if len(parts) != 3 or parts[0] != prefix or not _KEY_ID.fullmatch(parts[1]) or \ + not _BASE64URL.fullmatch(parts[2]): + _fail() + key = self.keys.key_by_id(parts[1]) + if key is None: + _fail() + try: + envelope = base64.urlsafe_b64decode(parts[2] + "=" * (-len(parts[2]) % 4)) + except (ValueError, binascii.Error): + _fail() + if len(envelope) < 28 or base64.urlsafe_b64encode(envelope).rstrip(b"=").decode() != parts[2]: + _fail() + aad = b"".join(_str(value) for value in (prefix, key.key_id, self.service, self.environment, purpose)) + try: + return key, AESGCM(self._derive(key, label)).decrypt(envelope[:12], envelope[12:], aad) + except InvalidTag: + _fail() + + def _times(self, lifetime_seconds: int) -> tuple[int, int]: + if type(lifetime_seconds) is not int or lifetime_seconds <= 0: + _fail() + issued = self._timestamp() + expires = issued + lifetime_seconds + if expires > _MAX_I64: + _fail() + return issued, expires + + def _authorize_snapshot(self, principal: TrustedReferencePrincipal, + snapshot: DocumentSnapshot) -> None: + self._authorize(principal, snapshot.scope, snapshot.aggregate) + for entity in snapshot.entities: + self._authorize(principal, snapshot.scope, entity.identity) + + def _verify(self, principal: TrustedReferencePrincipal, key: ReferenceKey, actor: bytes, + issued: int, expires: int, scope: ReferenceDocumentScope, + aggregate_type: str, purpose: str, document: T) -> VerifiedDocument[T]: + now = self._timestamp() + if expires <= now: + _fail("ROUND_TRIP_REFERENCE_EXPIRED") + if issued > now + 60: + _fail() + if not hmac.compare_digest(actor, self._actor(key, principal)) or \ + scope.purpose != purpose or scope.aggregate_type != aggregate_type: + _fail("ROUND_TRIP_REFERENCE_SCOPE_MISMATCH") + try: + return VerifiedDocument(document, key.key_id, + datetime.fromtimestamp(issued, timezone.utc), + datetime.fromtimestamp(expires, timezone.utc)) + except (OverflowError, OSError, ValueError): + _fail() + + def issue_document_snapshot(self, principal: TrustedReferencePrincipal, + snapshot: DocumentSnapshot, lifetime_seconds: int) -> str: + issued, expires = self._times(lifetime_seconds) + key = self.keys.current_key() + plaintext = _encode_snapshot(snapshot, 1, self._actor(key, principal), issued, expires) + self._authorize_snapshot(principal, snapshot) + return self._seal("tqd1", b"document-snapshot-encryption", snapshot.scope.purpose, plaintext, key) + + def consume_document_snapshot(self, principal: TrustedReferencePrincipal, token: str, + aggregate_type: str, purpose: str) -> VerifiedDocument[DocumentSnapshot]: + key, plaintext = self._open("tqd1", b"document-snapshot-encryption", purpose, token) + reader = _Reader(plaintext) + actor, issued, expires, snapshot = _decode_snapshot(reader, 1) + reader.end() + verified = self._verify(principal, key, actor, issued, expires, snapshot.scope, + aggregate_type, purpose, snapshot) + self._authorize_snapshot(principal, snapshot) + return verified + + def issue_bound_document_snapshot(self, principal: TrustedReferencePrincipal, + document: BoundDocumentSnapshot, lifetime_seconds: int) -> str: + issued, expires = self._times(lifetime_seconds) + key = self.keys.current_key() + bindings = _bindings(document) + parts = [_encode_snapshot(document.snapshot, 2, self._actor(key, principal), issued, expires), + _count(len(bindings), 10000)] + for binding in bindings: + parts.extend((_str(binding.row_key), _str(binding.canonical_field), _str(binding.entity_type), + _flag(binding.stable_key is not None))) + if binding.stable_key is not None: + parts.append(_str(binding.stable_key)) + parts.extend((_u64(binding.binding_version), _flag(binding.nullable), _flag(binding.writable))) + self._authorize_snapshot(principal, document.snapshot) + return self._seal("tqd2", b"document-snapshot-v2-encryption", document.snapshot.scope.purpose, + b"".join(parts), key) + + def consume_bound_document_snapshot(self, principal: TrustedReferencePrincipal, token: str, + aggregate_type: str, purpose: str) -> VerifiedDocument[BoundDocumentSnapshot]: + key, plaintext = self._open("tqd2", b"document-snapshot-v2-encryption", purpose, token) + reader = _Reader(plaintext) + actor, issued, expires, snapshot = _decode_snapshot(reader, 2) + bindings = [] + for _ in range(reader.count(10000)): + row_key, field, entity_type = reader.text(), reader.text(), reader.text() + stable_key = reader.text() if reader.flag() else None + bindings.append(DocumentBlobBindingSnapshot(row_key, field, entity_type, stable_key, + reader.u64(), reader.flag(), reader.flag())) + reader.end() + document = BoundDocumentSnapshot(snapshot, tuple(bindings)) + if document.bindings != _bindings(document): + _fail() + verified = self._verify(principal, key, actor, issued, expires, snapshot.scope, + aggregate_type, purpose, document) + self._authorize_snapshot(principal, snapshot) + return verified + + def issue_blob_selection(self, principal: TrustedReferencePrincipal, document_token: str, + claim: BlobSelectionClaim, lifetime_seconds: int) -> str: + document = self.consume_bound_document_snapshot(principal, document_token, + claim.target.scope.aggregate_type, + claim.target.scope.purpose) + _target_allowed(document.document, claim.target) + _blob_key(claim.stable_key) + issued, expires = self._times(lifetime_seconds) + if expires > int(document.expires_at.timestamp()): + _fail() + key = self.keys.current_key() + payload = b"".join((b"\x01", self._actor(key, principal), _i64(issued), _i64(expires), + _scope(claim.target.scope), _str(claim.target.target_entity), + _str(claim.target.canonical_field), + bytes((1 if claim.target.kind == "EXISTING_ROW" else 2,)), + _str(claim.target.target_key), _str(claim.stable_key), + _str(claim.target.command_id))) + return self._seal("tqs1", b"blob-selection-v1-encryption", claim.target.scope.purpose, + payload, key) + + def consume_blob_selection(self, principal: TrustedReferencePrincipal, document_token: str, + token: str, expected_target: BlobSelectionTarget) -> VerifiedDocument[BlobSelectionClaim]: + document = self.consume_bound_document_snapshot(principal, document_token, + expected_target.scope.aggregate_type, + expected_target.scope.purpose) + _target_allowed(document.document, expected_target) + key, plaintext = self._open("tqs1", b"blob-selection-v1-encryption", + expected_target.scope.purpose, token) + reader = _Reader(plaintext) + if reader.byte() != 1: + _fail() + actor, issued, expires = reader.take(32), reader.i64(), reader.i64() + scope, target_entity, field = _read_scope(reader), reader.text(), reader.text() + kind_byte = reader.byte() + if kind_byte not in (1, 2): + _fail() + target_key, stable_key, command_id = reader.text(), _blob_key(reader.text()), reader.text() + reader.end() + target = BlobSelectionTarget(scope, target_entity, field, + "EXISTING_ROW" if kind_byte == 1 else "NEW_CLIENT_KEY", + target_key, command_id) + if target != expected_target: + _fail("ROUND_TRIP_REFERENCE_SCOPE_MISMATCH") + if expires > int(document.expires_at.timestamp()): + _fail() + return self._verify(principal, key, actor, issued, expires, scope, + expected_target.scope.aggregate_type, expected_target.scope.purpose, + BlobSelectionClaim(target, stable_key)) diff --git a/packages/teaql-secure-document-reference/src/teaql_secure_document_reference/round_trip_reference.py b/packages/teaql-secure-document-reference/src/teaql_secure_document_reference/round_trip_reference.py new file mode 100644 index 00000000..247e7c4a --- /dev/null +++ b/packages/teaql-secure-document-reference/src/teaql_secure_document_reference/round_trip_reference.py @@ -0,0 +1,308 @@ +"""Governed tqr1 entity references compatible with the Java/Rust wire contract.""" + +from __future__ import annotations + +import base64 +import binascii +import hmac +import math +import os +import re +import struct +from dataclasses import dataclass +from datetime import datetime, timezone +from hashlib import sha256 +from typing import Callable + +from cryptography.exceptions import InvalidTag +from cryptography.hazmat.primitives import hashes +from cryptography.hazmat.primitives.ciphers.aead import AESGCM +from cryptography.hazmat.primitives.kdf.hkdf import HKDF + + +_SALT = b"teaql.round-trip-reference.v1" +_KEY_ID = re.compile(r"[A-Za-z0-9_-]{1,64}\Z", re.ASCII) +_BASE64URL = re.compile(r"[A-Za-z0-9_-]+\Z", re.ASCII) +_MAX_U64 = (1 << 64) - 1 +_MAX_I64 = (1 << 63) - 1 + + +class RoundTripReferenceError(ValueError): + def __init__(self, code: str) -> None: + self.code = code + super().__init__(code) + + +def _fail(code: str = "ROUND_TRIP_REFERENCE_INVALID"): + raise RoundTripReferenceError(code) + + +def _text(value: str) -> bytes: + if not isinstance(value, str) or not value or value != value.strip(): + _fail() + try: + encoded = value.encode("utf-8", errors="strict") + except UnicodeError: + _fail() + if len(encoded) > 4096: + _fail() + return encoded + + +def _sized(value: bytes) -> bytes: + if len(value) > 65535: + _fail() + return struct.pack(">H", len(value)) + value + + +def _string(value: str) -> bytes: + return _sized(_text(value)) + + +def _u64(value: int, positive: bool = False) -> bytes: + if type(value) is not int or value < (1 if positive else 0) or value > _MAX_U64: + _fail() + return struct.pack(">Q", value) + + +def _i64(value: int, nonnegative: bool = False) -> bytes: + if type(value) is not int or value < (0 if nonnegative else -1 << 63) or value > _MAX_I64: + _fail() + return struct.pack(">q", value) + + +@dataclass(frozen=True) +class TrustedReferencePrincipal: + authentication_realm: str + subject: str + domain_root_type: str + domain_root_id: int + + def wire(self) -> bytes: + return b"".join((_string(self.authentication_realm), _string(self.subject), + _string(self.domain_root_type), _u64(self.domain_root_id, True))) + + +@dataclass(frozen=True) +class ReferenceDocumentScope: + document_id: str + purpose: str + aggregate_type: str + aggregate_id: int + aggregate_revision: int + + def validate(self) -> None: + _text(self.document_id) + _text(self.purpose) + _text(self.aggregate_type) + _u64(self.aggregate_id, True) + _i64(self.aggregate_revision, True) + + +@dataclass(frozen=True) +class ReferenceIdentity: + entity_type: str + id: int + version: int + + def validate(self) -> None: + _text(self.entity_type) + _u64(self.id, True) + _i64(self.version, True) + + +@dataclass(frozen=True) +class ReferenceKey: + key_id: str + master_key: bytes + + def __post_init__(self) -> None: + if not isinstance(self.key_id, str) or not _KEY_ID.fullmatch(self.key_id) or \ + not isinstance(self.master_key, bytes) or len(self.master_key) != 32: + _fail("ROUND_TRIP_REFERENCE_CONFIGURATION_INVALID") + + def __repr__(self) -> str: + return f"ReferenceKey(key_id={self.key_id!r}, master_key=[REDACTED])" + + +class StaticReferenceKeyProvider: + def __init__(self, current_key_id: str, keys: tuple[ReferenceKey, ...]) -> None: + if not isinstance(current_key_id, str) or not _KEY_ID.fullmatch(current_key_id): + _fail("ROUND_TRIP_REFERENCE_CONFIGURATION_INVALID") + retained: dict[str, ReferenceKey] = {} + for key in keys: + if not isinstance(key, ReferenceKey) or key.key_id in retained: + _fail("ROUND_TRIP_REFERENCE_CONFIGURATION_INVALID") + retained[key.key_id] = key + if current_key_id not in retained: + _fail("ROUND_TRIP_REFERENCE_CONFIGURATION_INVALID") + self._keys = retained + self._current = current_key_id + + def current_key(self) -> ReferenceKey: + return self._keys[self._current] + + def key_by_id(self, key_id: str) -> ReferenceKey | None: + return self._keys.get(key_id) + + +@dataclass(frozen=True) +class VerifiedEntityReference: + identity: ReferenceIdentity + key_id: str + issued_at: datetime + expires_at: datetime + + +class _Reader: + def __init__(self, data: bytes) -> None: + self.data = data + self.offset = 0 + + def take(self, count: int) -> bytes: + if count < 0 or self.offset + count > len(self.data): + _fail() + result = self.data[self.offset:self.offset + count] + self.offset += count + return result + + def text(self) -> str: + length = struct.unpack(">H", self.take(2))[0] + try: + result = self.take(length).decode("utf-8", errors="strict") + except UnicodeError: + _fail() + _text(result) + return result + + def u64(self) -> int: + return struct.unpack(">Q", self.take(8))[0] + + def i64(self) -> int: + return struct.unpack(">q", self.take(8))[0] + + def end(self) -> None: + if self.offset != len(self.data): + _fail() + + +class ContextBoundReferenceRuntime: + def __init__(self, service: str, environment: str, keys: StaticReferenceKeyProvider, + authorize: Callable[[TrustedReferencePrincipal, ReferenceDocumentScope, + ReferenceIdentity], None], + now: Callable[[], datetime] | None = None, + nonce: Callable[[], bytes] | None = None) -> None: + _text(service) + _text(environment) + if not keys or not callable(authorize): + _fail("ROUND_TRIP_REFERENCE_CONFIGURATION_INVALID") + keys.current_key() + self.service, self.environment = service, environment + self.keys, self.authorize = keys, authorize + self.now = now or (lambda: datetime.now(timezone.utc)) + self.nonce = nonce or (lambda: os.urandom(12)) + + def _timestamp(self) -> int: + value = self.now() + if not isinstance(value, datetime) or value.tzinfo is None: + _fail("ROUND_TRIP_REFERENCE_CONFIGURATION_INVALID") + return math.floor(value.timestamp()) + + def _derive(self, key: ReferenceKey, label: bytes) -> bytes: + info = b"".join((_sized(label), _string(self.service), _string(self.environment), + _string(key.key_id))) + return HKDF(algorithm=hashes.SHA256(), + length=32, salt=_SALT, info=info).derive(key.master_key) + + def _actor(self, key: ReferenceKey, principal: TrustedReferencePrincipal) -> bytes: + return hmac.new(self._derive(key, b"actor-fingerprint"), principal.wire(), sha256).digest() + + def _aad(self, key: ReferenceKey, purpose: str) -> bytes: + return b"".join(_string(value) for value in + ("tqr1", key.key_id, self.service, self.environment, purpose)) + + def _authorize(self, principal: TrustedReferencePrincipal, scope: ReferenceDocumentScope, + identity: ReferenceIdentity) -> None: + try: + self.authorize(principal, scope, identity) + except Exception: + _fail("ROUND_TRIP_REFERENCE_AUTHORIZATION_REQUIRED") + + def issue(self, principal: TrustedReferencePrincipal, scope: ReferenceDocumentScope, + identity: ReferenceIdentity, lifetime_seconds: int) -> str: + principal.wire() + scope.validate() + identity.validate() + if type(lifetime_seconds) is not int or lifetime_seconds <= 0: + _fail() + self._authorize(principal, scope, identity) + key = self.keys.current_key() + issued = self._timestamp() + expires = issued + lifetime_seconds + _i64(expires) + plaintext = b"".join((self._actor(key, principal), _string(identity.entity_type), + _u64(identity.id), _i64(identity.version), + _string(scope.document_id), _string(scope.aggregate_type), + _u64(scope.aggregate_id), _i64(scope.aggregate_revision), + _i64(issued), _i64(expires))) + nonce = self.nonce() + if not isinstance(nonce, bytes) or len(nonce) != 12: + _fail("ROUND_TRIP_REFERENCE_CONFIGURATION_INVALID") + encrypted = AESGCM(self._derive(key, b"entity-reference-encryption")).encrypt( + nonce, plaintext, self._aad(key, scope.purpose)) + body = base64.urlsafe_b64encode(nonce + encrypted).rstrip(b"=").decode("ascii") + token = f"tqr1.{key.key_id}.{body}" + if len(token) > 24000: + _fail() + return token + + def consume(self, principal: TrustedReferencePrincipal, scope: ReferenceDocumentScope, + token: str, expected_entity_type: str) -> VerifiedEntityReference: + principal.wire() + scope.validate() + _text(expected_entity_type) + if not isinstance(token, str) or len(token) > 24000: + _fail() + parts = token.split(".") + if len(parts) != 3 or parts[0] != "tqr1" or not _KEY_ID.fullmatch(parts[1]) or \ + not _BASE64URL.fullmatch(parts[2]): + _fail() + key = self.keys.key_by_id(parts[1]) + if key is None: + _fail() + try: + envelope = base64.urlsafe_b64decode(parts[2] + "=" * (-len(parts[2]) % 4)) + except (ValueError, binascii.Error): + _fail() + if len(envelope) < 28 or base64.urlsafe_b64encode(envelope).rstrip(b"=").decode() != parts[2]: + _fail() + try: + plaintext = AESGCM(self._derive(key, b"entity-reference-encryption")).decrypt( + envelope[:12], envelope[12:], self._aad(key, scope.purpose)) + except InvalidTag: + _fail() + reader = _Reader(plaintext) + actor = reader.take(32) + identity = ReferenceIdentity(reader.text(), reader.u64(), reader.i64()) + document_id, aggregate_type = reader.text(), reader.text() + aggregate_id, aggregate_revision = reader.u64(), reader.i64() + issued, expires = reader.i64(), reader.i64() + reader.end() + identity.validate() + now = self._timestamp() + if expires <= now: + _fail("ROUND_TRIP_REFERENCE_EXPIRED") + if issued > now + 60: + _fail() + if not hmac.compare_digest(actor, self._actor(key, principal)) or \ + document_id != scope.document_id or aggregate_type != scope.aggregate_type or \ + aggregate_id != scope.aggregate_id or aggregate_revision != scope.aggregate_revision or \ + identity.entity_type != expected_entity_type: + _fail("ROUND_TRIP_REFERENCE_SCOPE_MISMATCH") + self._authorize(principal, scope, identity) + try: + issued_at = datetime.fromtimestamp(issued, timezone.utc) + expires_at = datetime.fromtimestamp(expires, timezone.utc) + except (OverflowError, OSError, ValueError): + _fail() + return VerifiedEntityReference(identity, key.key_id, issued_at, expires_at) diff --git a/packages/teaql-secure-document-reference/tests/test_document_round_trip.py b/packages/teaql-secure-document-reference/tests/test_document_round_trip.py new file mode 100644 index 00000000..6789ef7c --- /dev/null +++ b/packages/teaql-secure-document-reference/tests/test_document_round_trip.py @@ -0,0 +1,159 @@ +from dataclasses import replace +from datetime import datetime, timezone +from hashlib import sha256 +from datetime import timedelta + +import pytest + +from teaql_secure_document_reference import ( + BlobSelectionClaim, BlobSelectionTarget, BoundDocumentSnapshot, + DocumentBlobBindingSnapshot, DocumentEntitySnapshot, DocumentRelationSnapshot, + DocumentSnapshot, ReferenceDocumentScope, ReferenceIdentity, ReferenceKey, + RoundTripReferenceError, SecureDocumentReferenceModule, StaticReferenceKeyProvider, + TrustedReferencePrincipal, PreparedBoundDocument, +) +from teaql.core.blob_ref import BlobRef +from teaql.runtime.context import UserContext +from teaql.runtime.capability import validate_capabilities +from teaql.runtime.view_preparation import ( + AuthorizedBlobSlot, AuthorizedResponseGraph, ProjectedBlobURL, + ViewOptions, ViewPreparationError, ViewPreparationService, +) + + +NOW = datetime.fromtimestamp(1800000000, timezone.utc) +ALICE = TrustedReferencePrincipal("oidc", "alice", "Merchant", 7) +SCOPE = ReferenceDocumentScope("doc-1", "edit-order", "Order", 10, 4) +SNAPSHOT = DocumentSnapshot("model-v1", "ORD-X1", SCOPE, + ReferenceIdentity("Order", 10, 4), + (DocumentEntitySnapshot("item:0", ReferenceIdentity("OrderItem", 20, 2), + frozenset(("attachment", "product_code", "quantity")), + frozenset(("attachment", "quantity")), + frozenset(("UPDATE",))),), + (DocumentRelationSnapshot("items", ("item:0",), "COMPLETE", True, + frozenset(("CREATE_CHILD", "REMOVE_CHILD"))),)) +DOCUMENT = BoundDocumentSnapshot(SNAPSHOT, + (DocumentBlobBindingSnapshot("item:0", "attachment", "OrderItem", + "blob_A", (1 << 64) - 1, True, True),)) +TARGET = BlobSelectionTarget(SCOPE, "OrderItem", "attachment", "EXISTING_ROW", "item:0", "command-1") +CLAIM = BlobSelectionClaim(TARGET, "blob_B") +KEYS = StaticReferenceKeyProvider("k1", (ReferenceKey("k1", bytes([0x42]) * 32),)) + + +def runtime(*, now=NOW, allowed=True, environment="test"): + def authorize(*_args): + if not allowed: + raise PermissionError("revoked") + return SecureDocumentReferenceModule("order-service", environment, KEYS, authorize, + lambda: now, lambda: bytes([0x33]) * 12) + + +def code(call): + with pytest.raises(RoundTripReferenceError) as error: + call() + return error.value.code + + +def test_shared_tqd2_tqs1_vectors_and_current_authorization(): + module = runtime() + token = module.issue_bound_document_snapshot(ALICE, DOCUMENT, 300) + assert sha256(token.encode()).hexdigest() == \ + "e7c81ca089b0a3e0585faefcc275aeb44b456d666d9a4a17949c92b1891089d7" + verified = module.consume_bound_document_snapshot(ALICE, token, "Order", "edit-order") + assert verified.document == DOCUMENT + selection = module.issue_blob_selection(ALICE, token, CLAIM, 120) + assert sha256(selection.encode()).hexdigest() == \ + "31303e3e251dd0e8963bf2a4d8897d1eccde63a485aaf287f88f29d40757ba96" + assert module.consume_blob_selection(ALICE, token, selection, TARGET).document == CLAIM + assert code(lambda: runtime(allowed=False).consume_bound_document_snapshot( + ALICE, token, "Order", "edit-order")) == "ROUND_TRIP_REFERENCE_AUTHORIZATION_REQUIRED" + + +def test_legacy_tqd1_rust_digest_and_v2_separation(): + legacy_entity = replace(SNAPSHOT.entities[0], + loaded_fields=frozenset(("product_code", "quantity")), + writable_fields=frozenset(("quantity",))) + legacy = replace(SNAPSHOT, entities=(legacy_entity,)) + token = runtime().issue_document_snapshot(ALICE, legacy, 600) + assert sha256(token.encode()).hexdigest() == \ + "69fee1ec01cd4a5b69dd3f795aeaffb3a3546e50d48e25cfc9b8f169a527b267" + assert runtime().consume_document_snapshot(ALICE, token, "Order", "edit-order").document == legacy + assert code(lambda: runtime().consume_bound_document_snapshot(ALICE, token, "Order", "edit-order")) == \ + "ROUND_TRIP_REFERENCE_INVALID" + + +def test_tamper_wrong_actor_context_target_and_expiry(): + token = runtime().issue_bound_document_snapshot(ALICE, DOCUMENT, 300) + selection = runtime().issue_blob_selection(ALICE, token, CLAIM, 120) + assert code(lambda: runtime().consume_bound_document_snapshot( + replace(ALICE, subject="bob"), token, "Order", "edit-order")) == \ + "ROUND_TRIP_REFERENCE_SCOPE_MISMATCH" + assert code(lambda: runtime(environment="other").consume_bound_document_snapshot( + ALICE, token, "Order", "edit-order")) == "ROUND_TRIP_REFERENCE_INVALID" + assert code(lambda: runtime().consume_blob_selection( + ALICE, token, selection, replace(TARGET, command_id="other"))) == \ + "ROUND_TRIP_REFERENCE_SCOPE_MISMATCH" + assert code(lambda: runtime(now=datetime.fromtimestamp(1800000120, timezone.utc)).consume_blob_selection( + ALICE, token, selection, TARGET)) == "ROUND_TRIP_REFERENCE_EXPIRED" + assert code(lambda: runtime().consume_blob_selection(ALICE, token, selection + "A", TARGET)) == \ + "ROUND_TRIP_REFERENCE_INVALID" + assert code(lambda: runtime().consume_blob_selection(ALICE, token, "blob_B", TARGET)) == \ + "ROUND_TRIP_REFERENCE_INVALID" + assert code(lambda: runtime().issue_blob_selection(ALICE, token, + replace(CLAIM, stable_key="https://example.test/file"), 120)) == \ + "ROUND_TRIP_REFERENCE_INVALID" + + +@pytest.mark.asyncio +async def test_passive_install_prepares_governed_view_and_checks_blob_binding(): + now = datetime.now(timezone.utc) + module = runtime(now=now) + bound = replace(DOCUMENT, bindings=(replace(DOCUMENT.bindings[0], binding_version=4),)) + reference = BlobRef("blob_A") + graph = AuthorizedResponseGraph({"attachment": reference}, + (AuthorizedBlobSlot(("attachment",), "OrderItem", "20", + "attachment", "tenant-a", 4, reference),)) + + class Authorizer: + async def authorize(self, _context, _result, _options): + return graph + + class Projector: + async def project_batch(self, _context, _graph, _expiry): + return (ProjectedBlobURL("https://files.example.invalid/private/asset", + datetime.now(timezone.utc) + timedelta(seconds=30)),) + + class Assembly: + def __init__(self, document): + self.document = document + + def principal(self, _context): + return ALICE + + async def snapshot(self, _context, _graph): + return PreparedBoundDocument(self.document, 300) + + context = UserContext().insert_resource("response_graph_authorizer", Authorizer()) + context.insert_resource("blob_response_contributor", Projector()) + module.install(context, Assembly(bound)) + context._required_capabilities = frozenset(("secure_document.v1",)) + await validate_capabilities(context) + with pytest.raises(RuntimeError, match="DOCUMENT_CONFIGURATION"): + module.install(context, Assembly(bound)) + view = await ViewPreparationService().prepare( + context, None, ViewOptions("governed", "committed")) + assert view.presentation == {"attachment": "https://files.example.invalid/private/asset"} + assert view.document["documentToken"].startswith("tqd2.k1.") + assert view.expires_at <= datetime.now(timezone.utc) + timedelta(seconds=30) + context.insert_resource("teaql.runtime.capability:secure_document.v1", None) + with pytest.raises(ViewPreparationError) as missing: + await ViewPreparationService().prepare(context, None, ViewOptions("governed", "committed")) + assert missing.value.code == "DOCUMENT_CONFIGURATION" + + wrong = UserContext().insert_resource("response_graph_authorizer", Authorizer()) + module.install(wrong, Assembly(replace(bound, bindings=(replace(bound.bindings[0], + stable_key="blob_other"),)))) + with pytest.raises(ViewPreparationError) as error: + await ViewPreparationService().prepare(wrong, None, ViewOptions("governed", "committed")) + assert (error.value.code, error.value.business_state, error.value.retry) == \ + ("DOCUMENT_PROJECTION_VIOLATION", "committed", "read") diff --git a/packages/teaql-secure-document-reference/tests/test_round_trip_reference.py b/packages/teaql-secure-document-reference/tests/test_round_trip_reference.py new file mode 100644 index 00000000..19db94f6 --- /dev/null +++ b/packages/teaql-secure-document-reference/tests/test_round_trip_reference.py @@ -0,0 +1,76 @@ +from datetime import datetime, timezone + +import pytest + +from teaql_secure_document_reference import ( + ContextBoundReferenceRuntime, ReferenceDocumentScope, ReferenceIdentity, + ReferenceKey, RoundTripReferenceError, StaticReferenceKeyProvider, + TrustedReferencePrincipal, +) + + +VECTOR = ("tqr1.k2.MzMzMzMzMzMzMzMzmhCET15vBdSc1BbFJcrvMMnRP1wAyjunShYlFPnDZL7Gk5V1LuptWyrbiQ8LQSDHFUmN3o6vb-MEgXZuh3N-YkeO55CYLeVgiyBBYslzuSu-fvWC2KMZcnZ5D8BRmWpGYA2Ok5sWe1Fg1aIyadTjVoKB-H2vPW_0okj-") +ALICE = TrustedReferencePrincipal("oidc", "alice", "Platform", 7) +SCOPE = ReferenceDocumentScope("doc-100", "edit-order", "Order", 100, 9) +IDENTITY = ReferenceIdentity("OrderItem", 42, 3) +NOW = datetime(2026, 9, 27, tzinfo=timezone.utc) +KEYS = StaticReferenceKeyProvider("k2", (ReferenceKey("k1", bytes([0x11]) * 32), + ReferenceKey("k2", bytes([0x22]) * 32))) + + +def runtime(*, now=NOW, allowed=True, environment="test-a"): + def authorize(principal, _scope, identity): + if not allowed or principal.subject != "alice" or identity.entity_type != "OrderItem" or identity.id != 42: + raise PermissionError("denied") + return ContextBoundReferenceRuntime("order-service", environment, KEYS, authorize, + lambda: now, lambda: bytes([0x33]) * 12) + + +def code(call): + with pytest.raises(RoundTripReferenceError) as error: + call() + return error.value.code + + +def test_tqr1_matches_rust_fixed_wire_vector_and_reauthorizes(): + assert runtime().issue(ALICE, SCOPE, IDENTITY, 600) == VECTOR + resolved = runtime().consume(ALICE, SCOPE, VECTOR, "OrderItem") + assert resolved.identity == IDENTITY + assert resolved.key_id == "k2" + assert resolved.issued_at == NOW + assert resolved.expires_at == datetime(2026, 9, 27, 0, 10, tzinfo=timezone.utc) + assert code(lambda: runtime(allowed=False).consume(ALICE, SCOPE, VECTOR, "OrderItem")) == \ + "ROUND_TRIP_REFERENCE_AUTHORIZATION_REQUIRED" + + +def test_tqr1_rejects_wrong_identity_scope_environment_and_expiry(): + assert code(lambda: runtime().consume(TrustedReferencePrincipal("oidc", "bob", "Platform", 7), + SCOPE, VECTOR, "OrderItem")) == \ + "ROUND_TRIP_REFERENCE_SCOPE_MISMATCH" + assert code(lambda: runtime().consume(ALICE, + ReferenceDocumentScope("other", "edit-order", "Order", 100, 9), + VECTOR, "OrderItem")) == "ROUND_TRIP_REFERENCE_SCOPE_MISMATCH" + assert code(lambda: runtime().consume(ALICE, + ReferenceDocumentScope("doc-100", "view-order", "Order", 100, 9), + VECTOR, "OrderItem")) == "ROUND_TRIP_REFERENCE_INVALID" + assert code(lambda: runtime(environment="other").consume(ALICE, SCOPE, VECTOR, "OrderItem")) == \ + "ROUND_TRIP_REFERENCE_INVALID" + assert code(lambda: runtime().consume(ALICE, SCOPE, VECTOR, "Order")) == \ + "ROUND_TRIP_REFERENCE_SCOPE_MISMATCH" + assert code(lambda: runtime(now=datetime(2026, 9, 27, 0, 10, tzinfo=timezone.utc)).consume( + ALICE, SCOPE, VECTOR, "OrderItem")) == "ROUND_TRIP_REFERENCE_EXPIRED" + + +def test_tqr1_rejects_tamper_raw_id_noncanonical_and_bad_config(): + assert code(lambda: runtime().consume(ALICE, SCOPE, VECTOR[:-1] + "a", "OrderItem")) == \ + "ROUND_TRIP_REFERENCE_INVALID" + assert code(lambda: runtime().consume(ALICE, SCOPE, "42", "OrderItem")) == \ + "ROUND_TRIP_REFERENCE_INVALID" + assert code(lambda: runtime().consume(ALICE, SCOPE, VECTOR + "=", "OrderItem")) == \ + "ROUND_TRIP_REFERENCE_INVALID" + assert code(lambda: runtime().issue(ALICE, SCOPE, ReferenceIdentity("OrderItem", 0, 3), 600)) == \ + "ROUND_TRIP_REFERENCE_INVALID" + assert code(lambda: runtime().issue(ALICE, SCOPE, IDENTITY, 0)) == \ + "ROUND_TRIP_REFERENCE_INVALID" + assert code(lambda: ReferenceKey("bad.key", bytes(32))) == \ + "ROUND_TRIP_REFERENCE_CONFIGURATION_INVALID" diff --git a/src/teaql/runtime/view_preparation.py b/src/teaql/runtime/view_preparation.py index 7b40c417..f982f3b0 100644 --- a/src/teaql/runtime/view_preparation.py +++ b/src/teaql/runtime/view_preparation.py @@ -76,7 +76,10 @@ class PreparedView: "VIEW_OPTIONS_INVALID", "VIEW_AUTHORIZER_MISSING", "VIEW_GRAPH_INVALID", "DOCUMENT_CONFIGURATION", "BLOB_CAPABILITY_MISSING", "BLOB_SIGN_FAILED", "BLOB_READ_DENIED", "BLOB_REFERENCE_EXPIRED", "BLOB_SCHEMA_MISSING", - "BLOB_CATALOG_FAILED", + "BLOB_CATALOG_FAILED", "DOCUMENT_PROJECTION_VIOLATION", + "ROUND_TRIP_REFERENCE_INVALID", "ROUND_TRIP_REFERENCE_EXPIRED", + "ROUND_TRIP_REFERENCE_SCOPE_MISMATCH", "ROUND_TRIP_REFERENCE_AUTHORIZATION_REQUIRED", + "ROUND_TRIP_REFERENCE_CONFIGURATION_INVALID", }) From 5acb9a0750bd14099c1608d1cb413f1ede4caef1 Mon Sep 17 00:00:00 2001 From: Philip Z Date: Tue, 27 Oct 2026 14:55:15 +0800 Subject: [PATCH 14/75] Assemble optional Python Blob reference module --- .github/workflows/optional-blob-wheel.yml | 4 +- .../src/teaql_blob_reference/__init__.py | 3 +- .../src/teaql_blob_reference/module.py | 50 +++++++++++++ tests/provider/sqlite/test_blob_module.py | 72 +++++++++++++++++++ 4 files changed, 126 insertions(+), 3 deletions(-) create mode 100644 packages/teaql-blob-reference/src/teaql_blob_reference/module.py create mode 100644 tests/provider/sqlite/test_blob_module.py diff --git a/.github/workflows/optional-blob-wheel.yml b/.github/workflows/optional-blob-wheel.yml index 69fba5a8..427f1359 100644 --- a/.github/workflows/optional-blob-wheel.yml +++ b/.github/workflows/optional-blob-wheel.yml @@ -17,7 +17,7 @@ jobs: - run: python -m pip install -e ".[test,object-storage-s3]" - run: python -m pip install --no-deps -e ./packages/teaql-blob-reference - run: python -m pip install --no-deps -e ./packages/teaql-secure-document-reference - - run: python -m pytest -q tests/provider/sqlite/test_blob_catalog_graph.py tests/provider/sqlite/test_file_blob_storage.py tests/provider/sqlite/test_blob_upload.py + - run: python -m pytest -q tests/provider/sqlite/test_blob_catalog_graph.py tests/provider/sqlite/test_file_blob_storage.py tests/provider/sqlite/test_blob_upload.py tests/provider/sqlite/test_blob_module.py - run: python -m pytest -q packages/teaql-secure-document-reference/tests - run: python -m build --wheel --no-isolation --outdir .artifacts/base . - run: python -m build --wheel --no-isolation --outdir .artifacts/blob ./packages/teaql-blob-reference @@ -42,7 +42,7 @@ jobs: assert any(name.startswith('teaql_blob_reference/') for name in entries) assert not any(name.startswith('teaql/') for name in entries) assert not any(name.startswith('teaql_secure_document_reference/') for name in entries) - for module in ('file_storage.py', 'sqlite_catalog.py', 'upload_service.py'): + for module in ('__init__.py', 'module.py', 'file_storage.py', 'sqlite_catalog.py', 'upload_service.py'): name = f'teaql_blob_reference/{module}' assert name in entries assert wheel.read(name) == (Path('packages/teaql-blob-reference/src') / name).read_bytes() diff --git a/packages/teaql-blob-reference/src/teaql_blob_reference/__init__.py b/packages/teaql-blob-reference/src/teaql_blob_reference/__init__.py index c9b09a8f..c12432ef 100644 --- a/packages/teaql-blob-reference/src/teaql_blob_reference/__init__.py +++ b/packages/teaql-blob-reference/src/teaql_blob_reference/__init__.py @@ -2,11 +2,12 @@ from .sqlite_catalog import BlobBindingChange, BlobBindingTarget, BlobCatalogRead, SqliteBlobCatalog from .read_service import SqliteBlobResponseContributor +from .module import BlobReferenceModule from .file_storage import FileObjectStorage from .upload_service import (BlobUploadInput, BlobUploadSession, BlobUploadTarget, BlobVerifiedContent, SqliteBlobUploadService) -__all__ = ["BlobBindingChange", "BlobBindingTarget", "BlobCatalogRead", +__all__ = ["BlobReferenceModule", "BlobBindingChange", "BlobBindingTarget", "BlobCatalogRead", "SqliteBlobCatalog", "SqliteBlobResponseContributor", "FileObjectStorage", "BlobUploadInput", "BlobUploadSession", "BlobUploadTarget", "BlobVerifiedContent", "SqliteBlobUploadService"] diff --git a/packages/teaql-blob-reference/src/teaql_blob_reference/module.py b/packages/teaql-blob-reference/src/teaql_blob_reference/module.py new file mode 100644 index 00000000..22e37253 --- /dev/null +++ b/packages/teaql-blob-reference/src/teaql_blob_reference/module.py @@ -0,0 +1,50 @@ +"""Passive assembly of the optional SQLite Blob reference services.""" + +from teaql.runtime.capability import install_runtime_capability + +from .read_service import SqliteBlobResponseContributor +from .sqlite_catalog import SqliteBlobCatalog +from .upload_service import SqliteBlobUploadService + + +class _BlobCapability: + capability = "blob.v1" + + def __init__(self, module: "BlobReferenceModule") -> None: + self.module = module + + async def validate_ready(self, context) -> None: + if context.get_resource("blob_response_contributor") is not self.module.read or \ + context.get_resource("object_storage_provider") is None: + raise RuntimeError("BLOB_CAPABILITY_MISSING") + owner = context.get_resource("dataService") + if owner is None or not callable(getattr(owner, "begin", None)): + raise RuntimeError("BLOB_CAPABILITY_MISSING") + await self.module.catalog.validate_schema() + + +class BlobReferenceModule: + """Host installs this module, then separately migrates and checks its catalog.""" + + def __init__(self, catalog: SqliteBlobCatalog, upload: SqliteBlobUploadService, + read_authorization, maximum_url_seconds: int = 600): + if not isinstance(catalog, SqliteBlobCatalog) or \ + not isinstance(upload, SqliteBlobUploadService) or upload.catalog is not catalog: + raise RuntimeError("BLOB_CAPABILITY_MISSING") + self.catalog = catalog + self.upload = upload + self.read = SqliteBlobResponseContributor(catalog, read_authorization, + maximum_url_seconds) + + def install(self, context) -> None: + if context is None or context.get_resource("blob_response_contributor") is not None or \ + context.get_resource("teaql.runtime.capability:blob.v1") is not None: + raise RuntimeError("BLOB_CAPABILITY_MISSING") + install_runtime_capability(context, _BlobCapability(self)) + context.insert_resource("blob_response_contributor", self.read) + + async def ready(self, context) -> None: + capability = context.get_resource("teaql.runtime.capability:blob.v1") + if not isinstance(capability, _BlobCapability) or capability.module is not self: + raise RuntimeError("BLOB_CAPABILITY_MISSING") + await capability.validate_ready(context) diff --git a/tests/provider/sqlite/test_blob_module.py b/tests/provider/sqlite/test_blob_module.py new file mode 100644 index 00000000..0f8bc2e1 --- /dev/null +++ b/tests/provider/sqlite/test_blob_module.py @@ -0,0 +1,72 @@ +"""The optional Blob assembly fails closed until the owner and catalog are ready.""" + +import os + +import pytest + +from teaql.provider.sqlite import SimpleSchemaProvider +from teaql.provider.sqlite.dialect import SqliteDialect +from teaql.provider.sqlite.transport import SqliteTransport +from teaql.runtime import RuntimeModule, validate_capabilities +from teaql.sql.executor import SqlDataServiceExecutor +from teaql_blob_reference import ( + BlobReferenceModule, FileObjectStorage, SqliteBlobCatalog, SqliteBlobUploadService, +) + + +class _ReadAuthority: + async def authorize(self, context, slot): + return True + + +class _UploadAuthority: + def scope(self, context): + return "tenant-a" + + def field(self, context, target): + raise AssertionError("readiness must not begin an upload") + + async def authorize(self, context, target, action, input): + return True + + +class _Verifier: + def verify(self, stream, extension, maximum_bytes): + raise AssertionError("readiness must not parse content") + + +@pytest.mark.asyncio +async def test_blob_module_requires_provider_owner_and_catalog(tmp_path): + path = str(tmp_path / "module.db") + transport = SqliteTransport(path) + catalog = SqliteBlobCatalog(transport) + upload = SqliteBlobUploadService(catalog, _UploadAuthority(), _Verifier(), 10_000) + module = BlobReferenceModule(catalog, upload, _ReadAuthority()) + context = RuntimeModule.new().require_capability("blob.v1").into_context() + with pytest.raises(RuntimeError, match="BLOB_CAPABILITY_MISSING"): + await validate_capabilities(context) + module.install(context) + assert context.get_resource("blob_response_contributor") is module.read + with pytest.raises(RuntimeError, match="BLOB_CAPABILITY_MISSING"): + await module.ready(context) + with pytest.raises(RuntimeError, match="BLOB_CAPABILITY_MISSING"): + module.install(context) + + context.insert_resource("dataService", SqlDataServiceExecutor( + SqliteDialect(), SqliteTransport(path), SimpleSchemaProvider())) + provider = FileObjectStorage(tmp_path / "private", os.urandom(32)) + context.with_object_storage(provider) + with pytest.raises(RuntimeError, match="BLOB_SCHEMA_MISSING"): + await module.ready(context) + await catalog.ensure_schema() + await module.ready(context) + await validate_capabilities(context) + provider.close() + + +def test_blob_module_rejects_an_upload_from_another_catalog(tmp_path): + first = SqliteBlobCatalog(SqliteTransport(str(tmp_path / "first.db"))) + second = SqliteBlobCatalog(SqliteTransport(str(tmp_path / "second.db"))) + upload = SqliteBlobUploadService(second, _UploadAuthority(), _Verifier(), 10_000) + with pytest.raises(RuntimeError, match="BLOB_CAPABILITY_MISSING"): + BlobReferenceModule(first, upload, _ReadAuthority()) From a75b38b06d669e991cc57a6c420908fb4233285b Mon Sep 17 00:00:00 2001 From: Philip Z Date: Tue, 27 Oct 2026 15:07:41 +0800 Subject: [PATCH 15/75] Bind Python Blob fields in native graph transactions --- .github/workflows/optional-blob-wheel.yml | 3 +- .../src/teaql_blob_reference/__init__.py | 3 +- .../src/teaql_blob_reference/module.py | 7 +- .../mutation_participant.py | 102 ++++++++++++++ src/teaql/core/meta.py | 13 ++ src/teaql/core/value.py | 11 ++ src/teaql/runtime/audit.py | 5 +- src/teaql/runtime/context.py | 4 +- src/teaql/sql/executor.py | 11 ++ tests/provider/sqlite/test_blob_module.py | 127 +++++++++++++++++- 10 files changed, 278 insertions(+), 8 deletions(-) create mode 100644 packages/teaql-blob-reference/src/teaql_blob_reference/mutation_participant.py diff --git a/.github/workflows/optional-blob-wheel.yml b/.github/workflows/optional-blob-wheel.yml index 427f1359..0670838a 100644 --- a/.github/workflows/optional-blob-wheel.yml +++ b/.github/workflows/optional-blob-wheel.yml @@ -42,7 +42,8 @@ jobs: assert any(name.startswith('teaql_blob_reference/') for name in entries) assert not any(name.startswith('teaql/') for name in entries) assert not any(name.startswith('teaql_secure_document_reference/') for name in entries) - for module in ('__init__.py', 'module.py', 'file_storage.py', 'sqlite_catalog.py', 'upload_service.py'): + for module in ('__init__.py', 'module.py', 'mutation_participant.py', + 'file_storage.py', 'sqlite_catalog.py', 'upload_service.py'): name = f'teaql_blob_reference/{module}' assert name in entries assert wheel.read(name) == (Path('packages/teaql-blob-reference/src') / name).read_bytes() diff --git a/packages/teaql-blob-reference/src/teaql_blob_reference/__init__.py b/packages/teaql-blob-reference/src/teaql_blob_reference/__init__.py index c12432ef..9e598681 100644 --- a/packages/teaql-blob-reference/src/teaql_blob_reference/__init__.py +++ b/packages/teaql-blob-reference/src/teaql_blob_reference/__init__.py @@ -3,11 +3,12 @@ from .sqlite_catalog import BlobBindingChange, BlobBindingTarget, BlobCatalogRead, SqliteBlobCatalog from .read_service import SqliteBlobResponseContributor from .module import BlobReferenceModule +from .mutation_participant import SqliteBlobMutationParticipant from .file_storage import FileObjectStorage from .upload_service import (BlobUploadInput, BlobUploadSession, BlobUploadTarget, BlobVerifiedContent, SqliteBlobUploadService) -__all__ = ["BlobReferenceModule", "BlobBindingChange", "BlobBindingTarget", "BlobCatalogRead", +__all__ = ["BlobReferenceModule", "SqliteBlobMutationParticipant", "BlobBindingChange", "BlobBindingTarget", "BlobCatalogRead", "SqliteBlobCatalog", "SqliteBlobResponseContributor", "FileObjectStorage", "BlobUploadInput", "BlobUploadSession", "BlobUploadTarget", "BlobVerifiedContent", "SqliteBlobUploadService"] diff --git a/packages/teaql-blob-reference/src/teaql_blob_reference/module.py b/packages/teaql-blob-reference/src/teaql_blob_reference/module.py index 22e37253..cc3d6fdc 100644 --- a/packages/teaql-blob-reference/src/teaql_blob_reference/module.py +++ b/packages/teaql-blob-reference/src/teaql_blob_reference/module.py @@ -3,6 +3,7 @@ from teaql.runtime.capability import install_runtime_capability from .read_service import SqliteBlobResponseContributor +from .mutation_participant import SqliteBlobMutationParticipant from .sqlite_catalog import SqliteBlobCatalog from .upload_service import SqliteBlobUploadService @@ -15,6 +16,7 @@ def __init__(self, module: "BlobReferenceModule") -> None: async def validate_ready(self, context) -> None: if context.get_resource("blob_response_contributor") is not self.module.read or \ + context.get_resource("blob_mutation_participant") is not self.module.mutation or \ context.get_resource("object_storage_provider") is None: raise RuntimeError("BLOB_CAPABILITY_MISSING") owner = context.get_resource("dataService") @@ -27,21 +29,24 @@ class BlobReferenceModule: """Host installs this module, then separately migrates and checks its catalog.""" def __init__(self, catalog: SqliteBlobCatalog, upload: SqliteBlobUploadService, - read_authorization, maximum_url_seconds: int = 600): + read_authorization, binding_authorization, maximum_url_seconds: int = 600): if not isinstance(catalog, SqliteBlobCatalog) or \ not isinstance(upload, SqliteBlobUploadService) or upload.catalog is not catalog: raise RuntimeError("BLOB_CAPABILITY_MISSING") self.catalog = catalog self.upload = upload + self.mutation = SqliteBlobMutationParticipant(catalog, binding_authorization) self.read = SqliteBlobResponseContributor(catalog, read_authorization, maximum_url_seconds) def install(self, context) -> None: if context is None or context.get_resource("blob_response_contributor") is not None or \ + context.get_resource("blob_mutation_participant") is not None or \ context.get_resource("teaql.runtime.capability:blob.v1") is not None: raise RuntimeError("BLOB_CAPABILITY_MISSING") install_runtime_capability(context, _BlobCapability(self)) context.insert_resource("blob_response_contributor", self.read) + context.insert_resource("blob_mutation_participant", self.mutation) async def ready(self, context) -> None: capability = context.get_resource("teaql.runtime.capability:blob.v1") diff --git a/packages/teaql-blob-reference/src/teaql_blob_reference/mutation_participant.py b/packages/teaql-blob-reference/src/teaql_blob_reference/mutation_participant.py new file mode 100644 index 00000000..58721921 --- /dev/null +++ b/packages/teaql-blob-reference/src/teaql_blob_reference/mutation_participant.py @@ -0,0 +1,102 @@ +"""Native SQLite Blob binding in the caller's physical graph transaction.""" + +from teaql.core.blob_ref import BlobRef +from teaql.core.mutation import DeleteCommand, InsertCommand, UpdateCommand +from teaql.core.value import DataType, Value +from teaql.sql.types import CompiledQuery + +from .sqlite_catalog import BlobBindingChange, BlobBindingTarget, SqliteBlobCatalog + + +class SqliteBlobMutationParticipant: + def __init__(self, catalog: SqliteBlobCatalog, authorization): + if not isinstance(catalog, SqliteBlobCatalog) or authorization is None or \ + not callable(getattr(authorization, "scope", None)) or \ + not callable(getattr(authorization, "authorize", None)): + raise RuntimeError("BLOB_CAPABILITY_MISSING") + self.catalog = catalog + self.authorization = authorization + + async def apply(self, context, descriptor, mutation, persisted) -> None: + graph = getattr(context, "_graph_session", None) + if graph is None or context is not graph.context: + raise RuntimeError("BLOB_TRANSACTION_REQUIRED") + if not isinstance(mutation, (InsertCommand, UpdateCommand, DeleteCommand)): + raise RuntimeError("BLOB_BIND_DENIED") + id_property = next((prop for prop in descriptor.properties if prop._is_id), None) + version_property = next((prop for prop in descriptor.properties if prop._is_version), None) + if id_property is None or version_property is None: + raise RuntimeError("BLOB_CAPABILITY_MISSING") + if isinstance(mutation, DeleteCommand): + raw_id = mutation.id.val + action = "Delete" + else: + if not isinstance(persisted, dict): + raise RuntimeError("BLOB_BIND_DENIED") + raw_id = persisted.get(id_property.name) + action = "Create" if isinstance(mutation, InsertCommand) else "Update" + if isinstance(raw_id, bool) or not isinstance(raw_id, int) or raw_id <= 0: + raise RuntimeError("BLOB_BIND_DENIED") + entity_id = str(raw_id) + try: + scope = self.authorization.scope(context) + except Exception: + raise RuntimeError("BLOB_BIND_DENIED") from None + if not isinstance(scope, str) or not scope or scope != scope.strip(): + raise RuntimeError("BLOB_BIND_DENIED") + + async def prior_version(target): + async def query(sql): + rows = await sql.fetch_all_sql(CompiledQuery( + "SELECT version FROM teaql_blob_binding WHERE scope = ? AND entity = ? " + "AND entity_id = ? AND field = ?", + [Value.Text(scope), Value.Text(target.entity), Value.Text(target.entity_id), + Value.Text(target.field)])) + if len(rows) > 1: + raise RuntimeError("BLOB_CATALOG_FAILED") + return rows[0]["version"] if rows else None + return await graph.with_sql_transport(query) + + for prop in descriptor.properties: + if prop.blob_spec is None: + continue + if prop.blob_spec.entity != descriptor._name or prop.blob_spec.field != prop.name: + raise RuntimeError("BLOB_REFERENCE_INVALID") + if action != "Delete" and prop.name not in mutation.values: + if action == "Create" and not prop.blob_spec.nullable: + raise RuntimeError("BLOB_BIND_DENIED") + if action != "Create": + continue + target = BlobBindingTarget(descriptor._name, entity_id, prop.name) + prior = await prior_version(target) + if prior is not None and (type(prior) is not int or prior < 0): + raise RuntimeError("BLOB_CATALOG_FAILED") + if action == "Create" and prior is not None: + raise RuntimeError("BLOB_BIND_DENIED") + if action == "Delete": + if prior is None: + raise RuntimeError("BLOB_BIND_DENIED") + next_ref = None + else: + value = mutation.values.get(prop.name, Value.Null()) + if not isinstance(value, Value): + raise RuntimeError("BLOB_REFERENCE_INVALID") + if value.val is None: + if not prop.blob_spec.nullable: + raise RuntimeError("BLOB_BIND_DENIED") + next_ref = None + elif value._type_hint == DataType.BlobRef: + try: + next_ref = BlobRef.from_sql(value.val) + except ValueError: + raise RuntimeError("BLOB_REFERENCE_INVALID") from None + else: + raise RuntimeError("BLOB_REFERENCE_INVALID") + if persisted.get(prop.name) != (next_ref.key if next_ref else None): + raise RuntimeError("BLOB_BIND_DENIED") + business_version = persisted.get(version_property.name) if persisted else None + next_version = max((prior or 0) + 1, + business_version if type(business_version) is int and business_version > 0 else 0) + await self.catalog.bind_in_graph(graph, context, + BlobBindingChange(target, next_ref, prior or 0, next_version, action), + self.authorization) diff --git a/src/teaql/core/meta.py b/src/teaql/core/meta.py index 50c9f3dd..dbf25524 100644 --- a/src/teaql/core/meta.py +++ b/src/teaql/core/meta.py @@ -7,6 +7,17 @@ def __init__(self, name: str, property_type: str = "String"): self._is_version = False self.nullable = True self.log_policy_val = 'unknown' + self.blob_spec = None + def blob(self, spec): + from .blob_ref import BlobFieldSpec + from .value import DataType + if not isinstance(spec, BlobFieldSpec) or spec.field != self.name: + raise ValueError("BLOB_POLICY_INVALID") + self.blob_spec = spec + self.property_type = DataType.Text + self.nullable = spec.nullable + self.log_policy_val = 'credential' + return self def log_policy(self, policy): """Trusted schema policy; incoming requests cannot override it.""" if policy not in ('plain', 'masked', 'credential', 'unknown'): @@ -59,6 +70,8 @@ def table_name(self, name): self.table_name_val = name return self def property(self, prop): + if prop.blob_spec is not None and prop.blob_spec.entity != self._name: + raise ValueError("BLOB_POLICY_INVALID") self.properties.append(prop) return self diff --git a/src/teaql/core/value.py b/src/teaql/core/value.py index d3344ffc..535e4da9 100644 --- a/src/teaql/core/value.py +++ b/src/teaql/core/value.py @@ -17,6 +17,7 @@ class DataType(Enum): Date = auto() Time = auto() Timestamp = auto() + BlobRef = auto() @dataclass class Timestamp: @@ -85,6 +86,13 @@ def Decimal(v: Decimal) -> 'Value': def Text(v: str) -> 'Value': return Value(v) + @staticmethod + def Blob(v: 'BlobRef') -> 'Value': + from .blob_ref import BlobRef + if not isinstance(v, BlobRef): + raise ValueError("BLOB_REFERENCE_INVALID") + return Value(v.to_sql(), DataType.BlobRef) + @staticmethod def Json(v: Any) -> 'Value': return Value(v, DataType.Json) @@ -126,8 +134,11 @@ def __eq__(self, other): @classmethod def from_any(cls, value: Any) -> 'Value': + from .blob_ref import BlobRef if isinstance(value, cls): return value + if isinstance(value, BlobRef): + return cls.Blob(value) if value is None: return cls.Null() if isinstance(value, bool): diff --git a/src/teaql/runtime/audit.py b/src/teaql/runtime/audit.py index 7e90bce1..82815d6b 100644 --- a/src/teaql/runtime/audit.py +++ b/src/teaql/runtime/audit.py @@ -30,14 +30,15 @@ class RawAuditEvent: mutation_governance: Any = None _intent_privacy: Any = field(default=None, repr=False, compare=False) - def safe(self, mask_fields: List[str], max_length: Optional[int]) -> "SafeAuditEvent": + def safe(self, mask_fields: List[str], max_length: Optional[int], + credential_fields=()) -> "SafeAuditEvent": from .log_privacy import REDACTED, credential_name, payload_has_credentials, plaintext_enabled, scrub, value_strings allow = plaintext_enabled() secrets = [] fields = [] for change in self.changes: value = None if change.new_value is None else str(getattr(change.new_value, "val", change.new_value)) - credential = (credential_name(change.field) + credential = (change.field in credential_fields or credential_name(change.field) or payload_has_credentials(change.new_value) or payload_has_credentials(change.old_value)) masked = credential or (change.field in mask_fields and not allow) diff --git a/src/teaql/runtime/context.py b/src/teaql/runtime/context.py index c2f14dab..beb27d33 100644 --- a/src/teaql/runtime/context.py +++ b/src/teaql/runtime/context.py @@ -791,7 +791,9 @@ async def send_audit_event(self, event: Any): descriptor = self.entity(event.entity) mask_fields = getattr(descriptor, "audit_mask_fields_val", []) if descriptor else [] max_len = getattr(descriptor, "audit_value_max_len_val", None) if descriptor else None - safe_event = event.safe(mask_fields, max_len) + credential_fields = {prop.name for prop in descriptor.properties + if getattr(prop, "blob_spec", None) is not None} if descriptor else set() + safe_event = event.safe(mask_fields, max_len, credential_fields) if self._graph_session is not None: self._graph_session.queue_audit(event, safe_event) return diff --git a/src/teaql/sql/executor.py b/src/teaql/sql/executor.py index 4393ecf0..edac2a66 100644 --- a/src/teaql/sql/executor.py +++ b/src/teaql/sql/executor.py @@ -1175,6 +1175,17 @@ async def _mutate(self, context: 'UserContext', request: MutationRequest, privac read_start, persisted_rows, None, privacy, read_scope.correlation) metadata = replace(metadata, statements=(metadata, read_metadata)) + if affected_rows > 0 and context is not None: + blob_fields = [prop for prop in entity_desc.properties + if getattr(prop, 'blob_spec', None) is not None] + if blob_fields and (isinstance(req_data, (InsertCommand, DeleteCommand, RecoverCommand)) + or isinstance(req_data, UpdateCommand) and any( + prop.name in req_data.values for prop in blob_fields)): + participant = context.get_resource('blob_mutation_participant') + if not callable(getattr(participant, 'apply', None)): + raise RuntimeError('BLOB_CAPABILITY_MISSING') + await participant.apply(context, entity_desc, req_data, persisted_record) + if affected_rows > 0 and context is not None: from teaql.runtime.audit import AuditFieldChange, MutationAuditKind, RawAuditEvent if isinstance(req_data, InsertCommand): diff --git a/tests/provider/sqlite/test_blob_module.py b/tests/provider/sqlite/test_blob_module.py index 0f8bc2e1..6acd5b4e 100644 --- a/tests/provider/sqlite/test_blob_module.py +++ b/tests/provider/sqlite/test_blob_module.py @@ -2,8 +2,14 @@ import os +import aiosqlite import pytest +from teaql.core.blob_ref import BlobFieldSpec, BlobRef +from teaql.core.entity import EntityKey +from teaql.core.meta import EntityDescriptor, PropertyDescriptor +from teaql.core.mutation import DeleteCommand, InsertCommand, UpdateCommand +from teaql.core.value import DataType from teaql.provider.sqlite import SimpleSchemaProvider from teaql.provider.sqlite.dialect import SqliteDialect from teaql.provider.sqlite.transport import SqliteTransport @@ -35,13 +41,31 @@ def verify(self, stream, extension, maximum_bytes): raise AssertionError("readiness must not parse content") +class _BindingAuthority: + allowed = True + + def scope(self, context): + return "tenant-a" + + async def authorize(self, context, target, previous, next_ref, action): + return self.allowed and target.entity == "Asset" and target.field == "document" + + +class _SafeAuditSink: + def __init__(self): + self.events = [] + + async def on_safe_event(self, context, event): + self.events.append(event) + + @pytest.mark.asyncio async def test_blob_module_requires_provider_owner_and_catalog(tmp_path): path = str(tmp_path / "module.db") transport = SqliteTransport(path) catalog = SqliteBlobCatalog(transport) upload = SqliteBlobUploadService(catalog, _UploadAuthority(), _Verifier(), 10_000) - module = BlobReferenceModule(catalog, upload, _ReadAuthority()) + module = BlobReferenceModule(catalog, upload, _ReadAuthority(), _BindingAuthority()) context = RuntimeModule.new().require_capability("blob.v1").into_context() with pytest.raises(RuntimeError, match="BLOB_CAPABILITY_MISSING"): await validate_capabilities(context) @@ -69,4 +93,103 @@ def test_blob_module_rejects_an_upload_from_another_catalog(tmp_path): second = SqliteBlobCatalog(SqliteTransport(str(tmp_path / "second.db"))) upload = SqliteBlobUploadService(second, _UploadAuthority(), _Verifier(), 10_000) with pytest.raises(RuntimeError, match="BLOB_CAPABILITY_MISSING"): - BlobReferenceModule(first, upload, _ReadAuthority()) + BlobReferenceModule(first, upload, _ReadAuthority(), _BindingAuthority()) + + +@pytest.mark.asyncio +async def test_native_mutation_binds_blob_in_same_sqlite_graph(tmp_path): + path = str(tmp_path / "native.db") + descriptor = EntityDescriptor("Asset").table_name("asset") + descriptor.property(PropertyDescriptor("id", DataType.I64).is_id()) + descriptor.property(PropertyDescriptor("version", DataType.I64).is_version()) + descriptor.property(PropertyDescriptor("document", DataType.Text).blob( + BlobFieldSpec("Asset", "document", ["pdf"], True, "default"))) + schema = SimpleSchemaProvider() + schema.register_entity(descriptor) + context = RuntimeModule.new().entity(descriptor).require_capability("blob.v1").into_context() + context.insert_resource("dataService", SqlDataServiceExecutor( + SqliteDialect(), SqliteTransport(path), schema)) + audit = _SafeAuditSink() + context.with_app_audit_event_sink(audit) + store = FileObjectStorage(tmp_path / "native-private", os.urandom(32)) + context.with_object_storage(store) + catalog = SqliteBlobCatalog(SqliteTransport(path)) + await catalog.ensure_schema() + async with aiosqlite.connect(path) as db: + await db.execute("CREATE TABLE asset (id INTEGER PRIMARY KEY, version INTEGER, document TEXT)") + for key in ("blob_native_one", "blob_native_two"): + await db.execute("INSERT INTO teaql_blob_catalog " + "(key,scope,object_key,content_version,file_name,content_type,size_bytes,status) " + "VALUES (?,?,?,?,?,?,?,?)", + (key, "tenant-a", f"objects/{key}", "v1", "report.pdf", + "application/pdf", 7, "uploaded_unbound")) + await db.commit() + authority = _BindingAuthority() + upload = SqliteBlobUploadService(catalog, _UploadAuthority(), _Verifier(), 10_000) + module = BlobReferenceModule(catalog, upload, _ReadAuthority(), authority) + module.install(context) + await module.ready(context) + + async def save(command, comment, asset_id=1): + async def work(graph): + return await graph.transaction.mutate(graph.context, + graph.request(command, graph.scope(EntityKey("Asset", asset_id)))) + return await context.execute_graph_save(work, comment=comment) + + first, second = BlobRef("blob_native_one"), BlobRef("blob_native_two") + await save(InsertCommand.new("Asset").value("id", 1).value("document", first), "create") + assert audit.events and any(field.field == "document" and field.value == "[REDACTED]" + for field in audit.events[-1].fields) + async with aiosqlite.connect(path) as db: + async with db.execute("SELECT document,version FROM asset WHERE id=1") as cursor: + assert await cursor.fetchone() == (first.key, 1) + async with db.execute("SELECT key,version FROM teaql_blob_binding") as cursor: + assert await cursor.fetchone() == (first.key, 1) + + await save(UpdateCommand.new("Asset", 1).expected_version(1) + .value("document", second), "replace") + async with aiosqlite.connect(path) as db: + async with db.execute("SELECT document,version FROM asset WHERE id=1") as cursor: + assert await cursor.fetchone() == (second.key, 2) + async with db.execute("SELECT key,version FROM teaql_blob_binding") as cursor: + assert await cursor.fetchone() == (second.key, 2) + async with db.execute("SELECT key,status FROM teaql_blob_catalog ORDER BY key") as cursor: + assert await cursor.fetchall() == [(first.key, "uploaded_unbound"), + (second.key, "available")] + + authority.allowed = False + with pytest.raises(RuntimeError, match="BLOB_BIND_DENIED"): + await save(UpdateCommand.new("Asset", 1).expected_version(2) + .value("document", first), "deny replacement") + authority.allowed = True + with pytest.raises(RuntimeError, match="BLOB_REFERENCE_INVALID"): + await save(UpdateCommand.new("Asset", 1).expected_version(2) + .value("document", second.key), "reject raw Key") + async with aiosqlite.connect(path) as db: + async with db.execute("SELECT document,version FROM asset WHERE id=1") as cursor: + assert await cursor.fetchone() == (second.key, 2) + async with db.execute("SELECT key,version FROM teaql_blob_binding") as cursor: + assert await cursor.fetchone() == (second.key, 2) + await save(UpdateCommand.new("Asset", 1).expected_version(2) + .value("document", None), "clear") + async with aiosqlite.connect(path) as db: + async with db.execute("SELECT key,version FROM teaql_blob_binding") as cursor: + assert await cursor.fetchone() == (None, 3) + await save(UpdateCommand.new("Asset", 1).expected_version(3) + .value("document", second), "rebind") + await save(DeleteCommand.new("Asset", 1).expected_version(4).hard_delete(), "delete") + async with aiosqlite.connect(path) as db: + async with db.execute("SELECT id FROM asset") as cursor: + assert await cursor.fetchone() is None + async with db.execute("SELECT key,version FROM teaql_blob_binding") as cursor: + assert await cursor.fetchone() == (None, 5) + async with db.execute("SELECT status FROM teaql_blob_catalog WHERE key=?", + (second.key,)) as cursor: + assert await cursor.fetchone() == ("uploaded_unbound",) + await save(InsertCommand.new("Asset").value("id", 2), "create empty", asset_id=2) + async with aiosqlite.connect(path) as db: + async with db.execute("SELECT key,version FROM teaql_blob_binding WHERE entity_id='2'") as cursor: + assert await cursor.fetchone() == (None, 1) + await save(DeleteCommand.new("Asset", 2).expected_version(1).hard_delete(), + "delete empty", asset_id=2) + store.close() From f64566793cc7c1de5a818ba252816ca913939890 Mon Sep 17 00:00:00 2001 From: Philip Z Date: Tue, 27 Oct 2026 19:34:39 +0800 Subject: [PATCH 16/75] fix: recheck completed Blob uploads on replay --- .../teaql_blob_reference/upload_service.py | 41 +++++++++++++++++++ tests/provider/sqlite/test_blob_upload.py | 17 ++++++++ 2 files changed, 58 insertions(+) diff --git a/packages/teaql-blob-reference/src/teaql_blob_reference/upload_service.py b/packages/teaql-blob-reference/src/teaql_blob_reference/upload_service.py index dd5114d9..08da116a 100644 --- a/packages/teaql-blob-reference/src/teaql_blob_reference/upload_service.py +++ b/packages/teaql-blob-reference/src/teaql_blob_reference/upload_service.py @@ -337,6 +337,47 @@ async def work(tx): async def complete(self, actor, upload_id: str) -> BlobRef: row = await self._row(actor, upload_id, "Confirm") + if row["state"] == "completed": + if (not isinstance(row["written_sha256"], str) or + re.fullmatch(r"[0-9a-f]{64}", row["written_sha256"]) is None or + not row["written_etag"]): + raise RuntimeError("BLOB_UPLOAD_INCOMPLETE") + try: + rows = await self.catalog.transport.fetch_all_sql(_query( + "SELECT object_key,content_version,content_type,size_bytes,status " + "FROM teaql_blob_catalog WHERE key=? AND scope=?", + Value.Text(row["key"]), Value.Text(row["scope"]))) + if len(rows) != 1: + raise RuntimeError("BLOB_UPLOAD_INCOMPLETE") + catalog = rows[0] + mime = blob_mime_for_extension(row["extension"]) + if (catalog["status"] not in ("uploaded_unbound", "available") or + catalog["content_version"] != row["written_etag"] or + catalog["content_type"] != mime or + type(catalog["size_bytes"]) is not int or + catalog["size_bytes"] != row["declared_size"]): + raise RuntimeError("BLOB_UPLOAD_INCOMPLETE") + key = ObjectKey(catalog["object_key"]) + except Exception: + raise RuntimeError("BLOB_UPLOAD_INCOMPLETE") from None + + def recheck_stored(): + try: + metadata = actor.object_storage("recheck completed Blob upload").head(key) + if (metadata is None or metadata.size != row["declared_size"] or + metadata.etag != row["written_etag"]): + raise RuntimeError("BLOB_UPLOAD_INCOMPLETE") + except Exception: + raise RuntimeError("BLOB_UPLOAD_INCOMPLETE") from None + + await asyncio.to_thread(recheck_stored) + target = BlobUploadTarget(row["entity"], row["entity_id"], row["field"]) + await self._current(actor, target, "Confirm", + BlobUploadInput(row["file_name"], row["declared_size"], mime), + row["scope"]) + if self.now_ms() >= row["expires_ms"]: + raise RuntimeError("BLOB_UPLOAD_EXPIRED") + return BlobRef(row["key"]) if row["state"] != "written" or not row["written_sha256"] or not row["written_etag"]: raise RuntimeError("BLOB_UPLOAD_INCOMPLETE") key = await self._object_key(actor, row) diff --git a/tests/provider/sqlite/test_blob_upload.py b/tests/provider/sqlite/test_blob_upload.py index 5994af09..f30d073e 100644 --- a/tests/provider/sqlite/test_blob_upload.py +++ b/tests/provider/sqlite/test_blob_upload.py @@ -113,6 +113,12 @@ async def test_upload_real_sqlite_file_verifier_and_native_bind(tmp_path): authority.allowed = True reference = await service.complete(context, session.upload_id) assert reference == session.reference and verifier.calls == 1 + assert await service.complete(context, session.upload_id) == reference + assert verifier.calls == 1 + authority.allowed = False + with pytest.raises(RuntimeError, match="BLOB_UPLOAD_DENIED"): + await service.complete(context, session.upload_id) + authority.allowed = True async def bind(graph): command = (InsertCommand.new("Asset").value("id", 1) @@ -128,6 +134,17 @@ async def bind(graph): async with db.execute("SELECT status FROM teaql_blob_catalog WHERE key=?", (reference.key,)) as cursor: assert (await cursor.fetchone())[0] == "available" + assert await service.complete(context, session.upload_id) == reference + assert verifier.calls == 1 + async with aiosqlite.connect(path) as db: + async with db.execute("SELECT object_key FROM teaql_blob_catalog WHERE key=?", + (reference.key,)) as cursor: + completed_key = (await cursor.fetchone())[0] + context.object_storage("replace completed generation").put_bytes( + ObjectKey(completed_key), data, PutOptions(len(data), WriteMode.REPLACE, "application/pdf")) + with pytest.raises(RuntimeError, match="BLOB_UPLOAD_INCOMPLETE"): + await service.complete(context, session.upload_id) + assert verifier.calls == 1 second = await service.begin(context, target, input, "request-2") await service.write(context, second.upload_id, BytesIO(data)) From 752e33126958cbc61afcaf7f3753cc55629b98e6 Mon Sep 17 00:00:00 2001 From: Philip Z Date: Tue, 27 Oct 2026 20:19:48 +0800 Subject: [PATCH 17/75] Add SQLite secure document open and accept service --- .../__init__.py | 8 + .../sqlite_document_service.py | 417 ++++++++++++++++++ .../tests/test_sqlite_document_service.py | 192 ++++++++ 3 files changed, 617 insertions(+) create mode 100644 packages/teaql-secure-document-reference/src/teaql_secure_document_reference/sqlite_document_service.py create mode 100644 packages/teaql-secure-document-reference/tests/test_sqlite_document_service.py diff --git a/packages/teaql-secure-document-reference/src/teaql_secure_document_reference/__init__.py b/packages/teaql-secure-document-reference/src/teaql_secure_document_reference/__init__.py index 64400ed6..410047f3 100644 --- a/packages/teaql-secure-document-reference/src/teaql_secure_document_reference/__init__.py +++ b/packages/teaql-secure-document-reference/src/teaql_secure_document_reference/__init__.py @@ -11,6 +11,11 @@ DocumentSnapshot, PreparedBoundDocument, SecureDocumentAssembly, SecureDocumentReferenceModule, VerifiedDocument, ) +from .sqlite_document_service import ( + AcceptedDocument, ContextualDocument, CurrentBoundDocument, + DocumentCommandReceipt, SQLiteContextBoundDocumentService, + SQLiteDocumentCommandJournal, SubmittedDocumentCommand, +) __all__ = [ "ContextBoundReferenceRuntime", "ReferenceDocumentScope", "ReferenceIdentity", @@ -20,4 +25,7 @@ "DocumentBlobBindingSnapshot", "DocumentEntitySnapshot", "DocumentRelationSnapshot", "DocumentSnapshot", "PreparedBoundDocument", "SecureDocumentAssembly", "SecureDocumentReferenceModule", "VerifiedDocument", + "AcceptedDocument", "ContextualDocument", "CurrentBoundDocument", + "DocumentCommandReceipt", "SQLiteContextBoundDocumentService", + "SQLiteDocumentCommandJournal", "SubmittedDocumentCommand", ] diff --git a/packages/teaql-secure-document-reference/src/teaql_secure_document_reference/sqlite_document_service.py b/packages/teaql-secure-document-reference/src/teaql_secure_document_reference/sqlite_document_service.py new file mode 100644 index 00000000..e58b8f50 --- /dev/null +++ b/packages/teaql-secure-document-reference/src/teaql_secure_document_reference/sqlite_document_service.py @@ -0,0 +1,417 @@ +"""Host-backed secure documents and durable command receipts on a physical SQLite graph.""" + +from __future__ import annotations + +import hashlib +import json +import unicodedata +from dataclasses import dataclass +from typing import Any, Protocol + +from teaql.core.value import Value +from teaql.runtime.graph_session import GraphCommittedError +from teaql.sql.types import CompiledQuery + +from .document_round_trip import ( + BoundDocumentSnapshot, SecureDocumentReferenceModule, _bindings, _encode_snapshot, +) +from .round_trip_reference import ReferenceDocumentScope, TrustedReferencePrincipal + +_MAX_BYTES = 256000 +_MAX_SAFE_INTEGER = (1 << 53) - 1 + + +def _invalid(code: str) -> None: + raise RuntimeError(code) + + +def _text(value: Any, maximum: int) -> bool: + return (isinstance(value, str) and bool(value) and value == value.strip() and + all(not unicodedata.category(ch).startswith("C") for ch in value) and + len(value.encode("utf-8")) <= maximum) + + +def _copy_body(value: Any) -> dict[str, Any]: + remaining = 100000 + + def check(item: Any, depth: int) -> None: + nonlocal remaining + remaining -= 1 + if remaining < 0 or depth > 64: + _invalid("DOCUMENT_COMMAND_INVALID") + if item is None or type(item) is bool: + return + if type(item) is str: + if len(item.encode("utf-8")) > _MAX_BYTES: + _invalid("DOCUMENT_COMMAND_INVALID") + return + if type(item) is int: + if abs(item) > _MAX_SAFE_INTEGER: + _invalid("DOCUMENT_COMMAND_INVALID") + return + if type(item) is list: + for child in item: + check(child, depth + 1) + return + if type(item) is dict: + for key, child in item.items(): + if not _text(key, 16384) or key in {"__proto__", "constructor", "prototype"}: + _invalid("DOCUMENT_COMMAND_INVALID") + check(child, depth + 1) + return + _invalid("DOCUMENT_COMMAND_INVALID") + + if type(value) is not dict: + _invalid("DOCUMENT_COMMAND_INVALID") + check(value, 0) + try: + encoded = json.dumps(value, sort_keys=True, ensure_ascii=False, + separators=(",", ":"), allow_nan=False).encode("utf-8") + except (UnicodeError, ValueError, TypeError, RecursionError): + _invalid("DOCUMENT_COMMAND_INVALID") + if len(encoded) > _MAX_BYTES: + _invalid("DOCUMENT_COMMAND_INVALID") + return json.loads(encoded) + + +def _hash_parts(*parts: Any) -> str: + encoded = json.dumps(parts, ensure_ascii=False, separators=(",", ":"), + sort_keys=True).encode("utf-8") + return hashlib.sha256(encoded).hexdigest() + + +@dataclass(frozen=True, repr=False) +class SubmittedDocumentCommand: + business_id: str + document_token: str + command_id: str + body: dict[str, Any] + + def __repr__(self) -> str: + return "SubmittedDocumentCommand[redacted]" + + +@dataclass(frozen=True, repr=False) +class DocumentCommandReceipt: + previous_revision: int + new_revision: int + response: str + + def __repr__(self) -> str: + return "DocumentCommandReceipt[redacted]" + + +def _prepared(command: SubmittedDocumentCommand) -> tuple[str, SubmittedDocumentCommand]: + if (not isinstance(command, SubmittedDocumentCommand) or + not _text(command.business_id, 16384) or + not _text(command.document_token, _MAX_BYTES) or + not _text(command.command_id, 256)): + _invalid("DOCUMENT_COMMAND_INVALID") + copied = _copy_body(command.body) + return _hash_parts(command.business_id, command.document_token, + command.command_id, copied), SubmittedDocumentCommand( + command.business_id, command.document_token, command.command_id, copied) + + +def _receipt(value: DocumentCommandReceipt) -> DocumentCommandReceipt: + if (not isinstance(value, DocumentCommandReceipt) or + type(value.previous_revision) is not int or + type(value.new_revision) is not int or + value.previous_revision < 0 or value.new_revision < value.previous_revision or + not _text(value.response, _MAX_BYTES)): + _invalid("DOCUMENT_COMMAND_INVALID") + return value + + +def _scope_key(scope: ReferenceDocumentScope) -> str: + scope.validate() + return _hash_parts(scope.document_id, scope.purpose, scope.aggregate_type, + str(scope.aggregate_id)) + + +def _actor_key(principal: TrustedReferencePrincipal) -> str: + principal.wire() + return _hash_parts(principal.authentication_realm, principal.subject, + principal.domain_root_type, str(principal.domain_root_id)) + + +class SQLiteDocumentCommandJournal: + """Receipt table installation is explicit; writes borrow a live graph SQL lease.""" + + def __init__(self, transport) -> None: + if transport is None: + _invalid("DOCUMENT_CAPABILITY_MISSING") + self.transport = transport + + async def ensure_schema(self) -> None: + try: + await self.transport.execute_sql(CompiledQuery("""CREATE TABLE IF NOT EXISTS + teaql_document_command_receipt ( + scope_hash TEXT NOT NULL, actor_hash TEXT NOT NULL, command_id TEXT NOT NULL, + request_hash TEXT NOT NULL, previous_revision BIGINT NOT NULL, + new_revision BIGINT NOT NULL, response TEXT NOT NULL, + PRIMARY KEY (scope_hash, actor_hash, command_id))""", [])) + except Exception: + _invalid("DOCUMENT_CATALOG_FAILED") + + async def validate_schema(self) -> None: + try: + rows = await self.transport.fetch_all_sql(CompiledQuery( + "PRAGMA table_info(teaql_document_command_receipt)", [])) + except Exception: + _invalid("DOCUMENT_SCHEMA_MISSING") + names = ("scope_hash", "actor_hash", "command_id", "request_hash", + "previous_revision", "new_revision", "response") + if len(rows) != len(names): + _invalid("DOCUMENT_SCHEMA_MISSING") + for index, name in enumerate(names): + row = rows[index] + if (row.get("name") != name or row.get("notnull") != 1 or + row.get("pk") != (index + 1 if index < 3 else 0)): + _invalid("DOCUMENT_SCHEMA_MISSING") + + async def apply_in_graph(self, graph, scope: ReferenceDocumentScope, + command: SubmittedDocumentCommand, authority, mutate + ) -> tuple[DocumentCommandReceipt, bool]: + if graph is None or not callable(getattr(graph, "with_sql_transport", None)): + _invalid("DOCUMENT_TRANSACTION_REQUIRED") + scope_key = _scope_key(scope) + request_hash, frozen = _prepared(command) + + async def work(sql): + async def current_actor(): + try: + principal = await authority(graph.context, scope, sql) + return _actor_key(principal) + except Exception: + _invalid("DOCUMENT_AUTHORIZATION_REQUIRED") + + actor_key = await current_actor() + params = [Value.Text(scope_key), Value.Text(actor_key), Value.Text(command.command_id)] + try: + rows = await sql.fetch_all_sql(CompiledQuery("""SELECT request_hash, + previous_revision,new_revision,response FROM teaql_document_command_receipt + WHERE scope_hash=? AND actor_hash=? AND command_id=?""", params)) + except Exception: + _invalid("DOCUMENT_CATALOG_FAILED") + if len(rows) > 1: + _invalid("DOCUMENT_CATALOG_FAILED") + if rows: + row = rows[0] + if row.get("request_hash") != request_hash: + _invalid("DOCUMENT_REPLAY_CONFLICT") + try: + found = _receipt(DocumentCommandReceipt(row["previous_revision"], + row["new_revision"], row["response"])) + except (KeyError, RuntimeError): + _invalid("DOCUMENT_CATALOG_FAILED") + if await current_actor() != actor_key: + _invalid("DOCUMENT_AUTHORIZATION_REQUIRED") + return found, True + produced = _receipt(await mutate(sql, frozen)) + if _prepared(frozen)[0] != request_hash or _prepared(command)[0] != request_hash: + _invalid("DOCUMENT_COMMAND_INVALID") + if await current_actor() != actor_key: + _invalid("DOCUMENT_AUTHORIZATION_REQUIRED") + try: + affected, _ = await sql.execute_sql(CompiledQuery("""INSERT INTO + teaql_document_command_receipt + (scope_hash,actor_hash,command_id,request_hash,previous_revision,new_revision,response) + VALUES (?,?,?,?,?,?,?)""", [*params, Value.Text(request_hash), + Value.I64(produced.previous_revision), Value.I64(produced.new_revision), + Value.Text(produced.response)])) + except Exception: + _invalid("DOCUMENT_CATALOG_FAILED") + if affected != 1: + _invalid("DOCUMENT_CATALOG_FAILED") + return produced, False + + return await graph.with_sql_transport(work) + + +@dataclass(frozen=True, repr=False) +class ContextualDocument: + business_id: str + document_token: str + body: dict[str, Any] + + def __repr__(self) -> str: + return "ContextualDocument[redacted]" + + +@dataclass(frozen=True, repr=False) +class AcceptedDocument: + document: ContextualDocument + previous_revision: int + new_revision: int + replayed: bool + + def __repr__(self) -> str: + return "AcceptedDocument[redacted]" + + +@dataclass(frozen=True) +class CurrentBoundDocument: + document: BoundDocumentSnapshot + body: dict[str, Any] + lifetime_seconds: int + + +class SQLiteContextBoundDocumentHost(Protocol): + def principal(self, context) -> TrustedReferencePrincipal: ... + async def authorize(self, context, principal: TrustedReferencePrincipal, + scope: ReferenceDocumentScope, sql) -> None: ... + async def load(self, context, business_id: str, sql) -> CurrentBoundDocument: ... + async def apply(self, context, issued: BoundDocumentSnapshot, + submitted: SubmittedDocumentCommand, sql) -> None: ... + async def present(self, context, document: ContextualDocument) -> None: ... + + +def _valid_current(current: CurrentBoundDocument, business_id: str, + aggregate_type: str, purpose: str) -> None: + if (not isinstance(current, CurrentBoundDocument) or + current.document.snapshot.business_id != business_id or + current.document.snapshot.scope.aggregate_type != aggregate_type or + current.document.snapshot.scope.purpose != purpose or + type(current.lifetime_seconds) is not int or current.lifetime_seconds <= 0): + _invalid("DOCUMENT_PROJECTION_VIOLATION") + try: + _bound_fingerprint(current.document) + _copy_body(current.body) + except Exception: + _invalid("DOCUMENT_PROJECTION_VIOLATION") + + +def _bound_fingerprint(document: BoundDocumentSnapshot): + return (_encode_snapshot(document.snapshot, 2, bytes(32), 0, 1), + _bindings(document)) + + +class SQLiteContextBoundDocumentService: + def __init__(self, context, transport, module: SecureDocumentReferenceModule, + journal: SQLiteDocumentCommandJournal, host: SQLiteContextBoundDocumentHost): + if (context is None or transport is None or not callable(getattr(transport, "begin_sql", None)) + or not isinstance(module, SecureDocumentReferenceModule) or + not isinstance(journal, SQLiteDocumentCommandJournal) or host is None): + _invalid("DOCUMENT_CONFIGURATION") + self.context, self.transport, self.module = context, transport, module + self.journal, self.host = journal, host + + def _principal(self, context) -> TrustedReferencePrincipal: + if context is not self.context: + _invalid("DOCUMENT_CONFIGURATION") + try: + principal = self.host.principal(context) + principal.wire() + return principal + except Exception: + _invalid("DOCUMENT_AUTHORIZATION_REQUIRED") + + async def open(self, context, aggregate_type: str, business_id: str, + purpose: str, lifetime_seconds: int) -> ContextualDocument: + if (not all(_text(v, 16384) for v in (aggregate_type, business_id, purpose)) or + type(lifetime_seconds) is not int or lifetime_seconds <= 0): + _invalid("DOCUMENT_COMMAND_INVALID") + principal = self._principal(context) + tx = await self.transport.begin_sql() + committed = False + try: + current = await self.host.load(context, business_id, tx) + _valid_current(current, business_id, aggregate_type, purpose) + if self._principal(context) != principal: + _invalid("DOCUMENT_AUTHORIZATION_REQUIRED") + try: + await self.host.authorize(context, principal, current.document.snapshot.scope, tx) + except Exception: + _invalid("DOCUMENT_AUTHORIZATION_REQUIRED") + if self._principal(context) != principal: + _invalid("DOCUMENT_AUTHORIZATION_REQUIRED") + token = self.module.issue_bound_document_snapshot( + principal, current.document, min(lifetime_seconds, current.lifetime_seconds)) + result = ContextualDocument(business_id, token, _copy_body(current.body)) + await tx.commit_sql() + committed = True + finally: + if not committed: + try: + await tx.rollback_sql() + except Exception: + pass + await self.host.present(context, result) + return result + + async def accept(self, context, aggregate_type: str, purpose: str, + command: SubmittedDocumentCommand) -> AcceptedDocument: + if not _text(aggregate_type, 16384) or not _text(purpose, 16384): + _invalid("DOCUMENT_COMMAND_INVALID") + principal = self._principal(context) + _prepared(command) + issued = self.module.consume_bound_document_snapshot( + principal, command.document_token, aggregate_type, purpose).document + scope = issued.snapshot.scope + if issued.snapshot.business_id != command.business_id: + _invalid("DOCUMENT_PROJECTION_VIOLATION") + + async def graph_work(graph): + async def authority(_view, expected_scope, sql): + current = self._principal(context) + if current != principal or expected_scope != scope: + _invalid("DOCUMENT_AUTHORIZATION_REQUIRED") + await self.host.authorize(context, current, expected_scope, sql) + return current + + async def mutate(sql, frozen): + before = await self.host.load(context, frozen.business_id, sql) + _valid_current(before, frozen.business_id, aggregate_type, purpose) + if _bound_fingerprint(before.document) != _bound_fingerprint(issued): + _invalid("DOCUMENT_REVISION_CONFLICT") + await self.host.apply(context, issued, frozen, sql) + after = await self.host.load(context, frozen.business_id, sql) + _valid_current(after, frozen.business_id, aggregate_type, purpose) + next_scope = after.document.snapshot.scope + if (next_scope.document_id != scope.document_id or + next_scope.aggregate_id != scope.aggregate_id or + next_scope.aggregate_type != scope.aggregate_type or + next_scope.purpose != scope.purpose or + next_scope.aggregate_revision < scope.aggregate_revision): + _invalid("DOCUMENT_PROJECTION_VIOLATION") + try: + await self.host.authorize(context, principal, next_scope, sql) + except Exception: + _invalid("DOCUMENT_AUTHORIZATION_REQUIRED") + token = self.module.issue_bound_document_snapshot( + principal, after.document, after.lifetime_seconds) + body = _copy_body(after.body) + response = json.dumps({"businessId": frozen.business_id, + "documentToken": token, "body": body}, + sort_keys=True, ensure_ascii=False, separators=(",", ":")) + return DocumentCommandReceipt(scope.aggregate_revision, + next_scope.aggregate_revision, response) + + return await self.journal.apply_in_graph(graph, scope, command, authority, mutate) + + receipt, replayed = await context.execute_graph_save( + graph_work, comment="accept protected document command") + try: + data = json.loads(receipt.response) + if (type(data) is not dict or set(data) != {"businessId", "documentToken", "body"} or + data["businessId"] != command.business_id or + not _text(data["documentToken"], _MAX_BYTES)): + _invalid("DOCUMENT_CATALOG_FAILED") + if self._principal(context) != principal: + _invalid("DOCUMENT_AUTHORIZATION_REQUIRED") + verified = self.module.consume_bound_document_snapshot( + principal, data["documentToken"], aggregate_type, purpose).document + if (verified.snapshot.business_id != command.business_id or + verified.snapshot.scope.document_id != scope.document_id or + verified.snapshot.scope.aggregate_id != scope.aggregate_id or + verified.snapshot.scope.aggregate_revision != receipt.new_revision or + receipt.previous_revision != scope.aggregate_revision): + _invalid("DOCUMENT_CATALOG_FAILED") + result = ContextualDocument(command.business_id, data["documentToken"], + _copy_body(data["body"])) + await self.host.present(context, result) + except Exception as error: + raise GraphCommittedError((error,)) from None + return AcceptedDocument(result, receipt.previous_revision, + receipt.new_revision, replayed) diff --git a/packages/teaql-secure-document-reference/tests/test_sqlite_document_service.py b/packages/teaql-secure-document-reference/tests/test_sqlite_document_service.py new file mode 100644 index 00000000..e59785a7 --- /dev/null +++ b/packages/teaql-secure-document-reference/tests/test_sqlite_document_service.py @@ -0,0 +1,192 @@ +"""A real SQLite application host exercises the optional secure-document service.""" + +from dataclasses import replace +from datetime import datetime, timezone + +import aiosqlite +import pytest + +from teaql.core.value import Value +from teaql.provider.sqlite.dialect import SqliteDialect +from teaql.provider.sqlite.transport import SqliteTransport +from teaql.runtime import RuntimeModule +from teaql.runtime.graph_session import GraphCommittedError +from teaql.sql.executor import SqlDataServiceExecutor +from teaql.sql.types import CompiledQuery +from teaql.provider.sqlite import SimpleSchemaProvider +from teaql_secure_document_reference import ( + BoundDocumentSnapshot, ContextualDocument, CurrentBoundDocument, + DocumentEntitySnapshot, DocumentSnapshot, ReferenceDocumentScope, + ReferenceIdentity, ReferenceKey, SecureDocumentReferenceModule, + SQLiteContextBoundDocumentService, SQLiteDocumentCommandJournal, + StaticReferenceKeyProvider, SubmittedDocumentCommand, TrustedReferencePrincipal, +) + + +class OrderHost: + def __init__(self): + self.actor = TrustedReferencePrincipal("oidc", "alice", "Merchant", 7) + self.mutations = 0 + self.fail_present = False + self.switch_actor_on_apply = False + + def principal(self, _context): + return self.actor + + async def authorize(self, _context, principal, _scope, sql): + rows = await sql.fetch_all_sql(CompiledQuery( + "SELECT can_edit FROM document_permission WHERE subject=?", + [Value.Text(principal.subject)])) + if len(rows) != 1 or rows[0]["can_edit"] != 1: + raise PermissionError("revoked") + + async def load(self, _context, business_id, sql): + rows = await sql.fetch_all_sql(CompiledQuery( + "SELECT id,version,quantity FROM document_order WHERE business_id=?", + [Value.Text(business_id)])) + if len(rows) != 1: + raise LookupError("order missing") + row = rows[0] + identity = ReferenceIdentity("Order", row["id"], row["version"]) + scope = ReferenceDocumentScope("edit-10", "edit-order", "Order", + row["id"], row["version"]) + snapshot = DocumentSnapshot("model-v1", business_id, scope, identity, + (DocumentEntitySnapshot( + "order:10", identity, frozenset({"quantity"}), + frozenset({"quantity"}), frozenset({"UPDATE"})),), ()) + return CurrentBoundDocument(BoundDocumentSnapshot(snapshot, ()), + {"quantity": row["quantity"]}, 300) + + async def apply(self, _context, issued, submitted, sql): + self.mutations += 1 + if submitted.body != {"quantity": 3}: + raise ValueError("invalid quantity") + affected, _ = await sql.execute_sql(CompiledQuery( + "UPDATE document_order SET quantity=?,version=version+1 WHERE id=? AND version=?", + [Value.I64(3), Value.I64(10), + Value.I64(issued.snapshot.scope.aggregate_revision)])) + if affected != 1: + raise RuntimeError("stale order") + if self.switch_actor_on_apply: + self.actor = replace(self.actor, subject="bob") + + async def present(self, _context, document: ContextualDocument): + assert document.business_id == "ORD-10" + if self.fail_present: + raise RuntimeError("presentation failed") + + +async def fixture(tmp_path): + path = str(tmp_path / "secure-document.sqlite") + async with aiosqlite.connect(path) as db: + await db.execute("""CREATE TABLE document_order (id INTEGER PRIMARY KEY, + business_id TEXT NOT NULL UNIQUE,version INTEGER NOT NULL,quantity INTEGER NOT NULL)""") + await db.execute("CREATE TABLE document_permission(subject TEXT PRIMARY KEY,can_edit INTEGER NOT NULL)") + await db.execute("INSERT INTO document_order VALUES (10,'ORD-10',1,1)") + await db.execute("INSERT INTO document_permission VALUES ('alice',1)") + await db.commit() + transport = SqliteTransport(path) + context = RuntimeModule.new().into_context() + context.insert_resource("dataService", SqlDataServiceExecutor( + SqliteDialect(), transport, SimpleSchemaProvider())) + keys = StaticReferenceKeyProvider("k1", (ReferenceKey("k1", bytes([0x42]) * 32),)) + module = SecureDocumentReferenceModule( + "order-service", "test", keys, lambda *_: None, + lambda: datetime.fromtimestamp(1800000000, timezone.utc), + lambda: bytes([0x33]) * 12) + host = OrderHost() + journal = SQLiteDocumentCommandJournal(transport) + await journal.ensure_schema() + await journal.validate_schema() + service = SQLiteContextBoundDocumentService(context, transport, module, journal, host) + return path, context, host, service + + +async def row(path): + async with aiosqlite.connect(path) as db: + async with db.execute("SELECT version,quantity FROM document_order WHERE id=10") as cursor: + return await cursor.fetchone() + + +async def receipts(path): + async with aiosqlite.connect(path) as db: + async with db.execute("SELECT COUNT(*) FROM teaql_document_command_receipt") as cursor: + return (await cursor.fetchone())[0] + + +@pytest.mark.asyncio +async def test_sqlite_open_accept_replay_authority_and_rollback(tmp_path): + path, context, host, service = await fixture(tmp_path) + opened = await service.open(context, "Order", "ORD-10", "edit-order", 60) + assert opened.document_token.startswith("tqd2.") and opened.body == {"quantity": 1} + assert opened.document_token not in repr(opened) + command = SubmittedDocumentCommand("ORD-10", opened.document_token, + "command-1", {"quantity": 3}) + first = await service.accept(context, "Order", "edit-order", command) + assert (first.previous_revision, first.new_revision, first.replayed) == (1, 2, False) + assert first.document.body == {"quantity": 3} + assert first.document.document_token not in repr(first) + assert await row(path) == (2, 3) and await receipts(path) == 1 + + replay = await service.accept(context, "Order", "edit-order", command) + assert replay.replayed and replay.document.document_token == first.document.document_token + assert host.mutations == 1 + with pytest.raises(RuntimeError, match="DOCUMENT_REPLAY_CONFLICT"): + await service.accept(context, "Order", "edit-order", + replace(command, body={"quantity": 4})) + with pytest.raises(RuntimeError, match="DOCUMENT_REVISION_CONFLICT"): + await service.accept(context, "Order", "edit-order", + replace(command, command_id="command-stale")) + + async with aiosqlite.connect(path) as db: + await db.execute("UPDATE document_permission SET can_edit=0 WHERE subject='alice'") + await db.commit() + with pytest.raises(RuntimeError, match="DOCUMENT_AUTHORIZATION_REQUIRED"): + await service.accept(context, "Order", "edit-order", command) + with pytest.raises(RuntimeError, match="DOCUMENT_AUTHORIZATION_REQUIRED"): + await service.open(context, "Order", "ORD-10", "edit-order", 60) + async with aiosqlite.connect(path) as db: + await db.execute("UPDATE document_permission SET can_edit=1 WHERE subject='alice'") + await db.commit() + + latest = await service.open(context, "Order", "ORD-10", "edit-order", 60) + followup = SubmittedDocumentCommand("ORD-10", latest.document_token, + "command-2", {"quantity": 3}) + host.fail_present = True + with pytest.raises(GraphCommittedError): + await service.accept(context, "Order", "edit-order", followup) + host.fail_present = False + recovered = await service.accept(context, "Order", "edit-order", followup) + assert recovered.replayed and recovered.new_revision == 3 and host.mutations == 2 + + async with aiosqlite.connect(path) as db: + await db.execute("""CREATE TRIGGER reject_document_receipt BEFORE INSERT + ON teaql_document_command_receipt WHEN NEW.command_id='command-trigger' + BEGIN SELECT RAISE(ABORT, 'receipt rejected'); END""") + await db.commit() + latest = await service.open(context, "Order", "ORD-10", "edit-order", 60) + with pytest.raises(RuntimeError, match="DOCUMENT_CATALOG_FAILED"): + await service.accept(context, "Order", "edit-order", SubmittedDocumentCommand( + "ORD-10", latest.document_token, "command-trigger", {"quantity": 3})) + assert await row(path) == (3, 3) and await receipts(path) == 2 + + host.switch_actor_on_apply = True + with pytest.raises(RuntimeError, match="DOCUMENT_AUTHORIZATION_REQUIRED"): + await service.accept(context, "Order", "edit-order", SubmittedDocumentCommand( + "ORD-10", latest.document_token, "command-actor-drift", {"quantity": 3})) + host.actor = replace(host.actor, subject="alice") + assert await row(path) == (3, 3) and await receipts(path) == 2 + + +@pytest.mark.asyncio +async def test_sqlite_document_journal_rejects_missing_replay_primary_key(tmp_path): + path = str(tmp_path / "broken-receipt.sqlite") + transport = SqliteTransport(path) + await transport.execute_sql(CompiledQuery("""CREATE TABLE teaql_document_command_receipt ( + scope_hash TEXT NOT NULL,actor_hash TEXT NOT NULL,command_id TEXT NOT NULL, + request_hash TEXT NOT NULL,previous_revision BIGINT NOT NULL, + new_revision BIGINT NOT NULL,response TEXT NOT NULL)""", [])) + journal = SQLiteDocumentCommandJournal(transport) + await journal.ensure_schema() + with pytest.raises(RuntimeError, match="DOCUMENT_SCHEMA_MISSING"): + await journal.validate_schema() From 62173511e08ce553272fda680d8d0a649be06347 Mon Sep 17 00:00:00 2001 From: Philip Z Date: Tue, 27 Oct 2026 20:57:29 +0800 Subject: [PATCH 18/75] Bind sealed document Blob selections on current SQLite graph lease --- .../teaql_blob_reference/sqlite_catalog.py | 11 +- .../__init__.py | 6 +- .../sqlite_document_service.py | 94 +++++++++++- .../tests/test_sqlite_document_service.py | 143 +++++++++++++++++- src/teaql/runtime/graph_session.py | 6 + src/teaql/sql/executor.py | 5 + 6 files changed, 256 insertions(+), 9 deletions(-) diff --git a/packages/teaql-blob-reference/src/teaql_blob_reference/sqlite_catalog.py b/packages/teaql-blob-reference/src/teaql_blob_reference/sqlite_catalog.py index 0a5f1bb8..9b37c7ac 100644 --- a/packages/teaql-blob-reference/src/teaql_blob_reference/sqlite_catalog.py +++ b/packages/teaql-blob-reference/src/teaql_blob_reference/sqlite_catalog.py @@ -157,7 +157,7 @@ async def current_read(self, slot: AuthorizedBlobSlot) -> BlobCatalogRead: return BlobCatalogRead(*values, size) async def bind_in_graph(self, graph, actor, change: BlobBindingChange, - authorization: BlobBindingAuthorization) -> int: + authorization: BlobBindingAuthorization, *, lease=None) -> int: if graph is None or not callable(getattr(graph, "with_sql_transport", None)): raise RuntimeError("BLOB_TRANSACTION_REQUIRED") @@ -256,4 +256,13 @@ async def bind(sql): raise RuntimeError("BLOB_CATALOG_FAILED") from None return change.next_version + if lease is not None: + if (actor is not graph.context or + not callable(getattr(graph, "is_current_sql_lease", None)) or + not graph.is_current_sql_lease(lease)): + raise RuntimeError("BLOB_TRANSACTION_REQUIRED") + # A secure-document journal already borrowed this physical lease. + # Reentering the owner here would reject or deadlock; its outer + # callback propagates failures and remains rollback-only. + return await bind(lease) return await graph.with_sql_transport(bind) diff --git a/packages/teaql-secure-document-reference/src/teaql_secure_document_reference/__init__.py b/packages/teaql-secure-document-reference/src/teaql_secure_document_reference/__init__.py index 410047f3..0313ea6e 100644 --- a/packages/teaql-secure-document-reference/src/teaql_secure_document_reference/__init__.py +++ b/packages/teaql-secure-document-reference/src/teaql_secure_document_reference/__init__.py @@ -13,7 +13,8 @@ ) from .sqlite_document_service import ( AcceptedDocument, ContextualDocument, CurrentBoundDocument, - DocumentCommandReceipt, SQLiteContextBoundDocumentService, + DocumentBlobSelectionSubmission, DocumentCommandReceipt, + VerifiedDocumentBlobSelection, SQLiteContextBoundDocumentService, SQLiteDocumentCommandJournal, SubmittedDocumentCommand, ) @@ -26,6 +27,7 @@ "DocumentSnapshot", "PreparedBoundDocument", "SecureDocumentAssembly", "SecureDocumentReferenceModule", "VerifiedDocument", "AcceptedDocument", "ContextualDocument", "CurrentBoundDocument", - "DocumentCommandReceipt", "SQLiteContextBoundDocumentService", + "DocumentBlobSelectionSubmission", "DocumentCommandReceipt", + "VerifiedDocumentBlobSelection", "SQLiteContextBoundDocumentService", "SQLiteDocumentCommandJournal", "SubmittedDocumentCommand", ] diff --git a/packages/teaql-secure-document-reference/src/teaql_secure_document_reference/sqlite_document_service.py b/packages/teaql-secure-document-reference/src/teaql_secure_document_reference/sqlite_document_service.py index e58b8f50..f3041d84 100644 --- a/packages/teaql-secure-document-reference/src/teaql_secure_document_reference/sqlite_document_service.py +++ b/packages/teaql-secure-document-reference/src/teaql_secure_document_reference/sqlite_document_service.py @@ -13,7 +13,8 @@ from teaql.sql.types import CompiledQuery from .document_round_trip import ( - BoundDocumentSnapshot, SecureDocumentReferenceModule, _bindings, _encode_snapshot, + BlobSelectionTarget, BoundDocumentSnapshot, SecureDocumentReferenceModule, + _bindings, _encode_snapshot, ) from .round_trip_reference import ReferenceDocumentScope, TrustedReferencePrincipal @@ -86,11 +87,24 @@ class SubmittedDocumentCommand: document_token: str command_id: str body: dict[str, Any] + blob_selections: tuple[DocumentBlobSelectionSubmission, ...] | None = None def __repr__(self) -> str: return "SubmittedDocumentCommand[redacted]" +@dataclass(frozen=True, repr=False) +class DocumentBlobSelectionSubmission: + token: str + target_entity: str + canonical_field: str + kind: str + target_key: str + + def __repr__(self) -> str: + return "DocumentBlobSelectionSubmission[redacted]" + + @dataclass(frozen=True, repr=False) class DocumentCommandReceipt: previous_revision: int @@ -108,9 +122,24 @@ def _prepared(command: SubmittedDocumentCommand) -> tuple[str, SubmittedDocument not _text(command.command_id, 256)): _invalid("DOCUMENT_COMMAND_INVALID") copied = _copy_body(command.body) - return _hash_parts(command.business_id, command.document_token, - command.command_id, copied), SubmittedDocumentCommand( - command.business_id, command.document_token, command.command_id, copied) + selections = command.blob_selections + if selections is not None: + if type(selections) is not tuple or len(selections) > 100: + _invalid("DOCUMENT_COMMAND_INVALID") + for selection in selections: + if (not isinstance(selection, DocumentBlobSelectionSubmission) or + not all(_text(value, _MAX_BYTES if index == 0 else 16384) + for index, value in enumerate((selection.token, + selection.target_entity, selection.canonical_field, + selection.target_key))) or + selection.kind not in {"EXISTING_ROW", "NEW_CLIENT_KEY"}): + _invalid("DOCUMENT_COMMAND_INVALID") + # Keep the original digest for receipts written without selections. + parts = [command.business_id, command.document_token, command.command_id, copied] + if selections is not None: + parts.append([selection.__dict__ for selection in selections]) + return _hash_parts(*parts), SubmittedDocumentCommand( + command.business_id, command.document_token, command.command_id, copied, selections) def _receipt(value: DocumentCommandReceipt) -> DocumentCommandReceipt: @@ -257,6 +286,49 @@ class CurrentBoundDocument: lifetime_seconds: int +@dataclass(frozen=True, repr=False) +class VerifiedDocumentBlobSelection: + target: BlobSelectionTarget + stable_key: str + entity_id: str + expected_version: int + next_version: int + + def __repr__(self) -> str: + return "VerifiedDocumentBlobSelection[redacted]" + + +def _verified_selections(module: SecureDocumentReferenceModule, + principal: TrustedReferencePrincipal, issued: BoundDocumentSnapshot, + command: SubmittedDocumentCommand) -> tuple[VerifiedDocumentBlobSelection, ...]: + result = [] + seen = set() + for selected in command.blob_selections or (): + target = BlobSelectionTarget(issued.snapshot.scope, selected.target_entity, + selected.canonical_field, selected.kind, + selected.target_key, command.command_id) + pair = (target.target_key, target.canonical_field) + if pair in seen: + _invalid("DOCUMENT_COMMAND_INVALID") + seen.add(pair) + claim = module.consume_blob_selection(principal, command.document_token, + selected.token, target).document + if target.kind != "EXISTING_ROW": + _invalid("DOCUMENT_CAPABILITY_MISSING") + row = next((item for item in issued.snapshot.entities + if item.row_key == target.target_key), None) + binding = next((item for item in issued.bindings + if item.row_key == target.target_key and + item.canonical_field == target.canonical_field), None) + if (row is None or binding is None or row.identity.entity_type != target.target_entity or + not binding.writable or binding.binding_version >= (1 << 63) - 1): + _invalid("DOCUMENT_PROJECTION_VIOLATION") + result.append(VerifiedDocumentBlobSelection(target, claim.stable_key, + str(row.identity.id), binding.binding_version, + binding.binding_version + 1)) + return tuple(result) + + class SQLiteContextBoundDocumentHost(Protocol): def principal(self, context) -> TrustedReferencePrincipal: ... async def authorize(self, context, principal: TrustedReferencePrincipal, @@ -365,7 +437,13 @@ async def mutate(sql, frozen): _valid_current(before, frozen.business_id, aggregate_type, purpose) if _bound_fingerprint(before.document) != _bound_fingerprint(issued): _invalid("DOCUMENT_REVISION_CONFLICT") + selections = _verified_selections(self.module, principal, issued, frozen) + binder = getattr(self.host, "bind_blob_selection", None) + if selections and not callable(binder): + _invalid("DOCUMENT_CAPABILITY_MISSING") await self.host.apply(context, issued, frozen, sql) + for selection in selections: + await binder(context, graph, selection, sql) after = await self.host.load(context, frozen.business_id, sql) _valid_current(after, frozen.business_id, aggregate_type, purpose) next_scope = after.document.snapshot.scope @@ -375,6 +453,14 @@ async def mutate(sql, frozen): next_scope.purpose != scope.purpose or next_scope.aggregate_revision < scope.aggregate_revision): _invalid("DOCUMENT_PROJECTION_VIOLATION") + for selection in selections: + binding = next((item for item in after.document.bindings + if item.row_key == selection.target.target_key and + item.canonical_field == selection.target.canonical_field), None) + if (binding is None or binding.entity_type != selection.target.target_entity or + binding.stable_key != selection.stable_key or + binding.binding_version != selection.next_version): + _invalid("DOCUMENT_PROJECTION_VIOLATION") try: await self.host.authorize(context, principal, next_scope, sql) except Exception: diff --git a/packages/teaql-secure-document-reference/tests/test_sqlite_document_service.py b/packages/teaql-secure-document-reference/tests/test_sqlite_document_service.py index e59785a7..cb1e1a8b 100644 --- a/packages/teaql-secure-document-reference/tests/test_sqlite_document_service.py +++ b/packages/teaql-secure-document-reference/tests/test_sqlite_document_service.py @@ -7,6 +7,7 @@ import pytest from teaql.core.value import Value +from teaql.core.blob_ref import BlobRef from teaql.provider.sqlite.dialect import SqliteDialect from teaql.provider.sqlite.transport import SqliteTransport from teaql.runtime import RuntimeModule @@ -15,12 +16,16 @@ from teaql.sql.types import CompiledQuery from teaql.provider.sqlite import SimpleSchemaProvider from teaql_secure_document_reference import ( - BoundDocumentSnapshot, ContextualDocument, CurrentBoundDocument, + BlobSelectionClaim, BlobSelectionTarget, BoundDocumentSnapshot, + ContextualDocument, CurrentBoundDocument, DocumentBlobBindingSnapshot, + DocumentBlobSelectionSubmission, DocumentEntitySnapshot, DocumentSnapshot, ReferenceDocumentScope, - ReferenceIdentity, ReferenceKey, SecureDocumentReferenceModule, + ReferenceIdentity, ReferenceKey, RoundTripReferenceError, + SecureDocumentReferenceModule, SQLiteContextBoundDocumentService, SQLiteDocumentCommandJournal, StaticReferenceKeyProvider, SubmittedDocumentCommand, TrustedReferencePrincipal, ) +from teaql_blob_reference import BlobBindingChange, BlobBindingTarget, SqliteBlobCatalog class OrderHost: @@ -190,3 +195,137 @@ async def test_sqlite_document_journal_rejects_missing_replay_primary_key(tmp_pa await journal.ensure_schema() with pytest.raises(RuntimeError, match="DOCUMENT_SCHEMA_MISSING"): await journal.validate_schema() + + +class BlobOrderHost(OrderHost): + def __init__(self, catalog): + super().__init__() + self.catalog = catalog + + async def load(self, context, business_id, sql): + current = await super().load(context, business_id, sql) + rows = await sql.fetch_all_sql(CompiledQuery( + "SELECT blob_key FROM document_order WHERE business_id=?", [Value.Text(business_id)])) + bindings = await sql.fetch_all_sql(CompiledQuery( + "SELECT key,version FROM teaql_blob_binding WHERE scope=? AND entity=? " + "AND entity_id=? AND field=?", [Value.Text("merchant:7"), Value.Text("Order"), + Value.Text("10"), Value.Text("attachment")])) + assert len(rows) == 1 and len(bindings) <= 1 + binding = bindings[0] if bindings else {"key": None, "version": 0} + assert rows[0]["blob_key"] == binding["key"] + entity = current.document.snapshot.entities[0] + entity = replace(entity, loaded_fields=frozenset({"attachment", "quantity"}), + writable_fields=frozenset({"attachment", "quantity"})) + snapshot = replace(current.document.snapshot, entities=(entity,)) + bound = DocumentBlobBindingSnapshot("order:10", "attachment", "Order", + binding["key"], binding["version"], True, True) + return replace(current, document=BoundDocumentSnapshot(snapshot, (bound,))) + + async def bind_blob_selection(self, _context, graph, selection, sql): + assert (selection.target.target_entity, selection.target.target_key, + selection.target.canonical_field, selection.entity_id) == ( + "Order", "order:10", "attachment", "10") + affected, _ = await sql.execute_sql(CompiledQuery( + "UPDATE document_order SET blob_key=? WHERE id=?", + [Value.Text(selection.stable_key), Value.I64(10)])) + assert affected == 1 + + class Authorization: + def scope(self, _actor): + return "merchant:7" + + async def authorize(self, _actor, target, _previous, next_ref, _action): + return (target == BlobBindingTarget("Order", "10", "attachment") and + next_ref == BlobRef.from_sql(selection.stable_key)) + + await self.catalog.bind_in_graph(graph, graph.context, + BlobBindingChange(BlobBindingTarget("Order", "10", "attachment"), + BlobRef.from_sql(selection.stable_key), + selection.expected_version, selection.next_version, "Update"), + Authorization(), lease=sql) + + +@pytest.mark.asyncio +async def test_sqlite_document_blob_selection_shares_business_binding_and_receipt(tmp_path): + path, context, _old_host, old_service = await fixture(tmp_path) + async with aiosqlite.connect(path) as db: + await db.execute("ALTER TABLE document_order ADD COLUMN blob_key TEXT") + await db.commit() + catalog = SqliteBlobCatalog(old_service.transport) + await catalog.ensure_schema() + await catalog.validate_schema() + async with aiosqlite.connect(path) as db: + for key in ("blob_A", "blob_B"): + await db.execute("""INSERT INTO teaql_blob_catalog + (key,scope,object_key,content_version,file_name,content_type,size_bytes,status) + VALUES (?,?,?,?,?,?,?,'uploaded_unbound')""", + (key, "merchant:7", "objects/" + key, "v1", key + ".pdf", "application/pdf", 3)) + await db.commit() + host = BlobOrderHost(catalog) + service = SQLiteContextBoundDocumentService(context, old_service.transport, + old_service.module, old_service.journal, host) + + async def open_document(): + return await service.open(context, "Order", "ORD-10", "edit-order", 60) + + def selection(opened, command_id, key): + issued = service.module.consume_bound_document_snapshot(host.actor, + opened.document_token, "Order", "edit-order").document + target = BlobSelectionTarget(issued.snapshot.scope, "Order", "attachment", + "EXISTING_ROW", "order:10", command_id) + token = service.module.issue_blob_selection(host.actor, opened.document_token, + BlobSelectionClaim(target, key), 60) + return DocumentBlobSelectionSubmission(token, "Order", "attachment", + "EXISTING_ROW", "order:10") + + opened = await open_document() + command = SubmittedDocumentCommand("ORD-10", opened.document_token, "command-1", + {"quantity": 3}, (selection(opened, "command-1", "blob_A"),)) + first = await service.accept(context, "Order", "edit-order", command) + assert (first.previous_revision, first.new_revision, first.replayed) == (1, 2, False) + async with aiosqlite.connect(path) as db: + async with db.execute("SELECT version,quantity,blob_key FROM document_order") as cursor: + assert await cursor.fetchone() == (2, 3, "blob_A") + async with db.execute("SELECT key,version FROM teaql_blob_binding") as cursor: + assert await cursor.fetchone() == ("blob_A", 1) + async with db.execute("SELECT status FROM teaql_blob_catalog WHERE key='blob_A'") as cursor: + assert await cursor.fetchone() == ("available",) + assert await receipts(path) == 1 + replayed = await service.accept(context, "Order", "edit-order", command) + assert replayed.replayed and host.mutations == 1 + with pytest.raises(RuntimeError, match="DOCUMENT_REPLAY_CONFLICT"): + await service.accept(context, "Order", "edit-order", replace(command, blob_selections=None)) + + fresh = await open_document() + wrong = SubmittedDocumentCommand("ORD-10", fresh.document_token, "command-wrong", + {"quantity": 3}, (selection(fresh, "other-command", "blob_A"),)) + with pytest.raises(RoundTripReferenceError, match="ROUND_TRIP_REFERENCE_SCOPE_MISMATCH"): + await service.accept(context, "Order", "edit-order", wrong) + duplicate = selection(fresh, "command-duplicate", "blob_A") + with pytest.raises(RuntimeError, match="DOCUMENT_COMMAND_INVALID"): + await service.accept(context, "Order", "edit-order", SubmittedDocumentCommand( + "ORD-10", fresh.document_token, "command-duplicate", {"quantity": 3}, + (duplicate, duplicate))) + with pytest.raises(RuntimeError, match="BLOB_BIND_DENIED"): + await service.accept(context, "Order", "edit-order", SubmittedDocumentCommand( + "ORD-10", fresh.document_token, "command-missing", {"quantity": 3}, + (selection(fresh, "command-missing", "blob_missing"),))) + assert await row(path) == (2, 3) and await receipts(path) == 1 + + async with aiosqlite.connect(path) as db: + await db.execute("""CREATE TRIGGER reject_selected_receipt BEFORE INSERT + ON teaql_document_command_receipt WHEN NEW.command_id='command-receipt-fail' + BEGIN SELECT RAISE(ABORT, 'receipt rejected'); END""") + await db.commit() + with pytest.raises(RuntimeError, match="DOCUMENT_CATALOG_FAILED"): + await service.accept(context, "Order", "edit-order", SubmittedDocumentCommand( + "ORD-10", fresh.document_token, "command-receipt-fail", {"quantity": 3}, + (selection(fresh, "command-receipt-fail", "blob_B"),))) + async with aiosqlite.connect(path) as db: + async with db.execute("SELECT version,quantity,blob_key FROM document_order") as cursor: + assert await cursor.fetchone() == (2, 3, "blob_A") + async with db.execute("SELECT key,version FROM teaql_blob_binding") as cursor: + assert await cursor.fetchone() == ("blob_A", 1) + async with db.execute("SELECT status FROM teaql_blob_catalog WHERE key='blob_B'") as cursor: + assert await cursor.fetchone() == ("uploaded_unbound",) + assert await receipts(path) == 1 diff --git a/src/teaql/runtime/graph_session.py b/src/teaql/runtime/graph_session.py index 127e534e..bad77acc 100644 --- a/src/teaql/runtime/graph_session.py +++ b/src/teaql/runtime/graph_session.py @@ -57,6 +57,12 @@ async def with_sql_transport(self, work): raise RuntimeError('BLOB_TRANSACTION_REQUIRED') return await owner(work) + def is_current_sql_lease(self, lease): + """Check an already borrowed owner lease without entering the owner again.""" + return (self._active and not self._committed and + callable(getattr(self.transaction, 'is_current_physical_sql_lease', None)) and + self.transaction.is_current_physical_sql_lease(lease)) + def scope(self, key, parent=None, local_reason=None): self._require_active() if parent is None: diff --git a/src/teaql/sql/executor.py b/src/teaql/sql/executor.py index edac2a66..6ecd5518 100644 --- a/src/teaql/sql/executor.py +++ b/src/teaql/sql/executor.py @@ -1423,6 +1423,11 @@ async def with_physical_sql(self, work): lease.close() self._physical_sql_active = False + def is_current_physical_sql_lease(self, lease): + return (not self._completed and self._physical_sql_active and + isinstance(lease, _PhysicalSQLLease) and lease._active and + lease._transport is self.transport) + def capabilities(self) -> DataServiceCapabilities: return DataServiceCapabilities( query=True, From 63ec20eab48493952f5143d236eee0ef18e2f5cd Mon Sep 17 00:00:00 2001 From: Philip Z Date: Tue, 27 Oct 2026 21:12:52 +0800 Subject: [PATCH 19/75] Bound total document command bytes with Blob selections --- .../sqlite_document_service.py | 6 +++++- .../tests/test_sqlite_document_service.py | 5 +++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/packages/teaql-secure-document-reference/src/teaql_secure_document_reference/sqlite_document_service.py b/packages/teaql-secure-document-reference/src/teaql_secure_document_reference/sqlite_document_service.py index f3041d84..4d8e242d 100644 --- a/packages/teaql-secure-document-reference/src/teaql_secure_document_reference/sqlite_document_service.py +++ b/packages/teaql-secure-document-reference/src/teaql_secure_document_reference/sqlite_document_service.py @@ -138,7 +138,11 @@ def _prepared(command: SubmittedDocumentCommand) -> tuple[str, SubmittedDocument parts = [command.business_id, command.document_token, command.command_id, copied] if selections is not None: parts.append([selection.__dict__ for selection in selections]) - return _hash_parts(*parts), SubmittedDocumentCommand( + encoded = json.dumps(parts, ensure_ascii=False, separators=(",", ":"), + sort_keys=True).encode("utf-8") + if len(encoded) > _MAX_BYTES: + _invalid("DOCUMENT_COMMAND_INVALID") + return hashlib.sha256(encoded).hexdigest(), SubmittedDocumentCommand( command.business_id, command.document_token, command.command_id, copied, selections) diff --git a/packages/teaql-secure-document-reference/tests/test_sqlite_document_service.py b/packages/teaql-secure-document-reference/tests/test_sqlite_document_service.py index cb1e1a8b..91ea68d4 100644 --- a/packages/teaql-secure-document-reference/tests/test_sqlite_document_service.py +++ b/packages/teaql-secure-document-reference/tests/test_sqlite_document_service.py @@ -295,6 +295,11 @@ def selection(opened, command_id, key): assert replayed.replayed and host.mutations == 1 with pytest.raises(RuntimeError, match="DOCUMENT_REPLAY_CONFLICT"): await service.accept(context, "Order", "edit-order", replace(command, blob_selections=None)) + oversized = DocumentBlobSelectionSubmission("tqs1." + "A" * (256000 - 5), + "Order", "attachment", "EXISTING_ROW", "order:10") + with pytest.raises(RuntimeError, match="DOCUMENT_COMMAND_INVALID"): + await service.accept(context, "Order", "edit-order", + replace(command, blob_selections=(oversized,))) fresh = await open_document() wrong = SubmittedDocumentCommand("ORD-10", fresh.document_token, "command-wrong", From 4ea7dd3b0fa3927dfc3a0afa490c8c5070ae1901 Mon Sep 17 00:00:00 2001 From: Philip Z Date: Tue, 27 Oct 2026 21:26:15 +0800 Subject: [PATCH 20/75] Bind new document child blob selections in SQLite graph --- .../__init__.py | 4 +- .../sqlite_document_service.py | 95 +++++++++-- .../tests/test_sqlite_document_service.py | 157 +++++++++++++++++- 3 files changed, 237 insertions(+), 19 deletions(-) diff --git a/packages/teaql-secure-document-reference/src/teaql_secure_document_reference/__init__.py b/packages/teaql-secure-document-reference/src/teaql_secure_document_reference/__init__.py index 0313ea6e..d640d308 100644 --- a/packages/teaql-secure-document-reference/src/teaql_secure_document_reference/__init__.py +++ b/packages/teaql-secure-document-reference/src/teaql_secure_document_reference/__init__.py @@ -13,7 +13,7 @@ ) from .sqlite_document_service import ( AcceptedDocument, ContextualDocument, CurrentBoundDocument, - DocumentBlobSelectionSubmission, DocumentCommandReceipt, + DocumentBlobSelectionSubmission, DocumentBlobSelectionClaim, DocumentCommandReceipt, VerifiedDocumentBlobSelection, SQLiteContextBoundDocumentService, SQLiteDocumentCommandJournal, SubmittedDocumentCommand, ) @@ -27,7 +27,7 @@ "DocumentSnapshot", "PreparedBoundDocument", "SecureDocumentAssembly", "SecureDocumentReferenceModule", "VerifiedDocument", "AcceptedDocument", "ContextualDocument", "CurrentBoundDocument", - "DocumentBlobSelectionSubmission", "DocumentCommandReceipt", + "DocumentBlobSelectionSubmission", "DocumentBlobSelectionClaim", "DocumentCommandReceipt", "VerifiedDocumentBlobSelection", "SQLiteContextBoundDocumentService", "SQLiteDocumentCommandJournal", "SubmittedDocumentCommand", ] diff --git a/packages/teaql-secure-document-reference/src/teaql_secure_document_reference/sqlite_document_service.py b/packages/teaql-secure-document-reference/src/teaql_secure_document_reference/sqlite_document_service.py index 4d8e242d..236351f4 100644 --- a/packages/teaql-secure-document-reference/src/teaql_secure_document_reference/sqlite_document_service.py +++ b/packages/teaql-secure-document-reference/src/teaql_secure_document_reference/sqlite_document_service.py @@ -4,6 +4,7 @@ import hashlib import json +import re import unicodedata from dataclasses import dataclass from typing import Any, Protocol @@ -294,6 +295,7 @@ class CurrentBoundDocument: class VerifiedDocumentBlobSelection: target: BlobSelectionTarget stable_key: str + row_key: str entity_id: str expected_version: int next_version: int @@ -302,9 +304,18 @@ def __repr__(self) -> str: return "VerifiedDocumentBlobSelection[redacted]" +@dataclass(frozen=True, repr=False) +class DocumentBlobSelectionClaim: + target: BlobSelectionTarget + stable_key: str + + def __repr__(self) -> str: + return "DocumentBlobSelectionClaim[redacted]" + + def _verified_selections(module: SecureDocumentReferenceModule, principal: TrustedReferencePrincipal, issued: BoundDocumentSnapshot, - command: SubmittedDocumentCommand) -> tuple[VerifiedDocumentBlobSelection, ...]: + command: SubmittedDocumentCommand) -> tuple[DocumentBlobSelectionClaim, ...]: result = [] seen = set() for selected in command.blob_selections or (): @@ -317,22 +328,39 @@ def _verified_selections(module: SecureDocumentReferenceModule, seen.add(pair) claim = module.consume_blob_selection(principal, command.document_token, selected.token, target).document - if target.kind != "EXISTING_ROW": - _invalid("DOCUMENT_CAPABILITY_MISSING") - row = next((item for item in issued.snapshot.entities - if item.row_key == target.target_key), None) - binding = next((item for item in issued.bindings - if item.row_key == target.target_key and - item.canonical_field == target.canonical_field), None) - if (row is None or binding is None or row.identity.entity_type != target.target_entity or - not binding.writable or binding.binding_version >= (1 << 63) - 1): - _invalid("DOCUMENT_PROJECTION_VIOLATION") - result.append(VerifiedDocumentBlobSelection(target, claim.stable_key, - str(row.identity.id), binding.binding_version, - binding.binding_version + 1)) + result.append(DocumentBlobSelectionClaim(target, claim.stable_key)) return tuple(result) +def _mapped_existing(issued: BoundDocumentSnapshot, + selection: DocumentBlobSelectionClaim) -> VerifiedDocumentBlobSelection: + target = selection.target + row = next((item for item in issued.snapshot.entities + if item.row_key == target.target_key), None) + binding = next((item for item in issued.bindings + if item.row_key == target.target_key and + item.canonical_field == target.canonical_field), None) + if (row is None or binding is None or row.identity.entity_type != target.target_entity or + not binding.writable or binding.binding_version >= (1 << 63) - 1): + _invalid("DOCUMENT_PROJECTION_VIOLATION") + return VerifiedDocumentBlobSelection(target, selection.stable_key, target.target_key, + str(row.identity.id), binding.binding_version, + binding.binding_version + 1) + + +def _mapped_new(issued: BoundDocumentSnapshot, selection: DocumentBlobSelectionClaim, + resolved: tuple[str, str]) -> VerifiedDocumentBlobSelection: + if (type(resolved) is not tuple or len(resolved) != 2 or + not _text(resolved[0], 16384) or type(resolved[1]) is not str or + re.fullmatch(r"[1-9][0-9]*", resolved[1], re.ASCII) is None or + len(resolved[1]) > 20 or + int(resolved[1]) > (1 << 64) - 1 or + any(row.row_key == resolved[0] for row in issued.snapshot.entities)): + _invalid("DOCUMENT_PROJECTION_VIOLATION") + return VerifiedDocumentBlobSelection(selection.target, selection.stable_key, + resolved[0], resolved[1], 0, 1) + + class SQLiteContextBoundDocumentHost(Protocol): def principal(self, context) -> TrustedReferencePrincipal: ... async def authorize(self, context, principal: TrustedReferencePrincipal, @@ -340,6 +368,9 @@ async def authorize(self, context, principal: TrustedReferencePrincipal, async def load(self, context, business_id: str, sql) -> CurrentBoundDocument: ... async def apply(self, context, issued: BoundDocumentSnapshot, submitted: SubmittedDocumentCommand, sql) -> None: ... + async def resolve_new_blob_selection(self, context, issued: BoundDocumentSnapshot, + submitted: SubmittedDocumentCommand, selection: DocumentBlobSelectionClaim, + sql) -> tuple[str, str]: ... async def present(self, context, document: ContextualDocument) -> None: ... @@ -445,8 +476,24 @@ async def mutate(sql, frozen): binder = getattr(self.host, "bind_blob_selection", None) if selections and not callable(binder): _invalid("DOCUMENT_CAPABILITY_MISSING") + resolver = getattr(self.host, "resolve_new_blob_selection", None) + if any(s.target.kind == "NEW_CLIENT_KEY" for s in selections) and not callable(resolver): + _invalid("DOCUMENT_CAPABILITY_MISSING") await self.host.apply(context, issued, frozen, sql) + mapped = [] + mapped_targets = set() for selection in selections: + if selection.target.kind == "EXISTING_ROW": + bound = _mapped_existing(issued, selection) + else: + bound = _mapped_new(issued, selection, + await resolver(context, issued, frozen, selection, sql)) + pair = (bound.row_key, bound.target.canonical_field) + if pair in mapped_targets: + _invalid("DOCUMENT_COMMAND_INVALID") + mapped_targets.add(pair) + mapped.append(bound) + for selection in mapped: await binder(context, graph, selection, sql) after = await self.host.load(context, frozen.business_id, sql) _valid_current(after, frozen.business_id, aggregate_type, purpose) @@ -457,14 +504,30 @@ async def mutate(sql, frozen): next_scope.purpose != scope.purpose or next_scope.aggregate_revision < scope.aggregate_revision): _invalid("DOCUMENT_PROJECTION_VIOLATION") - for selection in selections: + for selection in mapped: binding = next((item for item in after.document.bindings - if item.row_key == selection.target.target_key and + if item.row_key == selection.row_key and item.canonical_field == selection.target.canonical_field), None) if (binding is None or binding.entity_type != selection.target.target_entity or binding.stable_key != selection.stable_key or binding.binding_version != selection.next_version): _invalid("DOCUMENT_PROJECTION_VIOLATION") + if selection.target.kind == "NEW_CLIENT_KEY": + child = next((row for row in after.document.snapshot.entities + if row.row_key == selection.row_key), None) + if (child is None or child.identity.entity_type != selection.target.target_entity or + str(child.identity.id) != selection.entity_id or + selection.target.canonical_field not in child.writable_fields or + not any(relation.completeness == "COMPLETE" and relation.writable and + "CREATE_CHILD" in relation.allowed_mutations and + selection.row_key in relation.row_keys and + any(previous.name == relation.name and + previous.completeness == "COMPLETE" and previous.writable and + "CREATE_CHILD" in previous.allowed_mutations and + selection.row_key not in previous.row_keys + for previous in issued.snapshot.relations) + for relation in after.document.snapshot.relations)): + _invalid("DOCUMENT_PROJECTION_VIOLATION") try: await self.host.authorize(context, principal, next_scope, sql) except Exception: diff --git a/packages/teaql-secure-document-reference/tests/test_sqlite_document_service.py b/packages/teaql-secure-document-reference/tests/test_sqlite_document_service.py index 91ea68d4..7a75d29b 100644 --- a/packages/teaql-secure-document-reference/tests/test_sqlite_document_service.py +++ b/packages/teaql-secure-document-reference/tests/test_sqlite_document_service.py @@ -19,7 +19,7 @@ BlobSelectionClaim, BlobSelectionTarget, BoundDocumentSnapshot, ContextualDocument, CurrentBoundDocument, DocumentBlobBindingSnapshot, DocumentBlobSelectionSubmission, - DocumentEntitySnapshot, DocumentSnapshot, ReferenceDocumentScope, + DocumentEntitySnapshot, DocumentRelationSnapshot, DocumentSnapshot, ReferenceDocumentScope, ReferenceIdentity, ReferenceKey, RoundTripReferenceError, SecureDocumentReferenceModule, SQLiteContextBoundDocumentService, SQLiteDocumentCommandJournal, @@ -245,6 +245,161 @@ async def authorize(self, _actor, target, _previous, next_ref, _action): Authorization(), lease=sql) +class ChildBlobOrderHost(BlobOrderHost): + def __init__(self, catalog): + super().__init__(catalog) + self.wrong_resolve = False + + async def load(self, context, business_id, sql): + current = await super().load(context, business_id, sql) + children = await sql.fetch_all_sql(CompiledQuery( + "SELECT id,version,blob_key FROM document_order_line WHERE order_id=? ORDER BY id", + [Value.I64(10)])) + entities = list(current.document.snapshot.entities) + bindings = list(current.document.bindings) + row_keys = [] + for child in children: + row_key = f"line:{child['id']}" + row_keys.append(row_key) + actual = await sql.fetch_all_sql(CompiledQuery( + "SELECT key,version FROM teaql_blob_binding WHERE scope=? AND entity=? " + "AND entity_id=? AND field=?", [Value.Text("merchant:7"), Value.Text("OrderLine"), + Value.Text(str(child["id"])), Value.Text("attachment")])) + assert len(actual) <= 1 + binding = actual[0] if actual else {"key": None, "version": 0} + assert child["blob_key"] == binding["key"] + entities.append(DocumentEntitySnapshot(row_key, + ReferenceIdentity("OrderLine", child["id"], child["version"]), + frozenset({"attachment"}), frozenset({"attachment"}), + frozenset({"UPDATE"}))) + bindings.append(DocumentBlobBindingSnapshot(row_key, "attachment", "OrderLine", + binding["key"], binding["version"], True, True)) + relation = DocumentRelationSnapshot("lines", tuple(row_keys), "COMPLETE", True, + frozenset({"CREATE_CHILD"})) + snapshot = replace(current.document.snapshot, entities=tuple(entities), + relations=(relation,)) + return replace(current, document=BoundDocumentSnapshot(snapshot, tuple(bindings))) + + async def apply(self, context, issued, submitted, sql): + await super().apply(context, issued, submitted, sql) + for selection in submitted.blob_selections or (): + if selection.kind != "NEW_CLIENT_KEY": + continue + assert selection.target_entity == "OrderLine" and selection.canonical_field == "attachment" + assert selection.target_key in {"client-line-1", "client-line-2"} + child_id = 21 if selection.target_key == "client-line-1" else 22 + affected, _ = await sql.execute_sql(CompiledQuery( + "INSERT INTO document_order_line(id,order_id,client_key,version,blob_key) " + "VALUES (?,?,?,?,NULL)", [Value.I64(child_id), Value.I64(10), + Value.Text(selection.target_key), Value.I64(1)])) + assert affected == 1 + + async def resolve_new_blob_selection(self, _context, _issued, _submitted, selection, sql): + if self.wrong_resolve: + return "order:10", "10" + children = await sql.fetch_all_sql(CompiledQuery( + "SELECT id FROM document_order_line WHERE order_id=? AND client_key=?", + [Value.I64(10), Value.Text(selection.target.target_key)])) + assert len(children) == 1 + return f"line:{children[0]['id']}", str(children[0]["id"]) + + async def bind_blob_selection(self, context, graph, selection, sql): + if selection.target.kind != "NEW_CLIENT_KEY": + return await super().bind_blob_selection(context, graph, selection, sql) + assert selection.target.target_entity == "OrderLine" and selection.row_key == f"line:{selection.entity_id}" + affected, _ = await sql.execute_sql(CompiledQuery( + "UPDATE document_order_line SET blob_key=? WHERE id=?", + [Value.Text(selection.stable_key), Value.I64(int(selection.entity_id))])) + assert affected == 1 + + class Authorization: + def scope(self, _actor): + return "merchant:7" + + async def authorize(self, _actor, target, _previous, next_ref, action): + return (target == BlobBindingTarget("OrderLine", selection.entity_id, "attachment") + and next_ref == BlobRef.from_sql(selection.stable_key) and action == "Create") + + await self.catalog.bind_in_graph(graph, graph.context, + BlobBindingChange(BlobBindingTarget("OrderLine", selection.entity_id, "attachment"), + BlobRef.from_sql(selection.stable_key), + selection.expected_version, selection.next_version, "Create"), + Authorization(), lease=sql) + + +@pytest.mark.asyncio +async def test_sqlite_new_child_blob_selection_commits_and_receipt_failure_rolls_back(tmp_path): + path, context, _old_host, old_service = await fixture(tmp_path) + async with aiosqlite.connect(path) as db: + await db.execute("ALTER TABLE document_order ADD COLUMN blob_key TEXT") + await db.execute("""CREATE TABLE document_order_line ( + id INTEGER PRIMARY KEY, order_id INTEGER NOT NULL, client_key TEXT UNIQUE NOT NULL, + version INTEGER NOT NULL, blob_key TEXT)""") + await db.commit() + catalog = SqliteBlobCatalog(old_service.transport) + await catalog.ensure_schema() + await catalog.validate_schema() + async with aiosqlite.connect(path) as db: + for key in ("blob_C", "blob_D"): + await db.execute("""INSERT INTO teaql_blob_catalog + (key,scope,object_key,content_version,file_name,content_type,size_bytes,status) + VALUES (?,?,?,?,?,?,?,'uploaded_unbound')""", + (key, "merchant:7", "objects/" + key, "v1", key + ".pdf", "application/pdf", 3)) + await db.commit() + host = ChildBlobOrderHost(catalog) + service = SQLiteContextBoundDocumentService(context, old_service.transport, + old_service.module, old_service.journal, host) + + async def selected(opened, command_id, client_key, blob_key): + issued = service.module.consume_bound_document_snapshot(host.actor, + opened.document_token, "Order", "edit-order").document + target = BlobSelectionTarget(issued.snapshot.scope, "OrderLine", "attachment", + "NEW_CLIENT_KEY", client_key, command_id) + token = service.module.issue_blob_selection(host.actor, opened.document_token, + BlobSelectionClaim(target, blob_key), 60) + return DocumentBlobSelectionSubmission(token, "OrderLine", "attachment", + "NEW_CLIENT_KEY", client_key) + + opened = await service.open(context, "Order", "ORD-10", "edit-order", 60) + command = SubmittedDocumentCommand("ORD-10", opened.document_token, "command-child-1", + {"quantity": 3}, (await selected(opened, "command-child-1", "client-line-1", "blob_C"),)) + host.wrong_resolve = True + with pytest.raises(RuntimeError, match="DOCUMENT_PROJECTION_VIOLATION"): + await service.accept(context, "Order", "edit-order", command) + assert await row(path) == (1, 1) and await receipts(path) == 0 + host.wrong_resolve = False + first = await service.accept(context, "Order", "edit-order", command) + assert (first.previous_revision, first.new_revision, first.replayed) == (1, 2, False) + assert (await service.accept(context, "Order", "edit-order", command)).replayed + async with aiosqlite.connect(path) as db: + async with db.execute("SELECT id,version,blob_key FROM document_order_line") as cursor: + assert await cursor.fetchone() == (21, 1, "blob_C") + async with db.execute("SELECT key,version FROM teaql_blob_binding WHERE entity='OrderLine'") as cursor: + assert await cursor.fetchone() == ("blob_C", 1) + async with db.execute("SELECT status FROM teaql_blob_catalog WHERE key='blob_C'") as cursor: + assert await cursor.fetchone() == ("available",) + assert await row(path) == (2, 3) and await receipts(path) == 1 + + opened = await service.open(context, "Order", "ORD-10", "edit-order", 60) + failed = SubmittedDocumentCommand("ORD-10", opened.document_token, "command-child-2", + {"quantity": 3}, (await selected(opened, "command-child-2", "client-line-2", "blob_D"),)) + async with aiosqlite.connect(path) as db: + await db.execute("""CREATE TRIGGER reject_child_receipt BEFORE INSERT + ON teaql_document_command_receipt WHEN NEW.command_id='command-child-2' + BEGIN SELECT RAISE(ABORT, 'receipt rejected'); END""") + await db.commit() + with pytest.raises(RuntimeError, match="DOCUMENT_CATALOG_FAILED"): + await service.accept(context, "Order", "edit-order", failed) + async with aiosqlite.connect(path) as db: + async with db.execute("SELECT COUNT(*) FROM document_order_line") as cursor: + assert await cursor.fetchone() == (1,) + async with db.execute("SELECT COUNT(*) FROM teaql_blob_binding WHERE entity_id='22'") as cursor: + assert await cursor.fetchone() == (0,) + async with db.execute("SELECT status FROM teaql_blob_catalog WHERE key='blob_D'") as cursor: + assert await cursor.fetchone() == ("uploaded_unbound",) + assert await row(path) == (2, 3) and await receipts(path) == 1 + + @pytest.mark.asyncio async def test_sqlite_document_blob_selection_shares_business_binding_and_receipt(tmp_path): path, context, _old_host, old_service = await fixture(tmp_path) From 411651274f3da9a2f302596e69482cd80c30932f Mon Sep 17 00:00:00 2001 From: Philip Z Date: Tue, 27 Oct 2026 22:03:48 +0800 Subject: [PATCH 21/75] Fence expired SQLite Blob upload sessions --- .../teaql_blob_reference/upload_service.py | 24 +++++++++++++++---- tests/provider/sqlite/test_blob_upload.py | 21 ++++++++++++++++ 2 files changed, 40 insertions(+), 5 deletions(-) diff --git a/packages/teaql-blob-reference/src/teaql_blob_reference/upload_service.py b/packages/teaql-blob-reference/src/teaql_blob_reference/upload_service.py index 08da116a..1f2aaa90 100644 --- a/packages/teaql-blob-reference/src/teaql_blob_reference/upload_service.py +++ b/packages/teaql-blob-reference/src/teaql_blob_reference/upload_service.py @@ -214,8 +214,13 @@ async def work(tx): raise RuntimeError("BLOB_IDEMPOTENCY_CONFLICT") await self._current(actor, target, "Begin", normalized, scope) try: + expires_ms = row["expires_ms"] + if type(expires_ms) is not int: + raise RuntimeError("BLOB_CATALOG_FAILED") + if self.now_ms() >= expires_ms: + raise RuntimeError("BLOB_UPLOAD_EXPIRED") return BlobUploadSession(row["upload_id"], BlobRef(row["key"]), - datetime.fromtimestamp(row["expires_ms"] / 1000, timezone.utc)) + datetime.fromtimestamp(expires_ms / 1000, timezone.utc)) except (KeyError, TypeError, ValueError, OverflowError): raise RuntimeError("BLOB_CATALOG_FAILED") from None reference = BlobRef("blob_" + _token()) @@ -319,12 +324,18 @@ async def write(self, actor, upload_id: str, source: BinaryIO) -> int: digest = reader.digest.hexdigest() async def work(tx): + now = self.now_ms() + if now >= row["expires_ms"]: + raise RuntimeError("BLOB_UPLOAD_EXPIRED") try: affected, _ = await tx.execute_sql(_query( "UPDATE teaql_blob_upload SET state='written', written_sha256=?, written_etag=? " - "WHERE upload_id=? AND scope=? AND state='initiated'", + "WHERE upload_id=? AND scope=? AND state='initiated' AND expires_ms > ? " + "AND EXISTS (SELECT 1 FROM teaql_blob_catalog " + "WHERE key=teaql_blob_upload.key AND scope=teaql_blob_upload.scope " + "AND status='initiated')", Value.Text(digest), Value.Text(metadata.etag), - Value.Text(upload_id), Value.Text(row["scope"]))) + Value.Text(upload_id), Value.Text(row["scope"]), Value.I64(now))) except Exception: raise RuntimeError("BLOB_CATALOG_FAILED") from None if affected != 1: @@ -418,12 +429,15 @@ def verify_stored(): raise RuntimeError("BLOB_UPLOAD_EXPIRED") async def work(tx): + now = self.now_ms() + if now >= row["expires_ms"]: + raise RuntimeError("BLOB_UPLOAD_EXPIRED") try: affected, _ = await tx.execute_sql(_query( "UPDATE teaql_blob_upload SET state='completed' WHERE upload_id=? AND scope=? " - "AND state='written' AND written_sha256=? AND written_etag=?", + "AND state='written' AND written_sha256=? AND written_etag=? AND expires_ms > ?", Value.Text(upload_id), Value.Text(row["scope"]), - Value.Text(row["written_sha256"]), Value.Text(row["written_etag"]))) + Value.Text(row["written_sha256"]), Value.Text(row["written_etag"]), Value.I64(now))) if affected != 1: raise RuntimeError("BLOB_IDEMPOTENCY_CONFLICT") affected, _ = await tx.execute_sql(_query( diff --git a/tests/provider/sqlite/test_blob_upload.py b/tests/provider/sqlite/test_blob_upload.py index f30d073e..a6fc1abb 100644 --- a/tests/provider/sqlite/test_blob_upload.py +++ b/tests/provider/sqlite/test_blob_upload.py @@ -64,6 +64,27 @@ def verify(self, stream, extension, maximum_bytes): return BlobVerifiedContent(len(data), sha256(data).hexdigest(), "application/pdf") +@pytest.mark.asyncio +async def test_expired_replay_key_cannot_recover_upload_session(tmp_path): + catalog = SqliteBlobCatalog(SqliteTransport(str(tmp_path / "expired.db"))) + await catalog.ensure_schema() + clock = [1_700_000_000_000] + service = SqliteBlobUploadService(catalog, UploadAuthority(), FixturePDFVerifier(), + 10_000, lease_seconds=1, now_ms=lambda: clock[0]) + context = RuntimeModule.new().into_context() + target = BlobUploadTarget("Asset", "1", "document") + input = BlobUploadInput("report.pdf", 42) + first = await service.begin(context, target, input, "expired-request") + assert (await service.begin(context, target, input, "expired-request")).upload_id == first.upload_id + clock[0] = int(first.expires_at.timestamp() * 1000) + with pytest.raises(RuntimeError, match="BLOB_UPLOAD_EXPIRED"): + await service.begin(context, target, input, "expired-request") + second = await service.begin(context, target, input, "fresh-request") + assert second.upload_id != first.upload_id + with pytest.raises(RuntimeError, match="BLOB_UPLOAD_EXPIRED"): + await service.write(context, first.upload_id, BytesIO(b"x" * 42)) + + @pytest.mark.asyncio async def test_upload_real_sqlite_file_verifier_and_native_bind(tmp_path): path = str(tmp_path / "upload.db") From 26c144abd0113cf90340a41db27e1bc89f242150 Mon Sep 17 00:00:00 2001 From: Philip Z Date: Tue, 27 Oct 2026 22:15:20 +0800 Subject: [PATCH 22/75] Hold SQLite Blob upload owner across object I/O --- .../teaql_blob_reference/upload_service.py | 95 ++++++++++++++----- tests/provider/sqlite/test_blob_upload.py | 92 ++++++++++++++++++ 2 files changed, 163 insertions(+), 24 deletions(-) diff --git a/packages/teaql-blob-reference/src/teaql_blob_reference/upload_service.py b/packages/teaql-blob-reference/src/teaql_blob_reference/upload_service.py index 1f2aaa90..e9eab54d 100644 --- a/packages/teaql-blob-reference/src/teaql_blob_reference/upload_service.py +++ b/packages/teaql-blob-reference/src/teaql_blob_reference/upload_service.py @@ -307,26 +307,49 @@ async def write(self, actor, upload_id: str, source: BinaryIO) -> int: key = await self._object_key(actor, row) mime = blob_mime_for_extension(row["extension"]) reader = _CountingReader(source, row["declared_size"]) - try: - metadata = await asyncio.to_thread( - actor.object_storage("write Blob upload").put, key, reader, - PutOptions(row["declared_size"], WriteMode.CREATE_ONLY, mime)) - except Exception: - raise RuntimeError("BLOB_STORE_FAILED") from None target = BlobUploadTarget(row["entity"], row["entity_id"], row["field"]) - await self._current(actor, target, "Write", - BlobUploadInput(row["file_name"], row["declared_size"]), row["scope"]) - if self.now_ms() >= row["expires_ms"]: - raise RuntimeError("BLOB_UPLOAD_EXPIRED") - if (not reader.eof or reader.count != row["declared_size"] or - metadata.size != reader.count or not metadata.etag): - raise RuntimeError("BLOB_UPLOAD_INCOMPLETE") - digest = reader.digest.hexdigest() async def work(tx): + # BEGIN IMMEDIATE remains owned until the File write and journal + # transition finish. A second writer or collector must recheck + # the authoritative row after this owner commits. + try: + current = await tx.fetch_all_sql(_query( + "SELECT state,expires_ms,key FROM teaql_blob_upload " + "WHERE upload_id=? AND scope=?", Value.Text(upload_id), + Value.Text(row["scope"]))) + catalog = await tx.fetch_all_sql(_query( + "SELECT object_key,status FROM teaql_blob_catalog WHERE key=? AND scope=?", + Value.Text(row["key"]), Value.Text(row["scope"]))) + except Exception: + raise RuntimeError("BLOB_CATALOG_FAILED") from None + if (len(current) != 1 or current[0]["state"] != "initiated" or + current[0]["key"] != row["key"] or + current[0]["expires_ms"] != row["expires_ms"] or + len(catalog) != 1 or catalog[0]["status"] != "initiated" or + catalog[0]["object_key"] != key.value): + raise RuntimeError("BLOB_IDEMPOTENCY_CONFLICT") + await self._current(actor, target, "Write", + BlobUploadInput(row["file_name"], row["declared_size"]), + row["scope"]) + if self.now_ms() >= row["expires_ms"]: + raise RuntimeError("BLOB_UPLOAD_EXPIRED") + try: + metadata = await asyncio.to_thread( + actor.object_storage("write Blob upload").put, key, reader, + PutOptions(row["declared_size"], WriteMode.CREATE_ONLY, mime)) + except Exception: + raise RuntimeError("BLOB_STORE_FAILED") from None + await self._current(actor, target, "Write", + BlobUploadInput(row["file_name"], row["declared_size"]), + row["scope"]) now = self.now_ms() if now >= row["expires_ms"]: raise RuntimeError("BLOB_UPLOAD_EXPIRED") + if (not reader.eof or reader.count != row["declared_size"] or + metadata.size != reader.count or not metadata.etag): + raise RuntimeError("BLOB_UPLOAD_INCOMPLETE") + digest = reader.digest.hexdigest() try: affected, _ = await tx.execute_sql(_query( "UPDATE teaql_blob_upload SET state='written', written_sha256=?, written_etag=? " @@ -340,8 +363,8 @@ async def work(tx): raise RuntimeError("BLOB_CATALOG_FAILED") from None if affected != 1: raise RuntimeError("BLOB_IDEMPOTENCY_CONFLICT") - await self._transaction(work) - return reader.count + return reader.count + return await self._transaction(work) finally: with suppress(Exception): source.close() @@ -420,15 +443,39 @@ def verify_stored(): except Exception: raise RuntimeError("BLOB_UPLOAD_INCOMPLETE") from None - verified = await asyncio.to_thread(verify_stored) target = BlobUploadTarget(row["entity"], row["entity_id"], row["field"]) - await self._current(actor, target, "Confirm", - BlobUploadInput(row["file_name"], row["declared_size"], - verified.content_type), row["scope"]) - if self.now_ms() >= row["expires_ms"]: - raise RuntimeError("BLOB_UPLOAD_EXPIRED") async def work(tx): + # Keep the same physical SQLite owner through parser I/O and both + # journal promotions. A concurrent confirmation rechecks after it. + try: + current = await tx.fetch_all_sql(_query( + "SELECT state,expires_ms,key,written_sha256,written_etag " + "FROM teaql_blob_upload WHERE upload_id=? AND scope=?", + Value.Text(upload_id), Value.Text(row["scope"]))) + catalog = await tx.fetch_all_sql(_query( + "SELECT object_key,status FROM teaql_blob_catalog WHERE key=? AND scope=?", + Value.Text(row["key"]), Value.Text(row["scope"]))) + except Exception: + raise RuntimeError("BLOB_CATALOG_FAILED") from None + if (len(current) != 1 or current[0]["state"] != "written" or + current[0]["expires_ms"] != row["expires_ms"] or + current[0]["key"] != row["key"] or + current[0]["written_sha256"] != row["written_sha256"] or + current[0]["written_etag"] != row["written_etag"] or + len(catalog) != 1 or catalog[0]["status"] != "initiated" or + catalog[0]["object_key"] != key.value): + raise RuntimeError("BLOB_IDEMPOTENCY_CONFLICT") + await self._current(actor, target, "Confirm", + BlobUploadInput(row["file_name"], row["declared_size"]), + row["scope"]) + now = self.now_ms() + if now >= row["expires_ms"]: + raise RuntimeError("BLOB_UPLOAD_EXPIRED") + verified = await asyncio.to_thread(verify_stored) + await self._current(actor, target, "Confirm", + BlobUploadInput(row["file_name"], row["declared_size"], + verified.content_type), row["scope"]) now = self.now_ms() if now >= row["expires_ms"]: raise RuntimeError("BLOB_UPLOAD_EXPIRED") @@ -452,5 +499,5 @@ async def work(tx): raise except Exception: raise RuntimeError("BLOB_CATALOG_FAILED") from None - await self._transaction(work) - return BlobRef(row["key"]) + return BlobRef(row["key"]) + return await self._transaction(work) diff --git a/tests/provider/sqlite/test_blob_upload.py b/tests/provider/sqlite/test_blob_upload.py index a6fc1abb..2db0d693 100644 --- a/tests/provider/sqlite/test_blob_upload.py +++ b/tests/provider/sqlite/test_blob_upload.py @@ -2,6 +2,8 @@ from hashlib import sha256 from io import BytesIO +from threading import Event +import asyncio import os import aiosqlite @@ -85,6 +87,96 @@ async def test_expired_replay_key_cannot_recover_upload_session(tmp_path): await service.write(context, first.upload_id, BytesIO(b"x" * 42)) +@pytest.mark.asyncio +async def test_upload_write_holds_sql_owner_across_file_io(tmp_path): + path = str(tmp_path / "locked.db") + catalog = SqliteBlobCatalog(SqliteTransport(path)) + await catalog.ensure_schema() + entered, release = Event(), Event() + file_store = FileObjectStorage(tmp_path / "private", os.urandom(32)) + + class BlockingStore: + puts = 0 + + def __getattr__(self, name): + return getattr(file_store, name) + + def put(self, invocation, key, content, options): + self.puts += 1 + result = file_store.put(invocation, key, content, options) + entered.set() + if not release.wait(5): + raise AssertionError("test did not release File write") + return result + + store = BlockingStore() + actor = RuntimeModule.new().into_context().with_object_storage(store) + actor.with_object_storage_observer(lambda event: None) + service = SqliteBlobUploadService(catalog, UploadAuthority(), FixturePDFVerifier(), 10_000) + data = b"%PDF-1.7\nlocked upload\n%%EOF" + session = await service.begin(actor, BlobUploadTarget("Asset", "1", "document"), + BlobUploadInput("report.pdf", len(data)), "lock-request") + first = asyncio.create_task(service.write(actor, session.upload_id, BytesIO(data))) + second = None + try: + assert await asyncio.to_thread(entered.wait, 5) + second = asyncio.create_task(service.write(actor, session.upload_id, BytesIO(data))) + await asyncio.sleep(0.05) + assert not second.done(), "second writer bypassed the physical SQL owner" + finally: + release.set() + assert await first == len(data) + with pytest.raises(RuntimeError, match="BLOB_IDEMPOTENCY_CONFLICT"): + await second + assert store.puts == 1 + async with aiosqlite.connect(path) as db: + async with db.execute("SELECT state FROM teaql_blob_upload WHERE upload_id=?", + (session.upload_id,)) as cursor: + assert (await cursor.fetchone())[0] == "written" + + +@pytest.mark.asyncio +async def test_upload_confirmation_holds_sql_owner_through_parser_io(tmp_path): + path = str(tmp_path / "confirm-locked.db") + catalog = SqliteBlobCatalog(SqliteTransport(path)) + await catalog.ensure_schema() + entered, release = Event(), Event() + + class BlockingVerifier(FixturePDFVerifier): + def verify(self, stream, extension, maximum_bytes): + result = super().verify(stream, extension, maximum_bytes) + entered.set() + if not release.wait(5): + raise AssertionError("test did not release parser") + return result + + verifier = BlockingVerifier() + actor = RuntimeModule.new().into_context().with_object_storage( + FileObjectStorage(tmp_path / "private", os.urandom(32))) + actor.with_object_storage_observer(lambda event: None) + service = SqliteBlobUploadService(catalog, UploadAuthority(), verifier, 10_000) + data = b"%PDF-1.7\nlocked confirmation\n%%EOF" + session = await service.begin(actor, BlobUploadTarget("Asset", "1", "document"), + BlobUploadInput("report.pdf", len(data)), "confirm-request") + await service.write(actor, session.upload_id, BytesIO(data)) + first = asyncio.create_task(service.complete(actor, session.upload_id)) + second = None + try: + assert await asyncio.to_thread(entered.wait, 5) + second = asyncio.create_task(service.complete(actor, session.upload_id)) + await asyncio.sleep(0.05) + assert not second.done(), "second confirmation bypassed the physical SQL owner" + finally: + release.set() + assert await first == session.reference + try: + assert await second == session.reference + except RuntimeError as error: + assert str(error) == "BLOB_IDEMPOTENCY_CONFLICT" + assert await service.complete(actor, session.upload_id) == session.reference + assert verifier.calls == 1 + + @pytest.mark.asyncio async def test_upload_real_sqlite_file_verifier_and_native_bind(tmp_path): path = str(tmp_path / "upload.db") From a28e796e268ed7f4420d7040367a54dc5cb83bbf Mon Sep 17 00:00:00 2001 From: Philip Z Date: Tue, 27 Oct 2026 22:21:52 +0800 Subject: [PATCH 23/75] fix(blob): replay concurrent completed confirmation after SQL owner --- .../src/teaql_blob_reference/upload_service.py | 12 +++++++++++- tests/provider/sqlite/test_blob_upload.py | 7 ++----- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/packages/teaql-blob-reference/src/teaql_blob_reference/upload_service.py b/packages/teaql-blob-reference/src/teaql_blob_reference/upload_service.py index e9eab54d..ffcad855 100644 --- a/packages/teaql-blob-reference/src/teaql_blob_reference/upload_service.py +++ b/packages/teaql-blob-reference/src/teaql_blob_reference/upload_service.py @@ -458,6 +458,15 @@ async def work(tx): Value.Text(row["key"]), Value.Text(row["scope"]))) except Exception: raise RuntimeError("BLOB_CATALOG_FAILED") from None + if (len(current) == 1 and current[0]["state"] == "completed" and + current[0]["expires_ms"] == row["expires_ms"] and + current[0]["key"] == row["key"] and + current[0]["written_sha256"] == row["written_sha256"] and + current[0]["written_etag"] == row["written_etag"]): + # Another confirmer won the lease. Reenter the regular replay + # path after releasing this SQL owner so catalog, generation, + # authorization and expiry are checked against current state. + return None if (len(current) != 1 or current[0]["state"] != "written" or current[0]["expires_ms"] != row["expires_ms"] or current[0]["key"] != row["key"] or @@ -500,4 +509,5 @@ async def work(tx): except Exception: raise RuntimeError("BLOB_CATALOG_FAILED") from None return BlobRef(row["key"]) - return await self._transaction(work) + result = await self._transaction(work) + return result if result is not None else await self.complete(actor, upload_id) diff --git a/tests/provider/sqlite/test_blob_upload.py b/tests/provider/sqlite/test_blob_upload.py index 2db0d693..c79ca6bb 100644 --- a/tests/provider/sqlite/test_blob_upload.py +++ b/tests/provider/sqlite/test_blob_upload.py @@ -169,11 +169,8 @@ def verify(self, stream, extension, maximum_bytes): finally: release.set() assert await first == session.reference - try: - assert await second == session.reference - except RuntimeError as error: - assert str(error) == "BLOB_IDEMPOTENCY_CONFLICT" - assert await service.complete(actor, session.upload_id) == session.reference + assert await second == session.reference + assert await service.complete(actor, session.upload_id) == session.reference assert verifier.calls == 1 From 6811c85e1ca108f912df3fe1f3c0fd534fa13308 Mon Sep 17 00:00:00 2001 From: Philip Z Date: Tue, 27 Oct 2026 22:28:00 +0800 Subject: [PATCH 24/75] fix(blob): recover identical file after upload journal rollback --- .../teaql_blob_reference/upload_service.py | 45 ++++++++++++++++++- tests/provider/sqlite/test_blob_upload.py | 38 ++++++++++++++++ 2 files changed, 81 insertions(+), 2 deletions(-) diff --git a/packages/teaql-blob-reference/src/teaql_blob_reference/upload_service.py b/packages/teaql-blob-reference/src/teaql_blob_reference/upload_service.py index ffcad855..470741b5 100644 --- a/packages/teaql-blob-reference/src/teaql_blob_reference/upload_service.py +++ b/packages/teaql-blob-reference/src/teaql_blob_reference/upload_service.py @@ -19,7 +19,9 @@ import time from teaql.core.blob_ref import BlobFieldSpec, BlobRef, blob_mime_for_extension -from teaql.core.object_storage import MAX_PUT_SIZE, ObjectKey, PutOptions, WriteMode +from teaql.core.object_storage import ( + MAX_PUT_SIZE, ObjectKey, PutOptions, StorageError, StorageErrorCode, WriteMode, +) from teaql.core.value import Value from teaql.sql.executor import SqlTransactionTransport from teaql.sql.types import CompiledQuery @@ -334,10 +336,49 @@ async def work(tx): row["scope"]) if self.now_ms() >= row["expires_ms"]: raise RuntimeError("BLOB_UPLOAD_EXPIRED") + store = actor.object_storage("write Blob upload") try: metadata = await asyncio.to_thread( - actor.object_storage("write Blob upload").put, key, reader, + store.put, key, reader, PutOptions(row["declared_size"], WriteMode.CREATE_ONLY, mime)) + except StorageError as error: + if error.code != StorageErrorCode.CONFLICT: + raise RuntimeError("BLOB_STORE_FAILED") from None + + def recover_existing(): + # A prior put can have committed while its SQL promotion + # rolled back. Only the same complete source may adopt it. + if not reader.eof or reader.count != row["declared_size"]: + raise RuntimeError("BLOB_STORE_FAILED") + before = store.head(key) + if (before is None or before.size != reader.count or + before.content_type != mime or not before.etag): + raise RuntimeError("BLOB_STORE_FAILED") + digest, count = sha256(), 0 + with store.get(key) as content: + if (content.metadata.size != before.size or + content.metadata.etag != before.etag or + content.metadata.content_type != mime): + raise RuntimeError("BLOB_STORE_FAILED") + while True: + block = content.stream.read(65536) + if not isinstance(block, bytes) or count + len(block) > reader.count: + raise RuntimeError("BLOB_STORE_FAILED") + if not block: + break + digest.update(block) + count += len(block) + after = store.head(key) + if (count != reader.count or digest.hexdigest() != reader.digest.hexdigest() + or after is None or after.size != before.size or + after.etag != before.etag or after.content_type != mime): + raise RuntimeError("BLOB_STORE_FAILED") + return before + + try: + metadata = await asyncio.to_thread(recover_existing) + except Exception: + raise RuntimeError("BLOB_STORE_FAILED") from None except Exception: raise RuntimeError("BLOB_STORE_FAILED") from None await self._current(actor, target, "Write", diff --git a/tests/provider/sqlite/test_blob_upload.py b/tests/provider/sqlite/test_blob_upload.py index c79ca6bb..e4d09f99 100644 --- a/tests/provider/sqlite/test_blob_upload.py +++ b/tests/provider/sqlite/test_blob_upload.py @@ -174,6 +174,44 @@ def verify(self, stream, extension, maximum_bytes): assert verifier.calls == 1 +@pytest.mark.asyncio +async def test_upload_retry_recovers_only_identical_file_after_journal_rollback(tmp_path): + path = str(tmp_path / "recover.db") + catalog = SqliteBlobCatalog(SqliteTransport(path)) + await catalog.ensure_schema() + actor = RuntimeModule.new().into_context().with_object_storage( + FileObjectStorage(tmp_path / "private", os.urandom(32))) + actor.with_object_storage_observer(lambda event: None) + verifier = FixturePDFVerifier() + service = SqliteBlobUploadService(catalog, UploadAuthority(), verifier, 10_000) + data = b"%PDF-1.7\nrecover committed File content\n%%EOF" + session = await service.begin(actor, BlobUploadTarget("Asset", "1", "document"), + BlobUploadInput("report.pdf", len(data)), "recovery-request") + async with aiosqlite.connect(path) as db: + await db.execute("""CREATE TRIGGER reject_upload_write BEFORE UPDATE ON teaql_blob_upload + WHEN NEW.state='written' BEGIN SELECT RAISE(ABORT, 'injected'); END""") + await db.commit() + with pytest.raises(RuntimeError, match="BLOB_CATALOG_FAILED"): + await service.write(actor, session.upload_id, BytesIO(data)) + async with aiosqlite.connect(path) as db: + async with db.execute("SELECT c.object_key,u.state FROM teaql_blob_upload u " + "JOIN teaql_blob_catalog c ON c.key=u.key WHERE u.upload_id=?", + (session.upload_id,)) as cursor: + object_key, state = await cursor.fetchone() + assert state == "initiated" + await db.execute("DROP TRIGGER reject_upload_write") + await db.commit() + stored = actor.object_storage("check recovery candidate").head(ObjectKey(object_key)) + assert stored is not None and stored.size == len(data) + with pytest.raises(RuntimeError, match="BLOB_STORE_FAILED"): + await service.write(actor, session.upload_id, BytesIO(b"x" * len(data))) + assert await service.write(actor, session.upload_id, BytesIO(data)) == len(data) + assert actor.object_storage("check recovered generation").head( + ObjectKey(object_key)).etag == stored.etag + assert await service.complete(actor, session.upload_id) == session.reference + assert verifier.calls == 1 + + @pytest.mark.asyncio async def test_upload_real_sqlite_file_verifier_and_native_bind(tmp_path): path = str(tmp_path / "upload.db") From c582193e9c2bdc9aefe9fed638405db464444b93 Mon Sep 17 00:00:00 2001 From: Philip Z Date: Tue, 27 Oct 2026 23:44:08 +0800 Subject: [PATCH 25/75] test: verify S3 signed range and expiry against local MinIO --- tests/runtime/test_object_storage.py | 41 ++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/tests/runtime/test_object_storage.py b/tests/runtime/test_object_storage.py index b5712553..9e14a662 100644 --- a/tests/runtime/test_object_storage.py +++ b/tests/runtime/test_object_storage.py @@ -2,6 +2,11 @@ from io import BytesIO from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from threading import Thread +from urllib.error import HTTPError +from urllib.request import Request, urlopen +from uuid import uuid4 +import os +import time import pytest import boto3 from botocore.config import Config @@ -146,3 +151,39 @@ def read(self, *_): server.shutdown() server.server_close() provider.close() + + +def test_minio_real_signed_range_and_expiry(): + """Explicit local S3-compatible gate; the host owns the disposable bucket.""" + endpoint = os.environ.get("TEAQL_S3_TEST_ENDPOINT") + bucket = os.environ.get("TEAQL_S3_TEST_BUCKET") + access = os.environ.get("TEAQL_S3_TEST_ACCESS_KEY") + secret = os.environ.get("TEAQL_S3_TEST_SECRET_KEY") + if not all((endpoint, bucket, access, secret)): + pytest.skip("explicit S3-compatible endpoint, bucket and credentials required") + provider = S3ObjectStorage.connect(bucket, endpoint_url=endpoint, + aws_access_key_id=access, aws_secret_access_key=secret) + storage = UserContext().with_object_storage(provider).object_storage("F4 real S3-compatible Python") + key = ObjectKey("f4/python/" + uuid4().hex) + content = b"%PDF-1.7\nF4 real S3-compatible bytes\n%%EOF" + try: + written = storage.put_bytes(key, content, PutOptions(len(content), WriteMode.REPLACE, "application/pdf")) + head = storage.head(key) + assert head is not None and head.size == len(content) + assert head.etag == written.etag and head.content_type == "application/pdf" + assert storage.get_bytes(key, 1024) == content + ticket = storage.get_read_url(key, ReadUrlOptions(timedelta(seconds=2), "attachment", "report.pdf")) + # Do not log the signed URL or credentials. + with urlopen(Request(ticket.url, headers={"Range": "bytes=0-7"}), timeout=5) as response: + assert response.status == 206 and response.read() == content[:8] + assert response.headers["Content-Type"] == "application/pdf" + assert "report.pdf" in response.headers["Content-Disposition"] + time.sleep(max(0, ticket.expires_at.timestamp() - time.time() + 1.3)) + with pytest.raises(HTTPError) as expired: + urlopen(ticket.url, timeout=5) + assert expired.value.code == 403 + finally: + try: + storage.delete(key) + finally: + provider.close() From 888273a9662c7c12080af0ef9c95c4de886c279e Mon Sep 17 00:00:00 2001 From: Philip Z Date: Wed, 28 Oct 2026 00:22:47 +0800 Subject: [PATCH 26/75] fix: replay committed Blob upload writes on original lease --- .../teaql_blob_reference/upload_service.py | 82 ++++++++++++++++-- tests/provider/sqlite/test_blob_upload.py | 83 ++++++++++++++++++- 2 files changed, 156 insertions(+), 9 deletions(-) diff --git a/packages/teaql-blob-reference/src/teaql_blob_reference/upload_service.py b/packages/teaql-blob-reference/src/teaql_blob_reference/upload_service.py index 470741b5..72921f04 100644 --- a/packages/teaql-blob-reference/src/teaql_blob_reference/upload_service.py +++ b/packages/teaql-blob-reference/src/teaql_blob_reference/upload_service.py @@ -113,6 +113,10 @@ def close(self): self.stream.close() +class _ReplayMismatch(Exception): + pass + + class SqliteBlobUploadService: """Host-invoked upload path; schema migration remains explicit.""" @@ -304,8 +308,6 @@ async def write(self, actor, upload_id: str, source: BinaryIO) -> int: raise RuntimeError("BLOB_UPLOAD_INCOMPLETE") try: row = await self._row(actor, upload_id, "Write") - if row["state"] != "initiated": - raise RuntimeError("BLOB_IDEMPOTENCY_CONFLICT") key = await self._object_key(actor, row) mime = blob_mime_for_extension(row["extension"]) reader = _CountingReader(source, row["declared_size"]) @@ -317,20 +319,86 @@ async def work(tx): # the authoritative row after this owner commits. try: current = await tx.fetch_all_sql(_query( - "SELECT state,expires_ms,key FROM teaql_blob_upload " + "SELECT state,expires_ms,key,written_sha256,written_etag FROM teaql_blob_upload " "WHERE upload_id=? AND scope=?", Value.Text(upload_id), Value.Text(row["scope"]))) catalog = await tx.fetch_all_sql(_query( - "SELECT object_key,status FROM teaql_blob_catalog WHERE key=? AND scope=?", + "SELECT object_key,status,content_version,content_type,size_bytes " + "FROM teaql_blob_catalog WHERE key=? AND scope=?", Value.Text(row["key"]), Value.Text(row["scope"]))) except Exception: raise RuntimeError("BLOB_CATALOG_FAILED") from None - if (len(current) != 1 or current[0]["state"] != "initiated" or - current[0]["key"] != row["key"] or + if (len(current) != 1 or current[0]["key"] != row["key"] or current[0]["expires_ms"] != row["expires_ms"] or - len(catalog) != 1 or catalog[0]["status"] != "initiated" or + len(catalog) != 1 or catalog[0]["object_key"] != key.value): raise RuntimeError("BLOB_IDEMPOTENCY_CONFLICT") + state = current[0]["state"] + if state in ("written", "completed"): + recorded_sha = current[0]["written_sha256"] + recorded_etag = current[0]["written_etag"] + if (not isinstance(recorded_sha, str) or + re.fullmatch(r"[0-9a-f]{64}", recorded_sha) is None or + not isinstance(recorded_etag, str) or not recorded_etag or + (state == "written" and catalog[0]["status"] != "initiated") or + (state == "completed" and ( + catalog[0]["status"] not in ("uploaded_unbound", "available") or + catalog[0]["content_version"] != recorded_etag or + catalog[0]["content_type"] != mime or + catalog[0]["size_bytes"] != row["declared_size"]))): + raise RuntimeError("BLOB_IDEMPOTENCY_CONFLICT") + await self._current(actor, target, "Write", + BlobUploadInput(row["file_name"], row["declared_size"]), + row["scope"]) + if self.now_ms() >= row["expires_ms"]: + raise RuntimeError("BLOB_UPLOAD_EXPIRED") + + def verify_replay(): + try: + while reader.read(65536): + pass + if (not reader.eof or reader.count != row["declared_size"] or + reader.digest.hexdigest() != recorded_sha): + raise _ReplayMismatch() + store = actor.object_storage("recheck written Blob upload") + before = store.head(key) + if (before is None or before.size != reader.count or + before.etag != recorded_etag or before.content_type != mime): + raise _ReplayMismatch() + digest, count = sha256(), 0 + with store.get(key) as content: + if (content.metadata.size != before.size or + content.metadata.etag != before.etag or + content.metadata.content_type != mime): + raise _ReplayMismatch() + while True: + block = content.stream.read(65536) + if not isinstance(block, bytes) or count + len(block) > reader.count: + raise _ReplayMismatch() + if not block: + break + digest.update(block) + count += len(block) + after = store.head(key) + if (count != reader.count or digest.hexdigest() != recorded_sha or + after is None or after.size != before.size or + after.etag != before.etag or after.content_type != mime): + raise _ReplayMismatch() + return count + except _ReplayMismatch: + raise RuntimeError("BLOB_IDEMPOTENCY_CONFLICT") from None + except Exception: + raise RuntimeError("BLOB_STORE_FAILED") from None + + count = await asyncio.to_thread(verify_replay) + await self._current(actor, target, "Write", + BlobUploadInput(row["file_name"], row["declared_size"]), + row["scope"]) + if self.now_ms() >= row["expires_ms"]: + raise RuntimeError("BLOB_UPLOAD_EXPIRED") + return count + if state != "initiated" or catalog[0]["status"] != "initiated": + raise RuntimeError("BLOB_IDEMPOTENCY_CONFLICT") await self._current(actor, target, "Write", BlobUploadInput(row["file_name"], row["declared_size"]), row["scope"]) diff --git a/tests/provider/sqlite/test_blob_upload.py b/tests/provider/sqlite/test_blob_upload.py index e4d09f99..484cec94 100644 --- a/tests/provider/sqlite/test_blob_upload.py +++ b/tests/provider/sqlite/test_blob_upload.py @@ -126,8 +126,7 @@ def put(self, invocation, key, content, options): finally: release.set() assert await first == len(data) - with pytest.raises(RuntimeError, match="BLOB_IDEMPOTENCY_CONFLICT"): - await second + assert await second == len(data) assert store.puts == 1 async with aiosqlite.connect(path) as db: async with db.execute("SELECT state FROM teaql_blob_upload WHERE upload_id=?", @@ -174,6 +173,86 @@ def verify(self, stream, extension, maximum_bytes): assert verifier.calls == 1 +@pytest.mark.asyncio +async def test_upload_write_commit_unknown_replays_only_same_bytes_and_generation(tmp_path): + path = str(tmp_path / "commit-unknown.db") + catalog = SqliteBlobCatalog(SqliteTransport(path)) + await catalog.ensure_schema() + file_store = FileObjectStorage(tmp_path / "private", os.urandom(32)) + + class CountingStore: + puts = 0 + + def __getattr__(self, name): + return getattr(file_store, name) + + def put(self, invocation, key, content, options): + self.puts += 1 + return file_store.put(invocation, key, content, options) + + store = CountingStore() + actor = RuntimeModule.new().into_context().with_object_storage(store) + actor.with_object_storage_observer(lambda event: None) + authority, verifier = UploadAuthority(), FixturePDFVerifier() + service = SqliteBlobUploadService(catalog, authority, verifier, 10_000) + data = b"%PDF-1.7\nSQL commit succeeded, reply lost\n%%EOF" + session = await service.begin(actor, BlobUploadTarget("Asset", "1", "document"), + BlobUploadInput("report.pdf", len(data)), "unknown-write") + begin_sql = catalog.transport.begin_sql + + class CommitThenFail: + def __init__(self, tx): + self.tx = tx + + def __getattr__(self, name): + return getattr(self.tx, name) + + async def commit_sql(self): + await self.tx.commit_sql() + raise OSError("PRIVATE-COMMIT-CANARY") + + async def uncertain_begin(): + return CommitThenFail(await begin_sql()) + + catalog.transport.begin_sql = uncertain_begin + try: + with pytest.raises(RuntimeError, match="^BLOB_CATALOG_FAILED$"): + await service.write(actor, session.upload_id, BytesIO(data)) + finally: + catalog.transport.begin_sql = begin_sql + async with aiosqlite.connect(path) as db: + async with db.execute("SELECT u.state,c.status,c.object_key,u.written_etag " + "FROM teaql_blob_upload u JOIN teaql_blob_catalog c ON c.key=u.key " + "WHERE u.upload_id=?", (session.upload_id,)) as cursor: + state, catalog_state, object_key, etag = await cursor.fetchone() + assert state == "written" and catalog_state == "initiated" and etag + assert store.puts == 1 + authority.allowed = False + with pytest.raises(RuntimeError, match="^BLOB_UPLOAD_DENIED$"): + await service.write(actor, session.upload_id, BytesIO(data)) + authority.allowed = True + with pytest.raises(RuntimeError, match="^BLOB_IDEMPOTENCY_CONFLICT$"): + await service.write(actor, session.upload_id, BytesIO(b"x" * len(data))) + class FailingSource(BytesIO): + def read(self, *_): + raise OSError("PRIVATE-SOURCE-CANARY") + + with pytest.raises(RuntimeError, match="^BLOB_STORE_FAILED$") as failed: + await service.write(actor, session.upload_id, FailingSource(data)) + assert "CANARY" not in str(failed.value) + assert await service.write(actor, session.upload_id, BytesIO(data)) == len(data) + assert store.puts == 1 + assert actor.object_storage("check unchanged version").head(ObjectKey(object_key)).etag == etag + assert await service.complete(actor, session.upload_id) == session.reference + assert verifier.calls == 1 + assert await service.write(actor, session.upload_id, BytesIO(data)) == len(data) + assert verifier.calls == 1 and store.puts == 1 + actor.object_storage("replace generation").put_bytes(ObjectKey(object_key), data, + PutOptions(len(data), WriteMode.REPLACE, "application/pdf")) + with pytest.raises(RuntimeError, match="^BLOB_IDEMPOTENCY_CONFLICT$"): + await service.write(actor, session.upload_id, BytesIO(data)) + + @pytest.mark.asyncio async def test_upload_retry_recovers_only_identical_file_after_journal_rollback(tmp_path): path = str(tmp_path / "recover.db") From 562f01a8c747892d5a31a2fbfc7c054c00c987e4 Mon Sep 17 00:00:00 2001 From: Philip Z Date: Wed, 28 Oct 2026 00:48:25 +0800 Subject: [PATCH 27/75] test(blob): verify committed confirmation reply loss --- tests/provider/sqlite/test_blob_upload.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/tests/provider/sqlite/test_blob_upload.py b/tests/provider/sqlite/test_blob_upload.py index 484cec94..c325b56d 100644 --- a/tests/provider/sqlite/test_blob_upload.py +++ b/tests/provider/sqlite/test_blob_upload.py @@ -243,6 +243,20 @@ def read(self, *_): assert await service.write(actor, session.upload_id, BytesIO(data)) == len(data) assert store.puts == 1 assert actor.object_storage("check unchanged version").head(ObjectKey(object_key)).etag == etag + catalog.transport.begin_sql = uncertain_begin + try: + with pytest.raises(RuntimeError, match="^BLOB_CATALOG_FAILED$"): + await service.complete(actor, session.upload_id) + finally: + catalog.transport.begin_sql = begin_sql + async with aiosqlite.connect(path) as db: + async with db.execute("SELECT state FROM teaql_blob_upload WHERE upload_id=?", + (session.upload_id,)) as cursor: + assert (await cursor.fetchone())[0] == "completed" + authority.allowed = False + with pytest.raises(RuntimeError, match="^BLOB_UPLOAD_DENIED$"): + await service.complete(actor, session.upload_id) + authority.allowed = True assert await service.complete(actor, session.upload_id) == session.reference assert verifier.calls == 1 assert await service.write(actor, session.upload_id, BytesIO(data)) == len(data) From 1f77c88d10247fa0e6e7d014f4751d0a16bb6d41 Mon Sep 17 00:00:00 2001 From: Philip Z Date: Wed, 28 Oct 2026 00:59:45 +0800 Subject: [PATCH 28/75] test(s3): verify atomic create-only against compatible server --- tests/runtime/test_object_storage.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/tests/runtime/test_object_storage.py b/tests/runtime/test_object_storage.py index 9e14a662..efcac037 100644 --- a/tests/runtime/test_object_storage.py +++ b/tests/runtime/test_object_storage.py @@ -1,4 +1,5 @@ from datetime import timedelta, datetime, timezone +from concurrent.futures import ThreadPoolExecutor from io import BytesIO from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from threading import Thread @@ -165,6 +166,8 @@ def test_minio_real_signed_range_and_expiry(): aws_access_key_id=access, aws_secret_access_key=secret) storage = UserContext().with_object_storage(provider).object_storage("F4 real S3-compatible Python") key = ObjectKey("f4/python/" + uuid4().hex) + create_key = ObjectKey("f4/python/create/" + uuid4().hex) + race_key = ObjectKey("f4/python/race/" + uuid4().hex) content = b"%PDF-1.7\nF4 real S3-compatible bytes\n%%EOF" try: written = storage.put_bytes(key, content, PutOptions(len(content), WriteMode.REPLACE, "application/pdf")) @@ -182,8 +185,32 @@ def test_minio_real_signed_range_and_expiry(): with pytest.raises(HTTPError) as expired: urlopen(ticket.url, timeout=5) assert expired.value.code == 403 + if os.environ.get("TEAQL_S3_TEST_CREATE_ONLY") == "1": + create = PutOptions(len(content), WriteMode.CREATE_ONLY, "application/pdf") + created = storage.put_bytes(create_key, content, create) + assert_error(StorageErrorCode.CONFLICT, + lambda: storage.put_bytes(create_key, b"x" * len(content), create)) + assert storage.head(create_key).etag == created.etag + assert storage.get_bytes(create_key, 1024) == content + candidates = (content, b"y" * len(content)) + with ThreadPoolExecutor(max_workers=2) as pool: + futures = [pool.submit(storage.put_bytes, race_key, value, create) + for value in candidates] + outcomes = [] + for future in futures: + try: + future.result() + outcomes.append("winner") + except StorageError as error: + assert error.code == StorageErrorCode.CONFLICT + outcomes.append("conflict") + assert outcomes.count("winner") == outcomes.count("conflict") == 1 + assert storage.get_bytes(race_key, 1024) == candidates[outcomes.index("winner")] finally: try: storage.delete(key) + if os.environ.get("TEAQL_S3_TEST_CREATE_ONLY") == "1": + storage.delete(create_key) + storage.delete(race_key) finally: provider.close() From dbb0f700c350f91e7482ceca491b2898f5a7bf9c Mon Sep 17 00:00:00 2001 From: Philip Z Date: Wed, 28 Oct 2026 01:12:11 +0800 Subject: [PATCH 29/75] test(blob): verify SQLite upload through real S3 provider --- tests/provider/sqlite/test_blob_upload.py | 59 +++++++++++++++++++++++ 1 file changed, 59 insertions(+) diff --git a/tests/provider/sqlite/test_blob_upload.py b/tests/provider/sqlite/test_blob_upload.py index c325b56d..80077e2d 100644 --- a/tests/provider/sqlite/test_blob_upload.py +++ b/tests/provider/sqlite/test_blob_upload.py @@ -3,6 +3,7 @@ from hashlib import sha256 from io import BytesIO from threading import Event +from uuid import uuid4 import asyncio import os @@ -66,6 +67,64 @@ def verify(self, stream, extension, maximum_bytes): return BlobVerifiedContent(len(data), sha256(data).hexdigest(), "application/pdf") +@pytest.mark.asyncio +async def test_upload_real_sqlite_s3_create_only_confirm_and_replay(tmp_path): + if os.environ.get("TEAQL_S3_TEST_BLOB_UPLOAD") != "1": + pytest.skip("explicit S3-compatible Blob upload gate required") + endpoint = os.environ.get("TEAQL_S3_TEST_ENDPOINT") + bucket = os.environ.get("TEAQL_S3_TEST_BUCKET") + access = os.environ.get("TEAQL_S3_TEST_ACCESS_KEY") + secret = os.environ.get("TEAQL_S3_TEST_SECRET_KEY") + assert all((endpoint, bucket, access, secret)), "explicit S3 test configuration required" + from teaql.provider.s3 import S3ObjectStorage + + provider = S3ObjectStorage.connect(bucket, endpoint_url=endpoint, + aws_access_key_id=access, aws_secret_access_key=secret) + actor = RuntimeModule.new().into_context().with_object_storage(provider) + actor.with_object_storage_observer(lambda event: None) + path = str(tmp_path / "s3-upload.db") + catalog = SqliteBlobCatalog(SqliteTransport(path)) + authority, verifier = UploadAuthority(), FixturePDFVerifier() + service = SqliteBlobUploadService(catalog, authority, verifier, 10_000) + data = b"%PDF-1.7\nreal S3 Blob upload\n%%EOF" + object_key = None + try: + await catalog.ensure_schema() + session = await service.begin(actor, BlobUploadTarget("Asset", "1", "document"), + BlobUploadInput("report.pdf", len(data), "application/pdf"), + uuid4().hex) + async with aiosqlite.connect(path) as db: + async with db.execute("SELECT object_key FROM teaql_blob_catalog WHERE key=?", + (session.reference.key,)) as cursor: + object_key = ObjectKey((await cursor.fetchone())[0]) + assert await service.write(actor, session.upload_id, BytesIO(data)) == len(data) + authority.allowed = False + with pytest.raises(RuntimeError, match="BLOB_UPLOAD_DENIED"): + await service.complete(actor, session.upload_id) + authority.allowed = True + assert await service.complete(actor, session.upload_id) == session.reference + assert await service.complete(actor, session.upload_id) == session.reference + assert verifier.calls == 1 + assert await service.write(actor, session.upload_id, BytesIO(data)) == len(data) + store = actor.object_storage("verify completed S3 Blob upload") + assert store.get_bytes(object_key, 10_000) == data + head = store.head(object_key) + async with aiosqlite.connect(path) as db: + async with db.execute( + "SELECT status, content_type, size_bytes, content_version " + "FROM teaql_blob_catalog WHERE key=?", (session.reference.key,) + ) as cursor: + status, mime, size, version = await cursor.fetchone() + assert (status, mime, size, version) == ( + "uploaded_unbound", "application/pdf", len(data), head.etag) + finally: + try: + if object_key is not None: + actor.object_storage("clean S3 Blob upload fixture").delete(object_key) + finally: + provider.close() + + @pytest.mark.asyncio async def test_expired_replay_key_cannot_recover_upload_session(tmp_path): catalog = SqliteBlobCatalog(SqliteTransport(str(tmp_path / "expired.db"))) From 369eea2d7bdaf4e82f3ce2ca59941fbcf8803a3f Mon Sep 17 00:00:00 2001 From: Philip Z Date: Wed, 28 Oct 2026 01:21:50 +0800 Subject: [PATCH 30/75] test(blob): verify S3 graph bind and authorized signed view --- tests/provider/sqlite/test_blob_upload.py | 77 ++++++++++++++++++++++- 1 file changed, 74 insertions(+), 3 deletions(-) diff --git a/tests/provider/sqlite/test_blob_upload.py b/tests/provider/sqlite/test_blob_upload.py index 80077e2d..7232fa0c 100644 --- a/tests/provider/sqlite/test_blob_upload.py +++ b/tests/provider/sqlite/test_blob_upload.py @@ -1,8 +1,10 @@ """Real SQLite/File upload confirmation; parser remains test-only.""" +from datetime import datetime, timedelta, timezone from hashlib import sha256 from io import BytesIO from threading import Event +from urllib.request import Request, urlopen from uuid import uuid4 import asyncio import os @@ -20,10 +22,15 @@ from teaql.provider.sqlite.dialect import SqliteDialect from teaql.provider.sqlite.transport import SqliteTransport from teaql.runtime import RuntimeModule +from teaql.runtime.view_preparation import ( + AuthorizedBlobSlot, AuthorizedResponseGraph, ViewOptions, + ViewPreparationError, ViewPreparationService, +) from teaql.sql.executor import SqlDataServiceExecutor from teaql_blob_reference import ( BlobBindingChange, BlobBindingTarget, BlobUploadInput, BlobUploadTarget, - BlobVerifiedContent, FileObjectStorage, SqliteBlobCatalog, SqliteBlobUploadService, + BlobVerifiedContent, FileObjectStorage, SqliteBlobCatalog, SqliteBlobResponseContributor, + SqliteBlobUploadService, ) @@ -80,15 +87,26 @@ async def test_upload_real_sqlite_s3_create_only_confirm_and_replay(tmp_path): provider = S3ObjectStorage.connect(bucket, endpoint_url=endpoint, aws_access_key_id=access, aws_secret_access_key=secret) - actor = RuntimeModule.new().into_context().with_object_storage(provider) - actor.with_object_storage_observer(lambda event: None) path = str(tmp_path / "s3-upload.db") + descriptor = EntityDescriptor("Asset").table_name("asset") + descriptor.property(PropertyDescriptor("id", DataType.I64).is_id()) + descriptor.property(PropertyDescriptor("version", DataType.I64).is_version()) + descriptor.property(PropertyDescriptor("name", DataType.Text)) + schema = SimpleSchemaProvider() + schema.register_entity(descriptor) + actor = RuntimeModule.new().entity(descriptor).into_context() + actor.insert_resource("dataService", SqlDataServiceExecutor( + SqliteDialect(), SqliteTransport(path), schema)) + actor.with_object_storage(provider).with_object_storage_observer(lambda event: None) catalog = SqliteBlobCatalog(SqliteTransport(path)) authority, verifier = UploadAuthority(), FixturePDFVerifier() service = SqliteBlobUploadService(catalog, authority, verifier, 10_000) data = b"%PDF-1.7\nreal S3 Blob upload\n%%EOF" object_key = None try: + async with aiosqlite.connect(path) as db: + await db.execute("CREATE TABLE asset (id INTEGER PRIMARY KEY, version INTEGER, name TEXT)") + await db.commit() await catalog.ensure_schema() session = await service.begin(actor, BlobUploadTarget("Asset", "1", "document"), BlobUploadInput("report.pdf", len(data), "application/pdf"), @@ -117,6 +135,59 @@ async def test_upload_real_sqlite_s3_create_only_confirm_and_replay(tmp_path): status, mime, size, version = await cursor.fetchone() assert (status, mime, size, version) == ( "uploaded_unbound", "application/pdf", len(data), head.etag) + + async def bind(graph): + command = (InsertCommand.new("Asset").value("id", 1) + .value("version", 1).value("name", "document")) + await graph.transaction.mutate(graph.context, + graph.request(command, graph.scope(EntityKey("Asset", 1)))) + assert await catalog.bind_in_graph(graph, graph.context, + BlobBindingChange(BlobBindingTarget("Asset", "1", "document"), + session.reference, 0, 1, "Create"), + BindingAuthority()) == 1 + + await actor.execute_graph_save(bind, comment="bind S3 Blob in business graph") + async with aiosqlite.connect(path) as db: + async with db.execute("SELECT status FROM teaql_blob_catalog WHERE key=?", + (session.reference.key,)) as cursor: + assert (await cursor.fetchone())[0] == "available" + + class ReadAuthorization: + allowed = True + async def authorize(self, _context, _slot): + return self.allowed + + class Authorizer: + version = 1 + async def authorize(self, _context, _result, _options): + slot = AuthorizedBlobSlot(("document",), "Asset", "1", "document", + "tenant-a", self.version, session.reference) + return AuthorizedResponseGraph({"document": session.reference}, (slot,)) + + read_permission, authorizer = ReadAuthorization(), Authorizer() + actor.insert_resource("response_graph_authorizer", authorizer) + actor.insert_resource("blob_response_contributor", + SqliteBlobResponseContributor(catalog, read_permission, 3)) + options = ViewOptions("ordinary", "read_only", + datetime.now(timezone.utc) + timedelta(seconds=60)) + view = await ViewPreparationService().prepare(actor, None, options) + link = view.presentation["document"] + assert link.startswith("http") and view.cache_control == "private, no-store" + with urlopen(Request(link, headers={"Range": "bytes=0-7"}), timeout=5) as response: + assert response.status == 206 and response.read() == data[:8] + assert response.headers["Content-Type"] == "application/pdf" + assert "report.pdf" in response.headers["Content-Disposition"] + read_permission.allowed = False + with pytest.raises(ViewPreparationError) as denied: + await ViewPreparationService().prepare(actor, None, options) + assert denied.value.code == "BLOB_READ_DENIED" + read_permission.allowed = True + authorizer.version = 0 + with pytest.raises(ViewPreparationError) as stale: + await ViewPreparationService().prepare(actor, None, + ViewOptions("ordinary", "committed", + datetime.now(timezone.utc) + timedelta(seconds=60))) + assert (stale.value.code, stale.value.business_state) == ("BLOB_READ_DENIED", "committed") finally: try: if object_key is not None: From c7418f9cc945d1b18768322e7311fa65098f09c3 Mon Sep 17 00:00:00 2001 From: Philip Z Date: Wed, 28 Oct 2026 01:46:46 +0800 Subject: [PATCH 31/75] feat(blob): add optional decoded JPEG and PNG verifier --- packages/teaql-blob-reference/README.md | 10 ++ packages/teaql-blob-reference/pyproject.toml | 3 + .../teaql_blob_reference/content_pillow.py | 133 ++++++++++++++++++ .../src/teaql_blob_reference/module.py | 3 + .../teaql_blob_reference/upload_service.py | 5 + tests/fixtures/blob-sample.jpg | Bin 0 -> 633 bytes tests/fixtures/blob-sample.png | Bin 0 -> 82 bytes .../sqlite/test_blob_content_pillow.py | 127 +++++++++++++++++ tests/provider/sqlite/test_blob_module.py | 10 +- 9 files changed, 290 insertions(+), 1 deletion(-) create mode 100644 packages/teaql-blob-reference/src/teaql_blob_reference/content_pillow.py create mode 100644 tests/fixtures/blob-sample.jpg create mode 100644 tests/fixtures/blob-sample.png create mode 100644 tests/provider/sqlite/test_blob_content_pillow.py diff --git a/packages/teaql-blob-reference/README.md b/packages/teaql-blob-reference/README.md index 9eed105f..32282e47 100644 --- a/packages/teaql-blob-reference/README.md +++ b/packages/teaql-blob-reference/README.md @@ -3,3 +3,13 @@ Optional Blob catalog and current-read services for TeaQL Python. Install this wheel alongside the matching `teaql` version. Catalog migration and runtime service installation are explicit host actions; this package does not install schema or expose routes on import. `SqliteBlobUploadService` offers a host-invoked, scope-bound upload journal. The host must provide current model/action authorization and a trusted parser that consumes every stored byte and returns actual format, size, and SHA256. The service writes through the context's object-storage provider, rechecks the stored generation before confirmation, and promotes the row to `uploaded_unbound` only after full verification. The included PDF verifier in tests is a fixture, not a production format parser. Failure recovery, retention/GC, other SQL backends, and production ten-format parsing remain open. + +For actual JPEG/PNG parsing, install `teaql-blob-reference[image]` and inject +`teaql_blob_reference.content_pillow.PillowBlobContentVerifier`. The image extra +pins Pillow 12.3.0, the newest release supporting Python 3.10 at this checkpoint. +The adapter validates complete JPEG/PNG containers and PNG CRCs, checks the +decoder's real format and loads all pixels with explicit input, pixel and +concurrent-work bounds. It has no hard CPU/process isolation. Construction is +passive; `BlobReferenceModule.ready` invokes its optional decoder readiness check. +It accepts `jpg`, `jpeg` and `png` only. The remaining eight formats need separate +production adapters; the earlier PDF fixture test does not establish them. diff --git a/packages/teaql-blob-reference/pyproject.toml b/packages/teaql-blob-reference/pyproject.toml index 190d7644..04bc1f9f 100644 --- a/packages/teaql-blob-reference/pyproject.toml +++ b/packages/teaql-blob-reference/pyproject.toml @@ -12,6 +12,9 @@ license = "Apache-2.0" authors = [{ name = "TeaQL" }] dependencies = ["teaql==0.3.1"] +[project.optional-dependencies] +image = ["Pillow==12.3.0"] + [tool.setuptools.packages.find] where = ["src"] include = ["teaql_blob_reference*"] diff --git a/packages/teaql-blob-reference/src/teaql_blob_reference/content_pillow.py b/packages/teaql-blob-reference/src/teaql_blob_reference/content_pillow.py new file mode 100644 index 00000000..9a0e37c8 --- /dev/null +++ b/packages/teaql-blob-reference/src/teaql_blob_reference/content_pillow.py @@ -0,0 +1,133 @@ +"""Optional bounded JPEG/PNG parser for the trusted Blob upload verifier SPI.""" + +from __future__ import annotations + +from hashlib import sha256 +from io import BytesIO +from threading import BoundedSemaphore +from warnings import catch_warnings, simplefilter +import struct +import zlib + +from .upload_service import BlobVerifiedContent + + +_MIME = {"jpg": "image/jpeg", "jpeg": "image/jpeg", "png": "image/png"} +_PNG_SIGNATURE = b"\x89PNG\r\n\x1a\n" + + +def _complete_png(data: bytes) -> bool: + if len(data) < 45 or not data.startswith(_PNG_SIGNATURE): + return False + offset = 8 + count = 0 + image_data = False + while offset <= len(data) - 12: + length = struct.unpack_from(">I", data, offset)[0] + if length > len(data) - offset - 12: + return False + kind = data[offset + 4:offset + 8] + if len(kind) != 4 or not all(65 <= c <= 90 or 97 <= c <= 122 for c in kind): + return False + if count == 0 and (kind != b"IHDR" or length != 13): + return False + expected = struct.unpack_from(">I", data, offset + 8 + length)[0] + if zlib.crc32(data[offset + 4:offset + 8 + length]) != expected: + return False + offset += length + 12 + count += 1 + if kind == b"IDAT": + image_data = True + if kind == b"IEND": + return length == 0 and image_data and offset == len(data) + return False + + +class PillowBlobContentVerifier: + """Passive optional adapter; no Pillow import or parser work at construction.""" + + supported_extensions = frozenset(_MIME) + + def __init__(self, maximum_input_bytes: int = 32 * 1024 * 1024, + maximum_decoded_bytes: int = 100 * 1024 * 1024, + concurrent_verifications: int = 2): + if (type(maximum_input_bytes) is not int or not 1 <= maximum_input_bytes <= 100 * 1024 * 1024 + or type(maximum_decoded_bytes) is not int + or not 4 <= maximum_decoded_bytes <= 512 * 1024 * 1024 + or type(concurrent_verifications) is not int + or not 1 <= concurrent_verifications <= 32): + raise RuntimeError("BLOB_POLICY_INVALID") + self.maximum_input_bytes = maximum_input_bytes + self.maximum_decoded_bytes = maximum_decoded_bytes + self._permits = BoundedSemaphore(concurrent_verifications) + + def validate_ready(self) -> None: + try: + from PIL import ImageFile, features + if ImageFile.LOAD_TRUNCATED_IMAGES: + raise RuntimeError("BLOB_CAPABILITY_MISSING") + if not features.check("jpg") or not features.check("zlib"): + raise RuntimeError("BLOB_CAPABILITY_MISSING") + except Exception: + raise RuntimeError("BLOB_CAPABILITY_MISSING") from None + + def verify(self, stream, extension: str, maximum_bytes: int) -> BlobVerifiedContent: + if extension not in _MIME: + raise RuntimeError("BLOB_FORMAT_NOT_ALLOWED") + if type(maximum_bytes) is not int or maximum_bytes < 1: + raise RuntimeError("BLOB_POLICY_INVALID") + if not self._permits.acquire(blocking=False): + raise RuntimeError("BLOB_VERIFICATION_BUSY") + try: + maximum = min(maximum_bytes, self.maximum_input_bytes) + chunks = [] + size = 0 + digest = sha256() + while True: + try: + block = stream.read(65536) + except Exception: + raise RuntimeError("BLOB_STORE_FAILED") from None + if not isinstance(block, bytes): + raise RuntimeError("BLOB_STORE_FAILED") + if not block: + break + if len(block) > maximum - size: + raise RuntimeError("BLOB_SIZE_EXCEEDED") + chunks.append(block) + digest.update(block) + size += len(block) + if not size: + raise RuntimeError("BLOB_FORMAT_NOT_ALLOWED") + data = b"".join(chunks) + if extension == "png": + complete = _complete_png(data) + else: + complete = len(data) >= 4 and data.startswith(b"\xff\xd8") and data.endswith(b"\xff\xd9") + if not complete: + raise RuntimeError("BLOB_FORMAT_NOT_ALLOWED") + try: + from PIL import Image, ImageFile + if ImageFile.LOAD_TRUNCATED_IMAGES: + raise RuntimeError("BLOB_CAPABILITY_MISSING") + expected = "PNG" if extension == "png" else "JPEG" + with catch_warnings(): + simplefilter("error", Image.DecompressionBombWarning) + with Image.open(BytesIO(data), formats=[expected]) as image: + width, height = image.size + if (image.format != expected or width < 1 or height < 1 + or width * height > self.maximum_decoded_bytes // 4 + or getattr(image, "n_frames", 1) != 1): + raise RuntimeError("BLOB_FORMAT_NOT_ALLOWED") + image.load() # Header detection alone is insufficient. + except RuntimeError as error: + if str(error) == "BLOB_CAPABILITY_MISSING": + raise + raise RuntimeError("BLOB_FORMAT_NOT_ALLOWED") from None + except ImportError: + raise RuntimeError("BLOB_CAPABILITY_MISSING") from None + except Exception: + raise RuntimeError("BLOB_FORMAT_NOT_ALLOWED") from None + return BlobVerifiedContent(size, digest.hexdigest(), _MIME[extension]) + finally: + self._permits.release() diff --git a/packages/teaql-blob-reference/src/teaql_blob_reference/module.py b/packages/teaql-blob-reference/src/teaql_blob_reference/module.py index cc3d6fdc..a66d6420 100644 --- a/packages/teaql-blob-reference/src/teaql_blob_reference/module.py +++ b/packages/teaql-blob-reference/src/teaql_blob_reference/module.py @@ -23,6 +23,9 @@ async def validate_ready(self, context) -> None: if owner is None or not callable(getattr(owner, "begin", None)): raise RuntimeError("BLOB_CAPABILITY_MISSING") await self.module.catalog.validate_schema() + verifier_ready = getattr(self.module.upload.verifier, "validate_ready", None) + if callable(verifier_ready): + verifier_ready() class BlobReferenceModule: diff --git a/packages/teaql-blob-reference/src/teaql_blob_reference/upload_service.py b/packages/teaql-blob-reference/src/teaql_blob_reference/upload_service.py index 72921f04..6663b537 100644 --- a/packages/teaql-blob-reference/src/teaql_blob_reference/upload_service.py +++ b/packages/teaql-blob-reference/src/teaql_blob_reference/upload_service.py @@ -549,6 +549,11 @@ def verify_stored(): if after is None or after.size != before.size or after.etag != before.etag: raise RuntimeError("BLOB_UPLOAD_INCOMPLETE") return verified + except RuntimeError as error: + if str(error) in ("BLOB_FORMAT_NOT_ALLOWED", "BLOB_SIZE_EXCEEDED", + "BLOB_VERIFICATION_BUSY", "BLOB_CAPABILITY_MISSING"): + raise + raise RuntimeError("BLOB_UPLOAD_INCOMPLETE") from None except Exception: raise RuntimeError("BLOB_UPLOAD_INCOMPLETE") from None diff --git a/tests/fixtures/blob-sample.jpg b/tests/fixtures/blob-sample.jpg new file mode 100644 index 0000000000000000000000000000000000000000..fc5cfd5c45cc675b08d76c8dd614684fc4c8d6ce GIT binary patch literal 633 zcmex=``2_j6xdp@o1cgOJMMZh|#U;coyG6VInuyV4pa*FVB^NNrR z{vTivX!XqN1l2cOC(lau%ic3n%$}1|Xnp;}i+B-VC zCQY6)b=ve9GiNPYykzOJeA&aSFc^aar4&0 zM~|O8efIpt%U2&ieg5+G+xH(oe}VkP$iNKo7LbH^49#DHKz}i@urRZ*gZ#zFR1U<< zf-J0xhHOHPf$WKe!b(Ps93oB=7j8VrscandK{To8BA1wo$wSqTAg_UaMx4i*$nqK7 fV+eoUV&GwB1V$dSAcH-_$106Qdz?QW{(lnyG!exo literal 0 HcmV?d00001 diff --git a/tests/fixtures/blob-sample.png b/tests/fixtures/blob-sample.png new file mode 100644 index 0000000000000000000000000000000000000000..2194b1e195e777bb0dc7636e85f6bfcbeb757992 GIT binary patch literal 82 zcmeAS@N?(olHy`uVBq!ia0vp^0wB!61SBU+%rFB|lAbP(Ar*6yLv+j26<7@>E~yGR fyrNBUDI>$nGDhb6^W-)H6)||a`njxgN@xNAvnCXY literal 0 HcmV?d00001 diff --git a/tests/provider/sqlite/test_blob_content_pillow.py b/tests/provider/sqlite/test_blob_content_pillow.py new file mode 100644 index 00000000..3c6dcc4b --- /dev/null +++ b/tests/provider/sqlite/test_blob_content_pillow.py @@ -0,0 +1,127 @@ +"""Real shared JPEG/PNG decoder and SQLite/File upload boundary checks.""" + +from hashlib import sha256 +from io import BytesIO +from pathlib import Path +from threading import Event, Thread +import os + +import aiosqlite +import pytest + +pytest.importorskip("PIL") + +from teaql.core.blob_ref import BlobFieldSpec +from teaql.runtime import RuntimeModule +from teaql.provider.sqlite.transport import SqliteTransport +from teaql_blob_reference import ( + BlobUploadInput, BlobUploadTarget, FileObjectStorage, SqliteBlobCatalog, + SqliteBlobUploadService, +) +from teaql_blob_reference.content_pillow import PillowBlobContentVerifier + + +def fixture(extension): + return (Path(__file__).parents[2] / "fixtures" / f"blob-sample.{extension}").read_bytes() + + +def test_real_images_and_malformed_or_bounded_input(): + verifier = PillowBlobContentVerifier() + verifier.validate_ready() + for extension in ("jpg", "png"): + data = fixture(extension) + checked = verifier.verify(BytesIO(data), extension, 10_000) + assert checked.size_bytes == len(data) + assert checked.sha256 == sha256(data).hexdigest() + assert checked.content_type == ("image/jpeg" if extension == "jpg" else "image/png") + wrong = "png" if extension == "jpg" else "jpg" + with pytest.raises(RuntimeError, match="BLOB_FORMAT_NOT_ALLOWED"): + verifier.verify(BytesIO(data), wrong, 10_000) + with pytest.raises(RuntimeError, match="BLOB_FORMAT_NOT_ALLOWED"): + verifier.verify(BytesIO(data[:-12]), extension, 10_000) + with pytest.raises(RuntimeError, match="BLOB_SIZE_EXCEEDED"): + verifier.verify(BytesIO(data), extension, len(data) - 1) + corrupt = bytearray(fixture("png")) + corrupt[-25] ^= 1 + with pytest.raises(RuntimeError, match="BLOB_FORMAT_NOT_ALLOWED"): + verifier.verify(BytesIO(corrupt), "png", 10_000) + with pytest.raises(RuntimeError, match="BLOB_FORMAT_NOT_ALLOWED"): + verifier.verify(BytesIO(b"not an image"), "jpg", 10_000) + with pytest.raises(RuntimeError, match="BLOB_FORMAT_NOT_ALLOWED"): + verifier.verify(BytesIO(b""), "png", 10_000) + limited = PillowBlobContentVerifier(maximum_input_bytes=10_000, + maximum_decoded_bytes=4) + with pytest.raises(RuntimeError, match="BLOB_FORMAT_NOT_ALLOWED"): + limited.verify(BytesIO(fixture("png")), "png", 10_000) + + +def test_concurrent_admission_releases_slot_after_decode(): + entered, release = Event(), Event() + data = fixture("jpg") + + class PausedInput(BytesIO): + def read(self, size=-1): + entered.set() + assert release.wait(5) + return super().read(size) + + verifier = PillowBlobContentVerifier(concurrent_verifications=1) + outcome = [] + first = Thread(target=lambda: outcome.append(verifier.verify(PausedInput(data), "jpg", 10_000))) + first.start() + try: + assert entered.wait(5) + with pytest.raises(RuntimeError, match="BLOB_VERIFICATION_BUSY"): + verifier.verify(BytesIO(data), "jpg", 10_000) + finally: + release.set() + first.join(5) + assert not first.is_alive() + assert outcome[0].content_type == "image/jpeg" + assert verifier.verify(BytesIO(data), "jpg", 10_000).size_bytes == len(data) + + +class ImageAuthority: + def scope(self, _context): + return "tenant-a" + + def field(self, _context, _target): + return BlobFieldSpec("Asset", "image", ["jpg"], True, "default") + + async def authorize(self, _context, target, action, _input): + return target.entity == "Asset" and target.field == "image" and action in ( + "Begin", "Write", "Confirm") + + +@pytest.mark.asyncio +async def test_actual_jpeg_upload_promotes_and_truncated_file_stays_initiated(tmp_path): + path = str(tmp_path / "image.db") + catalog = SqliteBlobCatalog(SqliteTransport(path)) + provider = FileObjectStorage(tmp_path / "objects", os.urandom(32)) + actor = RuntimeModule.new().into_context().with_object_storage(provider) + actor.with_object_storage_observer(lambda event: None) + service = SqliteBlobUploadService(catalog, ImageAuthority(), + PillowBlobContentVerifier(), 10_000) + target = BlobUploadTarget("Asset", "1", "image") + try: + await catalog.ensure_schema() + good = fixture("jpg") + upload = await service.begin(actor, target, BlobUploadInput("image.jpg", len(good)), "image-good") + assert await service.write(actor, upload.upload_id, BytesIO(good)) == len(good) + assert await service.complete(actor, upload.upload_id) == upload.reference + async with aiosqlite.connect(path) as db: + async with db.execute("SELECT status,content_type,size_bytes FROM teaql_blob_catalog WHERE key=?", + (upload.reference.key,)) as cursor: + assert await cursor.fetchone() == ("uploaded_unbound", "image/jpeg", len(good)) + bad = good[:-12] + rejected = await service.begin(actor, target, BlobUploadInput("broken.jpg", len(bad)), + "image-bad") + await service.write(actor, rejected.upload_id, BytesIO(bad)) + with pytest.raises(RuntimeError, match="BLOB_FORMAT_NOT_ALLOWED"): + await service.complete(actor, rejected.upload_id) + async with aiosqlite.connect(path) as db: + async with db.execute("SELECT status FROM teaql_blob_catalog WHERE key=?", + (rejected.reference.key,)) as cursor: + assert (await cursor.fetchone())[0] == "initiated" + finally: + provider.close() diff --git a/tests/provider/sqlite/test_blob_module.py b/tests/provider/sqlite/test_blob_module.py index 6acd5b4e..892c14d0 100644 --- a/tests/provider/sqlite/test_blob_module.py +++ b/tests/provider/sqlite/test_blob_module.py @@ -37,9 +37,15 @@ async def authorize(self, context, target, action, input): class _Verifier: + def __init__(self): + self.ready_calls = 0 + def verify(self, stream, extension, maximum_bytes): raise AssertionError("readiness must not parse content") + def validate_ready(self): + self.ready_calls += 1 + class _BindingAuthority: allowed = True @@ -64,7 +70,8 @@ async def test_blob_module_requires_provider_owner_and_catalog(tmp_path): path = str(tmp_path / "module.db") transport = SqliteTransport(path) catalog = SqliteBlobCatalog(transport) - upload = SqliteBlobUploadService(catalog, _UploadAuthority(), _Verifier(), 10_000) + verifier = _Verifier() + upload = SqliteBlobUploadService(catalog, _UploadAuthority(), verifier, 10_000) module = BlobReferenceModule(catalog, upload, _ReadAuthority(), _BindingAuthority()) context = RuntimeModule.new().require_capability("blob.v1").into_context() with pytest.raises(RuntimeError, match="BLOB_CAPABILITY_MISSING"): @@ -84,6 +91,7 @@ async def test_blob_module_requires_provider_owner_and_catalog(tmp_path): await module.ready(context) await catalog.ensure_schema() await module.ready(context) + assert verifier.ready_calls == 1 await validate_capabilities(context) provider.close() From 3d030621b1b10795d94a3a0f151541a12d9f256b Mon Sep 17 00:00:00 2001 From: Philip Z Date: Wed, 28 Oct 2026 02:20:02 +0800 Subject: [PATCH 32/75] Exercise real JPEG verifier through S3 Blob graph and view gate --- tests/provider/sqlite/test_blob_upload.py | 40 ++++++++++++++++++----- 1 file changed, 31 insertions(+), 9 deletions(-) diff --git a/tests/provider/sqlite/test_blob_upload.py b/tests/provider/sqlite/test_blob_upload.py index 7232fa0c..88fbda5a 100644 --- a/tests/provider/sqlite/test_blob_upload.py +++ b/tests/provider/sqlite/test_blob_upload.py @@ -3,6 +3,7 @@ from datetime import datetime, timedelta, timezone from hashlib import sha256 from io import BytesIO +from pathlib import Path from threading import Event from urllib.request import Request, urlopen from uuid import uuid4 @@ -35,15 +36,16 @@ class UploadAuthority: - def __init__(self): + def __init__(self, extensions=("pdf",)): self.current_scope = "tenant-a" self.allowed = True + self.extensions = extensions def scope(self, _context): return self.current_scope def field(self, _context, _target): - return BlobFieldSpec("Asset", "document", ["pdf"], True, "default") + return BlobFieldSpec("Asset", "document", self.extensions, True, "default") async def authorize(self, _context, target, action, _input): return (self.allowed and target.entity == "Asset" and target.field == "document" @@ -75,9 +77,12 @@ def verify(self, stream, extension, maximum_bytes): @pytest.mark.asyncio -async def test_upload_real_sqlite_s3_create_only_confirm_and_replay(tmp_path): +@pytest.mark.parametrize("image", [False, True], ids=["fixture-pdf", "real-jpeg"]) +async def test_upload_real_sqlite_s3_create_only_confirm_and_replay(tmp_path, image): if os.environ.get("TEAQL_S3_TEST_BLOB_UPLOAD") != "1": pytest.skip("explicit S3-compatible Blob upload gate required") + if image and os.environ.get("TEAQL_S3_TEST_REAL_IMAGE") != "1": + pytest.skip("explicit production JPEG S3 gate required") endpoint = os.environ.get("TEAQL_S3_TEST_ENDPOINT") bucket = os.environ.get("TEAQL_S3_TEST_BUCKET") access = os.environ.get("TEAQL_S3_TEST_ACCESS_KEY") @@ -99,9 +104,26 @@ async def test_upload_real_sqlite_s3_create_only_confirm_and_replay(tmp_path): SqliteDialect(), SqliteTransport(path), schema)) actor.with_object_storage(provider).with_object_storage_observer(lambda event: None) catalog = SqliteBlobCatalog(SqliteTransport(path)) - authority, verifier = UploadAuthority(), FixturePDFVerifier() + if image: + from teaql_blob_reference.content_pillow import PillowBlobContentVerifier + + class CountingImageVerifier: + def __init__(self): + self.inner = PillowBlobContentVerifier() + self.calls = 0 + + def verify(self, stream, extension, maximum_bytes): + self.calls += 1 + return self.inner.verify(stream, extension, maximum_bytes) + + authority, verifier = UploadAuthority(("jpg",)), CountingImageVerifier() + data = (Path(__file__).parents[2] / "fixtures" / "blob-sample.jpg").read_bytes() + file_name, content_type = "sample.jpg", "image/jpeg" + else: + authority, verifier = UploadAuthority(), FixturePDFVerifier() + data = b"%PDF-1.7\nreal S3 Blob upload\n%%EOF" + file_name, content_type = "report.pdf", "application/pdf" service = SqliteBlobUploadService(catalog, authority, verifier, 10_000) - data = b"%PDF-1.7\nreal S3 Blob upload\n%%EOF" object_key = None try: async with aiosqlite.connect(path) as db: @@ -109,7 +131,7 @@ async def test_upload_real_sqlite_s3_create_only_confirm_and_replay(tmp_path): await db.commit() await catalog.ensure_schema() session = await service.begin(actor, BlobUploadTarget("Asset", "1", "document"), - BlobUploadInput("report.pdf", len(data), "application/pdf"), + BlobUploadInput(file_name, len(data), content_type), uuid4().hex) async with aiosqlite.connect(path) as db: async with db.execute("SELECT object_key FROM teaql_blob_catalog WHERE key=?", @@ -134,7 +156,7 @@ async def test_upload_real_sqlite_s3_create_only_confirm_and_replay(tmp_path): ) as cursor: status, mime, size, version = await cursor.fetchone() assert (status, mime, size, version) == ( - "uploaded_unbound", "application/pdf", len(data), head.etag) + "uploaded_unbound", content_type, len(data), head.etag) async def bind(graph): command = (InsertCommand.new("Asset").value("id", 1) @@ -175,8 +197,8 @@ async def authorize(self, _context, _result, _options): assert link.startswith("http") and view.cache_control == "private, no-store" with urlopen(Request(link, headers={"Range": "bytes=0-7"}), timeout=5) as response: assert response.status == 206 and response.read() == data[:8] - assert response.headers["Content-Type"] == "application/pdf" - assert "report.pdf" in response.headers["Content-Disposition"] + assert response.headers["Content-Type"] == content_type + assert file_name in response.headers["Content-Disposition"] read_permission.allowed = False with pytest.raises(ViewPreparationError) as denied: await ViewPreparationService().prepare(actor, None, options) From 5e35d3a7c8f3ecd9d5f67b3a62dacd084e84d84b Mon Sep 17 00:00:00 2001 From: Philip Z Date: Wed, 28 Oct 2026 02:35:14 +0800 Subject: [PATCH 33/75] Add bounded real PDF verifier and S3 Blob upload gate --- packages/teaql-blob-reference/README.md | 10 +- packages/teaql-blob-reference/pyproject.toml | 1 + .../src/teaql_blob_reference/content_pypdf.py | 94 +++++++++++++ tests/fixtures/blob-sample.pdf | Bin 0 -> 533 bytes .../sqlite/test_blob_content_pypdf.py | 128 ++++++++++++++++++ tests/provider/sqlite/test_blob_upload.py | 40 +++--- 6 files changed, 256 insertions(+), 17 deletions(-) create mode 100644 packages/teaql-blob-reference/src/teaql_blob_reference/content_pypdf.py create mode 100644 tests/fixtures/blob-sample.pdf create mode 100644 tests/provider/sqlite/test_blob_content_pypdf.py diff --git a/packages/teaql-blob-reference/README.md b/packages/teaql-blob-reference/README.md index 32282e47..e9eba614 100644 --- a/packages/teaql-blob-reference/README.md +++ b/packages/teaql-blob-reference/README.md @@ -11,5 +11,13 @@ The adapter validates complete JPEG/PNG containers and PNG CRCs, checks the decoder's real format and loads all pixels with explicit input, pixel and concurrent-work bounds. It has no hard CPU/process isolation. Construction is passive; `BlobReferenceModule.ready` invokes its optional decoder readiness check. -It accepts `jpg`, `jpeg` and `png` only. The remaining eight formats need separate +It accepts `jpg`, `jpeg` and `png` only. The other formats need separate production adapters; the earlier PDF fixture test does not establish them. + +For actual PDF parsing, install `teaql-blob-reference[pdf]` and inject +`teaql_blob_reference.content_pypdf.PypdfBlobContentVerifier`. The PDF extra +pins pypdf 6.21.3. This adapter checks a complete PDF container with strict +cross-reference parsing, a real page tree and content operators, and bounded +input, pages, decoded page content and concurrent work. It rejects encrypted +PDFs. This is parser validation, not malware scanning or hard CPU/process +isolation. The PDF fixture verifier used in older S3 tests remains test-only. diff --git a/packages/teaql-blob-reference/pyproject.toml b/packages/teaql-blob-reference/pyproject.toml index 04bc1f9f..d87b758d 100644 --- a/packages/teaql-blob-reference/pyproject.toml +++ b/packages/teaql-blob-reference/pyproject.toml @@ -14,6 +14,7 @@ dependencies = ["teaql==0.3.1"] [project.optional-dependencies] image = ["Pillow==12.3.0"] +pdf = ["pypdf==6.21.3"] [tool.setuptools.packages.find] where = ["src"] diff --git a/packages/teaql-blob-reference/src/teaql_blob_reference/content_pypdf.py b/packages/teaql-blob-reference/src/teaql_blob_reference/content_pypdf.py new file mode 100644 index 00000000..dc32d460 --- /dev/null +++ b/packages/teaql-blob-reference/src/teaql_blob_reference/content_pypdf.py @@ -0,0 +1,94 @@ +"""Optional bounded PDF structure/content parser for the Blob upload verifier SPI.""" + +from __future__ import annotations + +from hashlib import sha256 +from io import BytesIO +from math import isfinite +from threading import BoundedSemaphore + +from .upload_service import BlobVerifiedContent + + +class PypdfBlobContentVerifier: + """Host-selected PDF parser; construction does no parser I/O or imports.""" + + supported_extensions = frozenset(("pdf",)) + + def __init__(self, maximum_input_bytes: int = 32 * 1024 * 1024, + maximum_decoded_bytes: int = 64 * 1024 * 1024, + maximum_pages: int = 1000, concurrent_verifications: int = 2): + if (type(maximum_input_bytes) is not int or not 1 <= maximum_input_bytes <= 100 * 1024 * 1024 + or type(maximum_decoded_bytes) is not int + or not 1 <= maximum_decoded_bytes <= 64 * 1024 * 1024 + or type(maximum_pages) is not int or not 1 <= maximum_pages <= 1000 + or type(concurrent_verifications) is not int + or not 1 <= concurrent_verifications <= 32): + raise RuntimeError("BLOB_POLICY_INVALID") + self.maximum_input_bytes = maximum_input_bytes + self.maximum_decoded_bytes = maximum_decoded_bytes + self.maximum_pages = maximum_pages + self._permits = BoundedSemaphore(concurrent_verifications) + + def validate_ready(self) -> None: + try: + from pypdf import PdfReader # noqa: F401 + except ImportError: + raise RuntimeError("BLOB_CAPABILITY_MISSING") from None + + def verify(self, stream, extension: str, maximum_bytes: int) -> BlobVerifiedContent: + if extension != "pdf": + raise RuntimeError("BLOB_FORMAT_NOT_ALLOWED") + if type(maximum_bytes) is not int or maximum_bytes < 1: + raise RuntimeError("BLOB_POLICY_INVALID") + if not self._permits.acquire(blocking=False): + raise RuntimeError("BLOB_VERIFICATION_BUSY") + try: + maximum = min(maximum_bytes, self.maximum_input_bytes) + chunks = [] + size = 0 + digest = sha256() + while True: + try: + block = stream.read(65536) + except Exception: + raise RuntimeError("BLOB_STORE_FAILED") from None + if not isinstance(block, bytes): + raise RuntimeError("BLOB_STORE_FAILED") + if not block: + break + if len(block) > maximum - size: + raise RuntimeError("BLOB_SIZE_EXCEEDED") + chunks.append(block) + digest.update(block) + size += len(block) + data = b"".join(chunks) + if not data.startswith(b"%PDF-") or not data.rstrip(b" \t\r\n\f").endswith(b"%%EOF"): + raise RuntimeError("BLOB_FORMAT_NOT_ALLOWED") + try: + from pypdf import PdfReader + except ImportError: + raise RuntimeError("BLOB_CAPABILITY_MISSING") from None + try: + reader = PdfReader(BytesIO(data), strict=True, root_object_recovery_limit=256) + if reader.is_encrypted or not 1 <= len(reader.pages) <= self.maximum_pages: + raise ValueError("invalid PDF page set") + decoded = 0 + for page in reader.pages: + width = float(page.mediabox.width) + height = float(page.mediabox.height) + if (not isfinite(width) or not isfinite(height) + or not 0 < width <= 14400 or not 0 < height <= 14400): + raise ValueError("invalid PDF media box") + contents = page.get_contents() + if contents is not None: + payload = contents.get_data() + decoded += len(payload) + if decoded > self.maximum_decoded_bytes: + raise ValueError("PDF decoded content limit") + _ = contents.operations # Parse content operators, not only the header. + except Exception: + raise RuntimeError("BLOB_FORMAT_NOT_ALLOWED") from None + return BlobVerifiedContent(size, digest.hexdigest(), "application/pdf") + finally: + self._permits.release() diff --git a/tests/fixtures/blob-sample.pdf b/tests/fixtures/blob-sample.pdf new file mode 100644 index 0000000000000000000000000000000000000000..5fc6a3027972e09fbcffa6a68de8927e3107c58b GIT binary patch literal 533 zcmZWmK~BRk5WMdz_7VvW)Nw-69;zNFR6>9Vfg9?- zY)c+{XJ*Gvbh$d?6Ca%KpU*Gg=%RXggZUg-q*dA^QsYUruhQ%k77LJNYOlS)v!{I} zkwqexHL#^>N`oBOO_A34jOX+)#`U1o|5v&cMr>40f|dgs?iBm zkXf?E?qphs3$-IzcjC5d>LJawrPh*+{c)4a($KNSV5DWwz1FQPbMuOPX31-#rP#td z+JM|{Pqz4A6X`CyY<5~^;9>~wVPF^pet;PsFiuK#V0pv>>U4y0PH~N~kh&OQ-jrPR tn9-uxNZtP7WAP!$3X7F8XrqsTU6+|^J7`m`jnJm`?E5otoU8j4`~oy-h* Date: Wed, 28 Oct 2026 02:48:01 +0800 Subject: [PATCH 34/75] Add fail-closed S3 version-conditional delete --- src/teaql/provider/s3/__init__.py | 20 +++++++++++++++++ src/teaql/runtime/object_storage.py | 17 ++++++++++++++ tests/runtime/test_object_storage.py | 33 ++++++++++++++++++++++++++++ 3 files changed, 70 insertions(+) diff --git a/src/teaql/provider/s3/__init__.py b/src/teaql/provider/s3/__init__.py index d852dd19..71c75cb7 100644 --- a/src/teaql/provider/s3/__init__.py +++ b/src/teaql/provider/s3/__init__.py @@ -121,6 +121,26 @@ def head(self, invocation, key): def delete(self, invocation, key): _sdk_call(lambda: self._client.delete_object(Bucket=self._bucket, Key=key.value)) + def delete_if_version(self, invocation, key, content_version: str): + if (not isinstance(content_version, str) or not content_version + or len(content_version) > 256 or content_version == "*" + or content_version.strip() != content_version + or any(ord(char) < 32 or 127 <= ord(char) <= 159 + for char in content_version)): + raise StorageError(StorageErrorCode.INVALID_INPUT) + try: + self._client.delete_object(Bucket=self._bucket, Key=key.value, + IfMatch=content_version) + return "deleted" + except Exception as error: + status = getattr(error, "response", {}).get( + "ResponseMetadata", {}).get("HTTPStatusCode") + if status in (404, 412): + return ("version_mismatch" if self.head(invocation, key) is not None + else "already_missing") + safe = _translate(error) + raise safe from None + def get_read_url(self, invocation, key, options): params = dict(Bucket=self._bucket, Key=key.value) disposition = options.content_disposition() diff --git a/src/teaql/runtime/object_storage.py b/src/teaql/runtime/object_storage.py index c88492b6..dd1b21b8 100644 --- a/src/teaql/runtime/object_storage.py +++ b/src/teaql/runtime/object_storage.py @@ -224,5 +224,22 @@ def exists(self, key): def delete(self, key): return self._call("delete", lambda: self._provider.delete(self._invocation, key)) + def delete_if_version(self, key, content_version: str): + if (not isinstance(content_version, str) or not content_version + or len(content_version) > 256 or content_version == "*" + or content_version.strip() != content_version + or any(ord(char) < 32 or 127 <= ord(char) <= 159 + for char in content_version)): + raise StorageError(StorageErrorCode.INVALID_INPUT) + conditional = getattr(self._provider, "delete_if_version", None) + if not callable(conditional): + raise StorageError(StorageErrorCode.UNSUPPORTED) + def delete(): + result = conditional(self._invocation, key, content_version) + if result not in ("deleted", "already_missing", "version_mismatch"): + raise StorageError(StorageErrorCode.UNAVAILABLE) + return result + return self._call("delete_if_version", delete) + def get_read_url(self, key, options=ReadUrlOptions()): return self._call("read_url", lambda: self._provider.get_read_url(self._invocation, key, options)) diff --git a/tests/runtime/test_object_storage.py b/tests/runtime/test_object_storage.py index efcac037..bb38fd0c 100644 --- a/tests/runtime/test_object_storage.py +++ b/tests/runtime/test_object_storage.py @@ -20,6 +20,7 @@ class Stub: size = 3 code = None seen = None + deletes = 0 def head(self, invocation, key): self.seen = invocation if self.code: @@ -30,6 +31,8 @@ def put(self, invocation, key, stream, options): return ObjectMetadata(key, options.size) def get(self, invocation, key): return ObjectContent(ObjectMetadata(key, self.size), BytesIO(b"abc")) + def delete(self, invocation, key): + self.deletes += 1 def assert_error(code, work): @@ -55,6 +58,16 @@ def test_object_storage_context_and_error_semantics(): stub.code = code assert_error(code, lambda: storage.exists(key)) +def test_conditional_delete_refuses_blind_provider(): + stub = Stub() + storage = UserContext().with_object_storage(stub).object_storage("collect old generation") + key = ObjectKey("PRIVATE-KEY-CANARY") + assert_error(StorageErrorCode.UNSUPPORTED, + lambda: storage.delete_if_version(key, '"old-etag"')) + assert_error(StorageErrorCode.INVALID_INPUT, + lambda: storage.delete_if_version(key, "*")) + assert stub.deletes == 0 + def test_object_storage_stream_bounds_ownership_and_completion(): events, stub = [], Stub() @@ -168,6 +181,7 @@ def test_minio_real_signed_range_and_expiry(): key = ObjectKey("f4/python/" + uuid4().hex) create_key = ObjectKey("f4/python/create/" + uuid4().hex) race_key = ObjectKey("f4/python/race/" + uuid4().hex) + conditional_key = ObjectKey("f2/python/conditional-delete/" + uuid4().hex) content = b"%PDF-1.7\nF4 real S3-compatible bytes\n%%EOF" try: written = storage.put_bytes(key, content, PutOptions(len(content), WriteMode.REPLACE, "application/pdf")) @@ -206,11 +220,30 @@ def test_minio_real_signed_range_and_expiry(): outcomes.append("conflict") assert outcomes.count("winner") == outcomes.count("conflict") == 1 assert storage.get_bytes(race_key, 1024) == candidates[outcomes.index("winner")] + if os.environ.get("TEAQL_S3_TEST_CONDITIONAL_DELETE") == "1": + first = storage.put_bytes(conditional_key, content, + PutOptions(len(content), WriteMode.CREATE_ONLY, "application/pdf")) + assert storage.delete_if_version( + conditional_key, '"00000000000000000000000000000000"') == "version_mismatch" + assert storage.head(conditional_key).etag == first.etag + replacement = b"next immutable generation" + second = storage.put_bytes(conditional_key, replacement, + PutOptions(len(replacement), WriteMode.REPLACE, "text/plain")) + assert storage.delete_if_version(conditional_key, first.etag) == "version_mismatch" + assert storage.head(conditional_key).etag == second.etag + assert storage.get_bytes(conditional_key, 1024) == replacement + assert storage.delete_if_version(conditional_key, second.etag) == "deleted" + assert storage.head(conditional_key) is None + assert storage.delete_if_version(conditional_key, second.etag) == "already_missing" + assert_error(StorageErrorCode.INVALID_INPUT, + lambda: storage.delete_if_version(conditional_key, "*")) finally: try: storage.delete(key) if os.environ.get("TEAQL_S3_TEST_CREATE_ONLY") == "1": storage.delete(create_key) storage.delete(race_key) + if os.environ.get("TEAQL_S3_TEST_CONDITIONAL_DELETE") == "1": + storage.delete(conditional_key) finally: provider.close() From ac0e1367711a01561a3aafee4384b07e4cadb42c Mon Sep 17 00:00:00 2001 From: Philip Z Date: Wed, 28 Oct 2026 03:26:50 +0800 Subject: [PATCH 35/75] Add Python SQLite expired-upload Blob maintenance --- packages/teaql-blob-reference/README.md | 4 +- .../src/teaql_blob_reference/__init__.py | 5 + .../maintenance_service.py | 307 ++++++++++++++++++ .../src/teaql_blob_reference/module.py | 15 +- .../teaql_blob_reference/sqlite_catalog.py | 10 + src/teaql/runtime/context.py | 6 + src/teaql/runtime/object_storage.py | 3 + .../provider/sqlite/test_blob_maintenance.py | 233 +++++++++++++ tests/provider/sqlite/test_blob_module.py | 16 +- 9 files changed, 594 insertions(+), 5 deletions(-) create mode 100644 packages/teaql-blob-reference/src/teaql_blob_reference/maintenance_service.py create mode 100644 tests/provider/sqlite/test_blob_maintenance.py diff --git a/packages/teaql-blob-reference/README.md b/packages/teaql-blob-reference/README.md index e9eba614..3987a32b 100644 --- a/packages/teaql-blob-reference/README.md +++ b/packages/teaql-blob-reference/README.md @@ -2,7 +2,9 @@ Optional Blob catalog and current-read services for TeaQL Python. Install this wheel alongside the matching `teaql` version. Catalog migration and runtime service installation are explicit host actions; this package does not install schema or expose routes on import. -`SqliteBlobUploadService` offers a host-invoked, scope-bound upload journal. The host must provide current model/action authorization and a trusted parser that consumes every stored byte and returns actual format, size, and SHA256. The service writes through the context's object-storage provider, rechecks the stored generation before confirmation, and promotes the row to `uploaded_unbound` only after full verification. The included PDF verifier in tests is a fixture, not a production format parser. Failure recovery, retention/GC, other SQL backends, and production ten-format parsing remain open. +`SqliteBlobUploadService` offers a host-invoked, scope-bound upload journal. The host must provide current model/action authorization and a trusted parser that consumes every stored byte and returns actual format, size, and SHA256. The service writes through the context's object-storage provider, rechecks the stored generation before confirmation, and promotes the row to `uploaded_unbound` only after full verification. The included PDF verifier in tests is a fixture, not a production format parser. Full retention/GC, other SQL backends, and production ten-format parsing remain open. + +`SqliteBlobMaintenanceService` is an optional, explicitly host-scheduled service for **expired, uncompleted, unbound server-stream uploads only**. The host supplies a current scope and async `can_maintain` policy, then calls `run_batch(context, cursor)` and may install the service through `BlobReferenceModule(..., maintenance=service)`. Run `catalog.ensure_schema()` as a separate migration before readiness; it adds a unique object-key index. Cleanup claims rows under SQLite `BEGIN IMMEDIATE`, pins the observed ETag, uses only version-conditional storage deletion, and can resume after a failed final catalog acknowledgment. File storage is rejected because it lacks cross-process version fencing. Keep each Blob object key unique and immutable, including across manual S3 writes. Completed unbound and old bound retention, File fencing and direct-upload quiescence remain outside this service. For actual JPEG/PNG parsing, install `teaql-blob-reference[image]` and inject `teaql_blob_reference.content_pillow.PillowBlobContentVerifier`. The image extra diff --git a/packages/teaql-blob-reference/src/teaql_blob_reference/__init__.py b/packages/teaql-blob-reference/src/teaql_blob_reference/__init__.py index 9e598681..796a9f7e 100644 --- a/packages/teaql-blob-reference/src/teaql_blob_reference/__init__.py +++ b/packages/teaql-blob-reference/src/teaql_blob_reference/__init__.py @@ -3,12 +3,17 @@ from .sqlite_catalog import BlobBindingChange, BlobBindingTarget, BlobCatalogRead, SqliteBlobCatalog from .read_service import SqliteBlobResponseContributor from .module import BlobReferenceModule +from .maintenance_service import (BlobMaintenanceAuthorization, BlobMaintenanceBatch, + BlobMaintenanceCursor, BlobMaintenanceOutcome, + SqliteBlobMaintenanceService) from .mutation_participant import SqliteBlobMutationParticipant from .file_storage import FileObjectStorage from .upload_service import (BlobUploadInput, BlobUploadSession, BlobUploadTarget, BlobVerifiedContent, SqliteBlobUploadService) __all__ = ["BlobReferenceModule", "SqliteBlobMutationParticipant", "BlobBindingChange", "BlobBindingTarget", "BlobCatalogRead", + "BlobMaintenanceAuthorization", "BlobMaintenanceBatch", "BlobMaintenanceCursor", + "BlobMaintenanceOutcome", "SqliteBlobMaintenanceService", "SqliteBlobCatalog", "SqliteBlobResponseContributor", "FileObjectStorage", "BlobUploadInput", "BlobUploadSession", "BlobUploadTarget", "BlobVerifiedContent", "SqliteBlobUploadService"] diff --git a/packages/teaql-blob-reference/src/teaql_blob_reference/maintenance_service.py b/packages/teaql-blob-reference/src/teaql_blob_reference/maintenance_service.py new file mode 100644 index 00000000..53d6bf41 --- /dev/null +++ b/packages/teaql-blob-reference/src/teaql_blob_reference/maintenance_service.py @@ -0,0 +1,307 @@ +"""Explicit SQLite/S3 cleanup of expired, uncompleted server-stream uploads.""" + +from __future__ import annotations + +import asyncio +from contextlib import suppress +from dataclasses import dataclass +import time +from typing import Callable, Protocol + +from teaql.core.blob_ref import BlobRef +from teaql.core.object_storage import ObjectKey, StorageError +from teaql.core.value import Value +from teaql.sql.executor import SqlTransactionTransport +from teaql.sql.types import CompiledQuery + +from .sqlite_catalog import SqliteBlobCatalog, _scope + + +class BlobMaintenanceAuthorization(Protocol): + def scope(self, context) -> str: ... + async def can_maintain(self, context, scope: str) -> bool: ... + + +@dataclass(frozen=True, repr=False) +class BlobMaintenanceCursor: + scope: str + after: str + + def __repr__(self): + return "BlobMaintenanceCursor[masked]" + + +@dataclass(frozen=True) +class BlobMaintenanceOutcome: + reference: BlobRef + status: str + error: str | None = None + + +@dataclass(frozen=True) +class BlobMaintenanceBatch: + outcomes: tuple[BlobMaintenanceOutcome, ...] + next: BlobMaintenanceCursor | None + + +def _query(sql, *params): + return CompiledQuery(sql, list(params)) + + +def _version(value): + return (isinstance(value, str) and value and value != "*" + and len(value) <= 256 and value == value.strip() + and not any(ord(ch) < 32 or 127 <= ord(ch) <= 159 for ch in value)) + + +class SqliteBlobMaintenanceService: + """The host schedules this service; installing a Blob module never runs it.""" + + def __init__(self, catalog: SqliteBlobCatalog, authorization: BlobMaintenanceAuthorization, + batch_size: int = 100, now_ms: Callable[[], int] | None = None): + if (not isinstance(catalog, SqliteBlobCatalog) + or not isinstance(catalog.transport, SqlTransactionTransport) + or authorization is None + or not callable(getattr(authorization, "scope", None)) + or not callable(getattr(authorization, "can_maintain", None)) + or type(batch_size) is not int or not 1 <= batch_size <= 1000): + raise RuntimeError("BLOB_POLICY_INVALID") + self.catalog = catalog + self.authorization = authorization + self.batch_size = batch_size + self.now_ms = now_ms or (lambda: time.time_ns() // 1_000_000) + + async def _authorize(self, actor, expected: str | None = None) -> str: + if actor is None: + raise RuntimeError("BLOB_MAINTENANCE_DENIED") + try: + scope = self.authorization.scope(actor) + allowed = await self.authorization.can_maintain(actor, scope) + except Exception: + raise RuntimeError("BLOB_MAINTENANCE_DENIED") from None + if not _scope(scope) or (expected is not None and scope != expected) or not allowed: + raise RuntimeError("BLOB_MAINTENANCE_DENIED") + return scope + + def _now(self) -> int: + value = self.now_ms() + if type(value) is not int or value < 0: + raise RuntimeError("BLOB_POLICY_INVALID") + return value + + async def _transaction(self, work): + try: + tx = await self.catalog.transport.begin_sql() + except Exception: + raise RuntimeError("BLOB_CATALOG_FAILED") from None + committed = False + try: + result = await work(tx) + try: + await tx.commit_sql() + except Exception: + raise RuntimeError("BLOB_CATALOG_FAILED") from None + committed = True + return result + finally: + if not committed: + with suppress(Exception): + await tx.rollback_sql() + + async def _row(self, sql, scope: str, key: str): + try: + rows = await sql.fetch_all_sql(_query( + "SELECT c.object_key,c.content_version,c.status,u.state,u.expires_ms,u.written_etag " + "FROM teaql_blob_catalog c JOIN teaql_blob_upload u " + "ON u.key=c.key AND u.scope=c.scope WHERE c.scope=? AND c.key=?", + Value.Text(scope), Value.Text(key))) + except Exception: + raise RuntimeError("BLOB_CATALOG_FAILED") from None + if not rows: + return None + if len(rows) != 1: + raise RuntimeError("BLOB_CATALOG_FAILED") + row = rows[0] + object_key, version = row.get("object_key"), row.get("content_version") + raw_written = row.get("written_etag") + written = "" if raw_written is None else raw_written + if (not isinstance(object_key, str) or not object_key + or not isinstance(version, str) or version and not _version(version) + or not isinstance(row.get("status"), str) + or not isinstance(row.get("state"), str) + or type(row.get("expires_ms")) is not int + or not isinstance(written, str) or written and not _version(written) + or row["state"] == "written" and not written): + raise RuntimeError("BLOB_CATALOG_FAILED") + return {"key": key, "object_key": object_key, "version": version, + "status": row["status"], "state": row["state"], + "expires_ms": row["expires_ms"], "written_etag": written} + + @staticmethod + def _eligible(row, now): + return (row is not None and row["expires_ms"] <= now + and row["state"] in ("initiated", "written") + and row["status"] in ("initiated", "deleting")) + + async def _count(self, sql, statement, key): + try: + rows = await sql.fetch_all_sql(_query(statement, Value.Text(key))) + except Exception: + raise RuntimeError("BLOB_CATALOG_FAILED") from None + if len(rows) != 1 or type(rows[0].get("n")) is not int or rows[0]["n"] < 0: + raise RuntimeError("BLOB_CATALOG_FAILED") + return rows[0]["n"] + + async def _claim(self, scope, key, now): + async def work(sql): + row = await self._row(sql, scope, key) + if not self._eligible(row, now): + return None + if await self._count(sql, "SELECT count(*) AS n FROM teaql_blob_binding WHERE key=?", key): + return None + owners = await self._count(sql, "SELECT count(*) AS n FROM teaql_blob_catalog WHERE object_key=?", + row["object_key"]) + if owners != 1: + raise RuntimeError("BLOB_CATALOG_FAILED") + if row["status"] == "initiated": + try: + affected, _ = await sql.execute_sql(_query( + "UPDATE teaql_blob_catalog SET status='deleting' " + "WHERE key=? AND scope=? AND status='initiated'", + Value.Text(key), Value.Text(scope))) + except Exception: + raise RuntimeError("BLOB_CATALOG_FAILED") from None + if affected != 1: + raise RuntimeError("BLOB_CATALOG_FAILED") + return row + return await self._transaction(work) + + async def _capture(self, scope, claim, version, now): + if not _version(version): + raise RuntimeError("BLOB_IDEMPOTENCY_CONFLICT") + async def work(sql): + row = await self._row(sql, scope, claim["key"]) + bound = await self._count(sql, "SELECT count(*) AS n FROM teaql_blob_binding WHERE key=?", + claim["key"]) + if (not self._eligible(row, now) or row["status"] != "deleting" + or row["object_key"] != claim["object_key"] or bound + or row["written_etag"] and row["written_etag"] != version + or row["version"] and row["version"] != version): + raise RuntimeError("BLOB_IDEMPOTENCY_CONFLICT") + if not row["version"]: + try: + affected, _ = await sql.execute_sql(_query( + "UPDATE teaql_blob_catalog SET content_version=? " + "WHERE key=? AND scope=? AND status='deleting' AND content_version=''", + Value.Text(version), Value.Text(claim["key"]), Value.Text(scope))) + except Exception: + raise RuntimeError("BLOB_CATALOG_FAILED") from None + if affected != 1: + raise RuntimeError("BLOB_IDEMPOTENCY_CONFLICT") + await self._transaction(work) + + async def _finish(self, scope, claim, version, now): + async def work(sql): + row = await self._row(sql, scope, claim["key"]) + if row is None or row["object_key"] != claim["object_key"] or row["version"] != version: + raise RuntimeError("BLOB_IDEMPOTENCY_CONFLICT") + if row["status"] == "deleted": + return + bound = await self._count(sql, "SELECT count(*) AS n FROM teaql_blob_binding WHERE key=?", + claim["key"]) + if not self._eligible(row, now) or row["status"] != "deleting" or bound: + raise RuntimeError("BLOB_IDEMPOTENCY_CONFLICT") + try: + affected, _ = await sql.execute_sql(_query( + "UPDATE teaql_blob_catalog SET status='deleted' " + "WHERE key=? AND scope=? AND status='deleting' AND content_version=?", + Value.Text(claim["key"]), Value.Text(scope), Value.Text(version))) + except Exception: + raise RuntimeError("BLOB_CATALOG_FAILED") from None + if affected != 1: + raise RuntimeError("BLOB_CATALOG_FAILED") + await self._transaction(work) + + async def _collect(self, actor, scope, key, store): + try: + await self._authorize(actor, scope) + claim = await self._claim(scope, key, self._now()) + if claim is None: + return BlobMaintenanceOutcome(BlobRef(key), "not_eligible") + await self._authorize(actor, scope) + try: + object_key = ObjectKey(claim["object_key"]) + except StorageError: + raise RuntimeError("BLOB_CATALOG_FAILED") from None + try: + observed = await asyncio.to_thread(store.head, object_key) + except Exception: + raise RuntimeError("BLOB_STORE_FAILED") from None + version = claim["version"] + if observed is not None: + if (not _version(observed.etag) or version and observed.etag != version + or claim["written_etag"] and observed.etag != claim["written_etag"]): + raise RuntimeError("BLOB_IDEMPOTENCY_CONFLICT") + version = observed.etag + await self._authorize(actor, scope) + await self._capture(scope, claim, version, self._now()) + await self._authorize(actor, scope) + result = "already_missing" + if version: + try: + deleted = await asyncio.to_thread(store.delete_if_version, object_key, version) + except Exception: + raise RuntimeError("BLOB_STORE_FAILED") from None + if deleted == "version_mismatch": + raise RuntimeError("BLOB_IDEMPOTENCY_CONFLICT") + result = deleted + await self._authorize(actor, scope) + await self._finish(scope, claim, version, self._now()) + return BlobMaintenanceOutcome(BlobRef(key), result) + except RuntimeError as error: + code = str(error) + if code in ("BLOB_MAINTENANCE_DENIED", "BLOB_TRANSACTION_REQUIRED"): + raise + if code == "BLOB_IDEMPOTENCY_CONFLICT": + return BlobMaintenanceOutcome(BlobRef(key), "version_conflict", code) + if code != "BLOB_CATALOG_FAILED": + code = "BLOB_STORE_FAILED" + return BlobMaintenanceOutcome(BlobRef(key), "failed", code) + + async def run_batch(self, actor, cursor: BlobMaintenanceCursor | None = None) -> BlobMaintenanceBatch: + scope = await self._authorize(actor) + if cursor is not None and (not isinstance(cursor, BlobMaintenanceCursor) + or cursor.scope != scope): + raise RuntimeError("BLOB_MAINTENANCE_DENIED") + actor.require_no_active_graph_sql() + try: + store = actor.object_storage("collect expired Blob upload") + except Exception: + raise RuntimeError("BLOB_CAPABILITY_MISSING") from None + if not store.supports_version_conditional_delete(): + raise RuntimeError("BLOB_CAPABILITY_MISSING") + await self.catalog.validate_schema() + now = self._now() + after = cursor.after if cursor else "" + try: + rows = await self.catalog.transport.fetch_all_sql(_query( + "SELECT DISTINCT c.key FROM teaql_blob_catalog c JOIN teaql_blob_upload u " + "ON u.key=c.key AND u.scope=c.scope WHERE c.scope=? AND c.key>? " + "AND u.expires_ms<=? AND u.state IN ('initiated','written') " + "AND c.status IN ('initiated','deleting') " + "AND NOT EXISTS (SELECT 1 FROM teaql_blob_binding b WHERE b.key=c.key) " + "ORDER BY c.key LIMIT ?", Value.Text(scope), Value.Text(after), + Value.I64(now), Value.I64(self.batch_size))) + except Exception: + raise RuntimeError("BLOB_CATALOG_FAILED") from None + keys = [] + for row in rows: + key = row.get("key") + try: + BlobRef.from_sql(key) + except (ValueError, TypeError): + raise RuntimeError("BLOB_CATALOG_FAILED") from None + keys.append(key) + outcomes = tuple([await self._collect(actor, scope, key, store) for key in keys]) + next_cursor = BlobMaintenanceCursor(scope, keys[-1]) if len(keys) == self.batch_size else None + return BlobMaintenanceBatch(outcomes, next_cursor) diff --git a/packages/teaql-blob-reference/src/teaql_blob_reference/module.py b/packages/teaql-blob-reference/src/teaql_blob_reference/module.py index a66d6420..94ca8e62 100644 --- a/packages/teaql-blob-reference/src/teaql_blob_reference/module.py +++ b/packages/teaql-blob-reference/src/teaql_blob_reference/module.py @@ -6,6 +6,7 @@ from .mutation_participant import SqliteBlobMutationParticipant from .sqlite_catalog import SqliteBlobCatalog from .upload_service import SqliteBlobUploadService +from .maintenance_service import SqliteBlobMaintenanceService class _BlobCapability: @@ -19,6 +20,9 @@ async def validate_ready(self, context) -> None: context.get_resource("blob_mutation_participant") is not self.module.mutation or \ context.get_resource("object_storage_provider") is None: raise RuntimeError("BLOB_CAPABILITY_MISSING") + if self.module.maintenance is not None and \ + context.get_resource("blob_maintenance_service") is not self.module.maintenance: + raise RuntimeError("BLOB_CAPABILITY_MISSING") owner = context.get_resource("dataService") if owner is None or not callable(getattr(owner, "begin", None)): raise RuntimeError("BLOB_CAPABILITY_MISSING") @@ -32,24 +36,31 @@ class BlobReferenceModule: """Host installs this module, then separately migrates and checks its catalog.""" def __init__(self, catalog: SqliteBlobCatalog, upload: SqliteBlobUploadService, - read_authorization, binding_authorization, maximum_url_seconds: int = 600): + read_authorization, binding_authorization, maximum_url_seconds: int = 600, + maintenance: SqliteBlobMaintenanceService | None = None): if not isinstance(catalog, SqliteBlobCatalog) or \ - not isinstance(upload, SqliteBlobUploadService) or upload.catalog is not catalog: + not isinstance(upload, SqliteBlobUploadService) or upload.catalog is not catalog or \ + maintenance is not None and (not isinstance(maintenance, SqliteBlobMaintenanceService) + or maintenance.catalog is not catalog): raise RuntimeError("BLOB_CAPABILITY_MISSING") self.catalog = catalog self.upload = upload self.mutation = SqliteBlobMutationParticipant(catalog, binding_authorization) self.read = SqliteBlobResponseContributor(catalog, read_authorization, maximum_url_seconds) + self.maintenance = maintenance def install(self, context) -> None: if context is None or context.get_resource("blob_response_contributor") is not None or \ context.get_resource("blob_mutation_participant") is not None or \ + (self.maintenance is not None and context.get_resource("blob_maintenance_service") is not None) or \ context.get_resource("teaql.runtime.capability:blob.v1") is not None: raise RuntimeError("BLOB_CAPABILITY_MISSING") install_runtime_capability(context, _BlobCapability(self)) context.insert_resource("blob_response_contributor", self.read) context.insert_resource("blob_mutation_participant", self.mutation) + if self.maintenance is not None: + context.insert_resource("blob_maintenance_service", self.maintenance) async def ready(self, context) -> None: capability = context.get_resource("teaql.runtime.capability:blob.v1") diff --git a/packages/teaql-blob-reference/src/teaql_blob_reference/sqlite_catalog.py b/packages/teaql-blob-reference/src/teaql_blob_reference/sqlite_catalog.py index 9b37c7ac..c1635592 100644 --- a/packages/teaql-blob-reference/src/teaql_blob_reference/sqlite_catalog.py +++ b/packages/teaql-blob-reference/src/teaql_blob_reference/sqlite_catalog.py @@ -89,6 +89,7 @@ async def ensure_schema(self): FOREIGN KEY (key) REFERENCES teaql_blob_catalog(key)) """, "CREATE INDEX IF NOT EXISTS teaql_blob_binding_key ON teaql_blob_binding(key)", + "CREATE UNIQUE INDEX IF NOT EXISTS teaql_blob_catalog_object_key ON teaql_blob_catalog(object_key)", """CREATE TABLE IF NOT EXISTS teaql_blob_upload ( upload_id TEXT PRIMARY KEY, scope TEXT NOT NULL, replay_key TEXT NOT NULL, request_hash TEXT NOT NULL, key TEXT NOT NULL, entity TEXT NOT NULL, @@ -125,6 +126,15 @@ async def validate_schema(self): names = {row["name"]: row for row in columns} if names["key"]["notnull"] != 0 or names["version"]["notnull"] != 1: raise RuntimeError("BLOB_SCHEMA_MISSING") + indexes = await self.transport.fetch_all_sql(CompiledQuery( + "PRAGMA index_list(teaql_blob_catalog)", [])) + if not any(row.get("name") == "teaql_blob_catalog_object_key" + and row.get("unique") == 1 for row in indexes): + raise RuntimeError("BLOB_SCHEMA_MISSING") + index_columns = await self.transport.fetch_all_sql(CompiledQuery( + "PRAGMA index_info(teaql_blob_catalog_object_key)", [])) + if len(index_columns) != 1 or index_columns[0].get("name") != "object_key": + raise RuntimeError("BLOB_SCHEMA_MISSING") except Exception: raise RuntimeError("BLOB_SCHEMA_MISSING") from None diff --git a/src/teaql/runtime/context.py b/src/teaql/runtime/context.py index beb27d33..03d2fa45 100644 --- a/src/teaql/runtime/context.py +++ b/src/teaql/runtime/context.py @@ -220,6 +220,12 @@ def decode_entity_reference( ) return claims + def require_no_active_graph_sql(self) -> None: + """Reject maintenance that would wait on its own graph SQL owner.""" + guard = self._graph_save_guard.get() + if self._graph_session is not None or (guard is not None and guard["active"]): + raise RuntimeError("BLOB_TRANSACTION_REQUIRED") + async def execute_graph_save(self, work, *, comment=None): """Run ``work(session)``; never implicitly join another root save.""" from teaql.core.request_intent import MutationIntent diff --git a/src/teaql/runtime/object_storage.py b/src/teaql/runtime/object_storage.py index dd1b21b8..f105a2fc 100644 --- a/src/teaql/runtime/object_storage.py +++ b/src/teaql/runtime/object_storage.py @@ -241,5 +241,8 @@ def delete(): return result return self._call("delete_if_version", delete) + def supports_version_conditional_delete(self) -> bool: + return callable(getattr(self._provider, "delete_if_version", None)) + def get_read_url(self, key, options=ReadUrlOptions()): return self._call("read_url", lambda: self._provider.get_read_url(self._invocation, key, options)) diff --git a/tests/provider/sqlite/test_blob_maintenance.py b/tests/provider/sqlite/test_blob_maintenance.py new file mode 100644 index 00000000..387b800e --- /dev/null +++ b/tests/provider/sqlite/test_blob_maintenance.py @@ -0,0 +1,233 @@ +"""Real SQLite state transitions for explicit, fenced Blob expiry cleanup.""" + +import aiosqlite +import pytest +from uuid import uuid4 +import os + +from teaql.core.object_storage import ObjectKey, ObjectMetadata, PutOptions, WriteMode +from teaql.provider.sqlite.transport import SqliteTransport +from teaql.runtime import RuntimeModule +from teaql_blob_reference import SqliteBlobCatalog, SqliteBlobMaintenanceService + + +class _MaintenanceAuthority: + def __init__(self): + self.current_scope = "tenant-a" + self.allowed = True + + def scope(self, context): + return self.current_scope + + async def can_maintain(self, context, scope): + return self.allowed + + +class _ConditionalStore: + def __init__(self): + self.objects = {} + self.deletes = 0 + + def head(self, invocation, key): + etag = self.objects.get(key.value) + return ObjectMetadata(key, 1, "application/pdf", etag) if etag else None + + def delete_if_version(self, invocation, key, version): + self.deletes += 1 + actual = self.objects.get(key.value) + if actual is None: + return "already_missing" + if actual != version: + return "version_mismatch" + del self.objects[key.value] + return "deleted" + + def delete(self, invocation, key): + raise AssertionError("unconditional delete is forbidden") + + +async def _seed(path, key, *, scope="tenant-a", state="written", version="", + written_etag=None, expires=9000, bound=False): + async with aiosqlite.connect(path) as db: + await db.execute("""INSERT INTO teaql_blob_catalog + (key,scope,object_key,content_version,file_name,content_type,size_bytes,status) + VALUES (?,?,?,?,?,?,?,?)""", (key, scope, "objects/" + key, version, + "file.pdf", "", 0, + "uploaded_unbound" if state == "completed" else "initiated")) + await db.execute("""INSERT INTO teaql_blob_upload + (upload_id,scope,replay_key,request_hash,key,entity,entity_id,field,file_name, + extension,declared_size,expires_ms,state,written_etag) + VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?)""", ("bu_" + key, scope, "replay_" + key, + "hash_" + key, key, "Asset", "1", "document", "file.pdf", "pdf", + 1, expires, state, written_etag)) + if bound: + await db.execute("""INSERT INTO teaql_blob_binding + (scope,entity,entity_id,field,key,version) VALUES (?,?,?,?,?,?)""", + (scope, "Asset", key, "document", key, 1)) + await db.commit() + + +async def _status(path, key): + async with aiosqlite.connect(path) as db: + async with db.execute("SELECT status FROM teaql_blob_catalog WHERE key=?", (key,)) as cursor: + return (await cursor.fetchone())[0] + + +@pytest.mark.asyncio +async def test_expired_upload_maintenance_claim_scope_version_and_recovery(tmp_path): + path = str(tmp_path / "maintenance.db") + catalog = SqliteBlobCatalog(SqliteTransport(path)) + await catalog.ensure_schema() + actor = RuntimeModule.new().into_context() + store = _ConditionalStore() + actor.with_object_storage(store).with_object_storage_observer(lambda event: None) + auth = _MaintenanceAuthority() + service = SqliteBlobMaintenanceService(catalog, auth, 2, lambda: 10000) + await _seed(path, "blob_absent", state="initiated") + await _seed(path, "blob_bound", state="initiated", bound=True) + await _seed(path, "blob_completed", state="completed", written_etag="etag-completed") + await _seed(path, "blob_future", expires=20000, written_etag="etag-future") + await _seed(path, "blob_other", scope="tenant-b", written_etag="etag-other") + await _seed(path, "blob_race", version="etag-old", written_etag="etag-old") + await _seed(path, "blob_written", written_etag="etag-written") + store.objects.update({"objects/blob_race": "etag-new", + "objects/blob_written": "etag-written", + "objects/blob_completed": "etag-completed", + "objects/blob_future": "etag-future", + "objects/blob_other": "etag-other"}) + await catalog.validate_schema() + auth.allowed = False + with pytest.raises(RuntimeError, match="^BLOB_MAINTENANCE_DENIED$"): + await service.run_batch(actor) + auth.allowed = True + first = await service.run_batch(actor) + assert [(o.reference.key, o.status) for o in first.outcomes] == [ + ("blob_absent", "already_missing"), ("blob_race", "version_conflict")] + assert first.next is not None + second = await service.run_batch(actor, first.next) + assert [(o.reference.key, o.status) for o in second.outcomes] == [ + ("blob_written", "deleted")] + assert store.deletes == 1 + assert store.objects["objects/blob_race"] == "etag-new" + for key, expected in {"blob_absent": "deleted", "blob_race": "deleting", + "blob_written": "deleted", "blob_completed": "uploaded_unbound", + "blob_future": "initiated", "blob_other": "initiated", + "blob_bound": "initiated"}.items(): + assert await _status(path, key) == expected + auth.current_scope = "tenant-b" + with pytest.raises(RuntimeError, match="^BLOB_MAINTENANCE_DENIED$"): + await service.run_batch(actor, first.next) + auth.current_scope = "tenant-a" + await _seed(path, "blob_retry", written_etag="etag-retry") + store.objects["objects/blob_retry"] = "etag-retry" + async with aiosqlite.connect(path) as db: + await db.execute("""CREATE TRIGGER fail_final_ack BEFORE UPDATE OF status ON teaql_blob_catalog + WHEN NEW.status='deleted' AND OLD.key='blob_retry' + BEGIN SELECT RAISE(ABORT,'private trigger detail'); END""") + await db.commit() + failed = await service.run_batch(actor) + assert failed.outcomes[1].status == "failed" + assert failed.outcomes[1].error == "BLOB_CATALOG_FAILED" + assert "objects/blob_retry" not in store.objects + assert await _status(path, "blob_retry") == "deleting" + async with aiosqlite.connect(path) as db: + await db.execute("DROP TRIGGER fail_final_ack") + await db.commit() + recovered = await service.run_batch(actor) + assert recovered.outcomes[1].status == "already_missing" + assert await _status(path, "blob_retry") == "deleted" + + +@pytest.mark.asyncio +async def test_expired_upload_maintenance_rejects_unfenced_store_graph_and_schema(tmp_path): + path = str(tmp_path / "maintenance-gates.db") + catalog = SqliteBlobCatalog(SqliteTransport(path)) + await catalog.ensure_schema() + actor = RuntimeModule.new().into_context() + auth = _MaintenanceAuthority() + service = SqliteBlobMaintenanceService(catalog, auth, 2, lambda: 10000) + actor.with_object_storage(object()) + with pytest.raises(RuntimeError, match="^BLOB_CAPABILITY_MISSING$"): + await service.run_batch(actor) + actor.with_object_storage(_ConditionalStore()) + token = actor._graph_save_guard.set({"active": True}) + try: + with pytest.raises(RuntimeError, match="^BLOB_TRANSACTION_REQUIRED$"): + await service.run_batch(actor) + finally: + actor._graph_save_guard.reset(token) + async with aiosqlite.connect(path) as db: + await db.execute("DROP INDEX teaql_blob_catalog_object_key") + await db.commit() + with pytest.raises(RuntimeError, match="^BLOB_SCHEMA_MISSING$"): + await service.run_batch(actor) + + +@pytest.mark.asyncio +async def test_real_s3_expired_upload_maintenance_delete_recovery_and_stale_version(tmp_path): + if os.environ.get("TEAQL_S3_TEST_BLOB_MAINTENANCE") != "1": + pytest.skip("explicit disposable S3 maintenance gate required") + endpoint = os.environ.get("TEAQL_S3_TEST_ENDPOINT") + bucket = os.environ.get("TEAQL_S3_TEST_BUCKET") + access = os.environ.get("TEAQL_S3_TEST_ACCESS_KEY") + secret = os.environ.get("TEAQL_S3_TEST_SECRET_KEY") + assert all((endpoint, bucket, access, secret)), "disposable S3 configuration required" + from teaql.provider.s3 import S3ObjectStorage + provider = S3ObjectStorage.connect(bucket, endpoint_url=endpoint, + aws_access_key_id=access, aws_secret_access_key=secret) + actor = RuntimeModule.new().into_context() + actor.with_object_storage(provider).with_object_storage_observer(lambda event: None) + store = actor.object_storage("real S3 expired Blob maintenance") + path = str(tmp_path / "s3-maintenance.db") + catalog = SqliteBlobCatalog(SqliteTransport(path)) + await catalog.ensure_schema() + service = SqliteBlobMaintenanceService(catalog, _MaintenanceAuthority(), 10, lambda: 10000) + prefix = uuid4().hex + keys = [] + + async def seed(suffix, content): + reference = "blob_" + prefix + "_" + suffix + key = ObjectKey("objects/" + reference) + keys.append(key) + created = store.put_bytes(key, content, PutOptions(len(content), WriteMode.CREATE_ONLY)) + await _seed(path, reference, written_etag=created.etag) + return reference, key, created.etag + + try: + first_ref, first_key, _ = await seed("first", b"first version") + first = await service.run_batch(actor) + assert [(o.reference.key, o.status) for o in first.outcomes] == [(first_ref, "deleted")] + assert store.head(first_key) is None + assert await _status(path, first_ref) == "deleted" + + recovery_ref, recovery_key, _ = await seed("recovery", b"recovery version") + async with aiosqlite.connect(path) as db: + await db.execute("""CREATE TRIGGER fail_final_ack BEFORE UPDATE OF status ON teaql_blob_catalog + WHEN NEW.status='deleted' AND OLD.key='""" + recovery_ref + """' + BEGIN SELECT RAISE(ABORT,'private trigger detail'); END""") + await db.commit() + failed = await service.run_batch(actor) + assert [(o.reference.key, o.status, o.error) for o in failed.outcomes] == [ + (recovery_ref, "failed", "BLOB_CATALOG_FAILED")] + assert store.head(recovery_key) is None + assert await _status(path, recovery_ref) == "deleting" + async with aiosqlite.connect(path) as db: + await db.execute("DROP TRIGGER fail_final_ack") + await db.commit() + recovered = await service.run_batch(actor) + assert [(o.reference.key, o.status) for o in recovered.outcomes] == [ + (recovery_ref, "already_missing")] + assert await _status(path, recovery_ref) == "deleted" + + race_ref, race_key, _ = await seed("stale", b"old version") + replacement_bytes = b"replacement version" + replacement = store.put_bytes(race_key, replacement_bytes, + PutOptions(len(replacement_bytes), WriteMode.REPLACE)) + conflict = await service.run_batch(actor) + assert [(o.reference.key, o.status) for o in conflict.outcomes] == [ + (race_ref, "version_conflict")] + assert store.head(race_key).etag == replacement.etag + assert await _status(path, race_ref) == "deleting" + finally: + for key in keys: + store.delete(key) diff --git a/tests/provider/sqlite/test_blob_module.py b/tests/provider/sqlite/test_blob_module.py index 892c14d0..e3d85100 100644 --- a/tests/provider/sqlite/test_blob_module.py +++ b/tests/provider/sqlite/test_blob_module.py @@ -16,7 +16,8 @@ from teaql.runtime import RuntimeModule, validate_capabilities from teaql.sql.executor import SqlDataServiceExecutor from teaql_blob_reference import ( - BlobReferenceModule, FileObjectStorage, SqliteBlobCatalog, SqliteBlobUploadService, + BlobReferenceModule, FileObjectStorage, SqliteBlobCatalog, + SqliteBlobMaintenanceService, SqliteBlobUploadService, ) @@ -57,6 +58,14 @@ async def authorize(self, context, target, previous, next_ref, action): return self.allowed and target.entity == "Asset" and target.field == "document" +class _MaintenanceAuthority: + def scope(self, context): + return "tenant-a" + + async def can_maintain(self, context, scope): + return scope == "tenant-a" + + class _SafeAuditSink: def __init__(self): self.events = [] @@ -72,12 +81,15 @@ async def test_blob_module_requires_provider_owner_and_catalog(tmp_path): catalog = SqliteBlobCatalog(transport) verifier = _Verifier() upload = SqliteBlobUploadService(catalog, _UploadAuthority(), verifier, 10_000) - module = BlobReferenceModule(catalog, upload, _ReadAuthority(), _BindingAuthority()) + maintenance = SqliteBlobMaintenanceService(catalog, _MaintenanceAuthority()) + module = BlobReferenceModule(catalog, upload, _ReadAuthority(), _BindingAuthority(), + maintenance=maintenance) context = RuntimeModule.new().require_capability("blob.v1").into_context() with pytest.raises(RuntimeError, match="BLOB_CAPABILITY_MISSING"): await validate_capabilities(context) module.install(context) assert context.get_resource("blob_response_contributor") is module.read + assert context.get_resource("blob_maintenance_service") is maintenance with pytest.raises(RuntimeError, match="BLOB_CAPABILITY_MISSING"): await module.ready(context) with pytest.raises(RuntimeError, match="BLOB_CAPABILITY_MISSING"): From 8496251cf2ff6512c37156b1c708eca84cca62b8 Mon Sep 17 00:00:00 2001 From: Philip Z Date: Wed, 28 Oct 2026 03:59:01 +0800 Subject: [PATCH 36/75] Fence Python File Blob deletion across processes --- packages/teaql-blob-reference/README.md | 2 +- .../src/teaql_blob_reference/file_storage.py | 112 +++++++++++++++--- .../maintenance_service.py | 27 +++-- .../provider/sqlite/test_blob_maintenance.py | 53 ++++++++- .../provider/sqlite/test_file_blob_storage.py | 54 +++++++++ 5 files changed, 219 insertions(+), 29 deletions(-) diff --git a/packages/teaql-blob-reference/README.md b/packages/teaql-blob-reference/README.md index 3987a32b..c42fe500 100644 --- a/packages/teaql-blob-reference/README.md +++ b/packages/teaql-blob-reference/README.md @@ -4,7 +4,7 @@ Optional Blob catalog and current-read services for TeaQL Python. Install this w `SqliteBlobUploadService` offers a host-invoked, scope-bound upload journal. The host must provide current model/action authorization and a trusted parser that consumes every stored byte and returns actual format, size, and SHA256. The service writes through the context's object-storage provider, rechecks the stored generation before confirmation, and promotes the row to `uploaded_unbound` only after full verification. The included PDF verifier in tests is a fixture, not a production format parser. Full retention/GC, other SQL backends, and production ten-format parsing remain open. -`SqliteBlobMaintenanceService` is an optional, explicitly host-scheduled service for **expired, uncompleted, unbound server-stream uploads only**. The host supplies a current scope and async `can_maintain` policy, then calls `run_batch(context, cursor)` and may install the service through `BlobReferenceModule(..., maintenance=service)`. Run `catalog.ensure_schema()` as a separate migration before readiness; it adds a unique object-key index. Cleanup claims rows under SQLite `BEGIN IMMEDIATE`, pins the observed ETag, uses only version-conditional storage deletion, and can resume after a failed final catalog acknowledgment. File storage is rejected because it lacks cross-process version fencing. Keep each Blob object key unique and immutable, including across manual S3 writes. Completed unbound and old bound retention, File fencing and direct-upload quiescence remain outside this service. +`SqliteBlobMaintenanceService` is an optional, explicitly host-scheduled service for **expired, uncompleted, unbound server-stream uploads only**. The host supplies a current scope and async `can_maintain` policy, then calls `run_batch(context, cursor)` and may install the service through `BlobReferenceModule(..., maintenance=service)`. Run `catalog.ensure_schema()` as a separate migration before readiness; it adds a unique object-key index. Cleanup claims rows under SQLite `BEGIN IMMEDIATE`, pins the observed ETag, uses only version-conditional storage deletion, and can resume after a failed final catalog acknowledgment. `FileObjectStorage` now guards manifest publication and conditional deletion with a persistent per-key OS file lock. Its ETag includes an immutable generation token, so identical-byte replacement cannot match an older version. A matching delete removes the current data generation and manifest; older immutable generations remain for a separate collector. Keep each Blob object key unique and immutable, including across manual S3 writes. Completed unbound and old bound retention and direct-upload quiescence remain outside this service. For actual JPEG/PNG parsing, install `teaql-blob-reference[image]` and inject `teaql_blob_reference.content_pillow.PillowBlobContentVerifier`. The image extra diff --git a/packages/teaql-blob-reference/src/teaql_blob_reference/file_storage.py b/packages/teaql-blob-reference/src/teaql_blob_reference/file_storage.py index 4044cc8f..56cbd3bf 100644 --- a/packages/teaql-blob-reference/src/teaql_blob_reference/file_storage.py +++ b/packages/teaql-blob-reference/src/teaql_blob_reference/file_storage.py @@ -1,4 +1,5 @@ """Optional private file store with an explicitly started authenticated local HTTP gateway.""" +from contextlib import contextmanager from datetime import datetime, timedelta, timezone from hashlib import sha256 from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer @@ -13,6 +14,11 @@ import stat import time +if os.name == "posix": + import fcntl +elif os.name == "nt": + import msvcrt + from cryptography.hazmat.primitives.ciphers.aead import AESGCM from teaql.core.object_storage import ( @@ -71,6 +77,53 @@ def _private_root(self, create=False): def _directory(self, key_hash): return self._root / key_hash[:2] / key_hash + @contextmanager + def _manifest_lock(self, key_hash): + """Persistent OS lock: never remove the lock file while this root is in use.""" + path = self._directory(key_hash) / "manifest.lock" + flags = os.O_RDWR | os.O_CREAT | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NOFOLLOW", 0) + fd = None + locked = False + try: + fd = os.open(path, flags, 0o600) + info = os.fstat(fd) + if not stat.S_ISREG(info.st_mode) or (os.name == "posix" and info.st_mode & 0o077): + raise _error(StorageErrorCode.UNAVAILABLE) + if os.name == "nt": + if info.st_size == 0: + os.write(fd, b"\0") + elif os.name != "posix": + raise _error(StorageErrorCode.UNSUPPORTED) + deadline = time.monotonic() + 30 + while True: + try: + if os.name == "posix": + fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB) + else: + os.lseek(fd, 0, os.SEEK_SET) + msvcrt.locking(fd, msvcrt.LK_NBLCK, 1) + break + except OSError: + if time.monotonic() >= deadline: + raise _error(StorageErrorCode.TIMEOUT) from None + time.sleep(0.02) + locked = True + yield + except StorageError: + raise + except OSError: + raise _error(StorageErrorCode.IO) from None + finally: + if fd is not None: + try: + if locked and os.name == "posix": + fcntl.flock(fd, fcntl.LOCK_UN) + elif locked and os.name == "nt": + os.lseek(fd, 0, os.SEEK_SET) + msvcrt.locking(fd, msvcrt.LK_UNLCK, 1) + finally: + os.close(fd) + @staticmethod def _hash(key): return sha256(key.value.encode("utf-8")).hexdigest() @@ -155,19 +208,20 @@ def put(self, invocation, key, content, options): output.flush() os.fsync(output.fileno()) destination = directory / "manifest.json" - if options.mode == WriteMode.CREATE_ONLY: - try: - os.link(temp_path, destination) - except FileExistsError: - raise _error(StorageErrorCode.CONFLICT) from None - published = True - try: - temp_path.unlink(missing_ok=True) - except OSError: - pass - else: - os.replace(temp_path, destination) - published = True + with self._manifest_lock(key_hash): + if options.mode == WriteMode.CREATE_ONLY: + try: + os.link(temp_path, destination) + except FileExistsError: + raise _error(StorageErrorCode.CONFLICT) from None + published = True + try: + temp_path.unlink(missing_ok=True) + except OSError: + pass + else: + os.replace(temp_path, destination) + published = True return self._metadata(key, manifest) except StorageError: raise @@ -198,6 +252,8 @@ def get(self, invocation, key): return ObjectContent(self._metadata(key, manifest), stream) def head(self, invocation, key): + if not self._root.exists(): + return None self._private_root() key_hash = self._hash(key) manifest = self._manifest(key_hash) @@ -207,9 +263,37 @@ def head(self, invocation, key): return self._metadata(key, manifest) def delete(self, invocation, key): + if not self._root.exists(): + return + self._private_root() + key_hash = self._hash(key) + if not self._directory(key_hash).exists(): + return + try: + with self._manifest_lock(key_hash): + (self._directory(key_hash) / "manifest.json").unlink(missing_ok=True) + except OSError: + raise _error(StorageErrorCode.IO) from None + + def delete_if_version(self, invocation, key, content_version): + if not self._root.exists(): + return "already_missing" self._private_root() + key_hash = self._hash(key) + if not self._directory(key_hash).exists(): + return "already_missing" try: - (self._directory(self._hash(key)) / "manifest.json").unlink(missing_ok=True) + with self._manifest_lock(key_hash): + manifest = self._manifest(key_hash) + if manifest is None: + return "already_missing" + if self._metadata(key, manifest).etag != content_version: + return "version_mismatch" + self._data_path(key_hash, manifest).unlink(missing_ok=True) + (self._directory(key_hash) / "manifest.json").unlink() + return "deleted" + except StorageError: + raise except OSError: raise _error(StorageErrorCode.IO) from None diff --git a/packages/teaql-blob-reference/src/teaql_blob_reference/maintenance_service.py b/packages/teaql-blob-reference/src/teaql_blob_reference/maintenance_service.py index 53d6bf41..da7fe24b 100644 --- a/packages/teaql-blob-reference/src/teaql_blob_reference/maintenance_service.py +++ b/packages/teaql-blob-reference/src/teaql_blob_reference/maintenance_service.py @@ -1,4 +1,4 @@ -"""Explicit SQLite/S3 cleanup of expired, uncompleted server-stream uploads.""" +"""Explicit SQLite cleanup of expired, uncompleted server-stream uploads.""" from __future__ import annotations @@ -233,18 +233,21 @@ async def _collect(self, actor, scope, key, store): object_key = ObjectKey(claim["object_key"]) except StorageError: raise RuntimeError("BLOB_CATALOG_FAILED") from None - try: - observed = await asyncio.to_thread(store.head, object_key) - except Exception: - raise RuntimeError("BLOB_STORE_FAILED") from None version = claim["version"] - if observed is not None: - if (not _version(observed.etag) or version and observed.etag != version - or claim["written_etag"] and observed.etag != claim["written_etag"]): - raise RuntimeError("BLOB_IDEMPOTENCY_CONFLICT") - version = observed.etag - await self._authorize(actor, scope) - await self._capture(scope, claim, version, self._now()) + if version and claim["written_etag"] and version != claim["written_etag"]: + raise RuntimeError("BLOB_IDEMPOTENCY_CONFLICT") + if not version: + try: + observed = await asyncio.to_thread(store.head, object_key) + except Exception: + raise RuntimeError("BLOB_STORE_FAILED") from None + if observed is not None: + if (not _version(observed.etag) + or claim["written_etag"] and observed.etag != claim["written_etag"]): + raise RuntimeError("BLOB_IDEMPOTENCY_CONFLICT") + version = observed.etag + await self._authorize(actor, scope) + await self._capture(scope, claim, version, self._now()) await self._authorize(actor, scope) result = "already_missing" if version: diff --git a/tests/provider/sqlite/test_blob_maintenance.py b/tests/provider/sqlite/test_blob_maintenance.py index 387b800e..dbfe2430 100644 --- a/tests/provider/sqlite/test_blob_maintenance.py +++ b/tests/provider/sqlite/test_blob_maintenance.py @@ -8,7 +8,7 @@ from teaql.core.object_storage import ObjectKey, ObjectMetadata, PutOptions, WriteMode from teaql.provider.sqlite.transport import SqliteTransport from teaql.runtime import RuntimeModule -from teaql_blob_reference import SqliteBlobCatalog, SqliteBlobMaintenanceService +from teaql_blob_reference import FileObjectStorage, SqliteBlobCatalog, SqliteBlobMaintenanceService class _MaintenanceAuthority: @@ -73,6 +73,55 @@ async def _status(path, key): return (await cursor.fetchone())[0] +@pytest.mark.asyncio +async def test_real_sqlite_file_maintenance_recovers_receipt_and_partial_data(tmp_path): + path = str(tmp_path / "file-maintenance.db") + catalog = SqliteBlobCatalog(SqliteTransport(path)) + await catalog.ensure_schema() + provider = FileObjectStorage(tmp_path / "file-objects", os.urandom(32)) + actor = RuntimeModule.new().into_context() + actor.with_object_storage(provider).with_object_storage_observer(lambda event: None) + store = actor.object_storage("real File expired Blob maintenance") + service = SqliteBlobMaintenanceService(catalog, _MaintenanceAuthority(), 10, lambda: 10000) + content = b"%PDF-" + key = ObjectKey("objects/blob_file_receipt") + store.put_bytes(key, content, PutOptions(len(content), WriteMode.CREATE_ONLY)) + await _seed(path, "blob_file_receipt", state="initiated") + async with aiosqlite.connect(path) as db: + await db.execute("""CREATE TRIGGER fail_file_receipt BEFORE UPDATE OF status ON teaql_blob_catalog + WHEN NEW.status='deleted' BEGIN SELECT RAISE(ABORT,'receipt failure'); END""") + await db.commit() + failed = await service.run_batch(actor) + assert [(o.reference.key, o.status) for o in failed.outcomes] == [("blob_file_receipt", "failed")] + assert await _status(path, "blob_file_receipt") == "deleting" + assert store.head(key) is None + async with aiosqlite.connect(path) as db: + await db.execute("DROP TRIGGER fail_file_receipt") + await db.commit() + retry = await service.run_batch(actor) + assert [(o.reference.key, o.status) for o in retry.outcomes] == [ + ("blob_file_receipt", "already_missing")] + assert await _status(path, "blob_file_receipt") == "deleted" + + stale_key = ObjectKey("objects/blob_file_stale") + old = store.put_bytes(stale_key, content, PutOptions(len(content), WriteMode.CREATE_ONLY)) + await _seed(path, "blob_file_stale", state="initiated", version=old.etag) + newer = store.put_bytes(stale_key, content, PutOptions(len(content), WriteMode.REPLACE)) + stale = await service.run_batch(actor) + assert [(o.reference.key, o.status) for o in stale.outcomes] == [ + ("blob_file_stale", "version_conflict")] + assert store.head(stale_key).etag == newer.etag + + partial_key = ObjectKey("objects/blob_file_partial") + partial = store.put_bytes(partial_key, content, PutOptions(len(content), WriteMode.CREATE_ONLY)) + await _seed(path, "blob_file_partial", state="initiated", version=partial.etag) + manifest = provider._manifest(provider._hash(partial_key)) + provider._data_path(provider._hash(partial_key), manifest).unlink() + recovered = await service.run_batch(actor) + assert ("blob_file_partial", "deleted") in [(o.reference.key, o.status) for o in recovered.outcomes] + assert await _status(path, "blob_file_partial") == "deleted" + + @pytest.mark.asyncio async def test_expired_upload_maintenance_claim_scope_version_and_recovery(tmp_path): path = str(tmp_path / "maintenance.db") @@ -107,7 +156,7 @@ async def test_expired_upload_maintenance_claim_scope_version_and_recovery(tmp_p second = await service.run_batch(actor, first.next) assert [(o.reference.key, o.status) for o in second.outcomes] == [ ("blob_written", "deleted")] - assert store.deletes == 1 + assert store.deletes == 2 # the stale pinned version is refused by conditional deletion assert store.objects["objects/blob_race"] == "etag-new" for key, expected in {"blob_absent": "deleted", "blob_race": "deleting", "blob_written": "deleted", "blob_completed": "uploaded_unbound", diff --git a/tests/provider/sqlite/test_file_blob_storage.py b/tests/provider/sqlite/test_file_blob_storage.py index 83c46459..5cbc5925 100644 --- a/tests/provider/sqlite/test_file_blob_storage.py +++ b/tests/provider/sqlite/test_file_blob_storage.py @@ -1,9 +1,12 @@ """Real local File/HTTP behavior for the independent optional wheel.""" +from concurrent.futures import ThreadPoolExecutor from datetime import timedelta from urllib.error import HTTPError from urllib.request import Request, urlopen import hashlib import os +import subprocess +import sys import time import pytest @@ -15,6 +18,57 @@ from teaql.runtime import UserContext +def test_file_conditional_delete_fences_same_bytes_and_external_process(tmp_path): + root = tmp_path / "objects" + first = FileObjectStorage(root, os.urandom(32)) + second = FileObjectStorage(root, os.urandom(32)) + actor = UserContext().with_object_storage(first).with_object_storage_observer(lambda event: None) + writer = actor.object_storage("write File generation") + actor.with_object_storage(second) + collector = actor.object_storage("delete File generation") + key = ObjectKey("blob/same-bytes") + assert collector.head(key) is None + assert collector.delete_if_version(key, "absent") == "already_missing" + data = b"same bytes" + old = writer.put_bytes(key, data, PutOptions(len(data), WriteMode.CREATE_ONLY)) + current = writer.put_bytes(key, data, PutOptions(len(data), WriteMode.REPLACE)) + assert old.etag != current.etag + assert collector.delete_if_version(key, old.etag) == "version_mismatch" + assert collector.get_bytes(key, len(data)) == data + assert collector.delete_if_version(key, current.etag) == "deleted" + assert collector.head(key) is None + assert collector.delete_if_version(key, current.etag) == "already_missing" + key_hash = second._hash(key) + assert not (second._directory(key_hash) / ("data-" + current.etag.split(".")[-1])).exists() + + if os.name != "posix": + return + locked_key = ObjectKey("blob/external-lock") + written = writer.put_bytes(locked_key, data, PutOptions(len(data), WriteMode.CREATE_ONLY)) + lock_path = second._directory(second._hash(locked_key)) / "manifest.lock" + marker = tmp_path / "locked" + script = ("import fcntl, os, sys, time; " + "fd=os.open(sys.argv[1], os.O_RDWR); fcntl.flock(fd, fcntl.LOCK_EX); " + "open(sys.argv[2], 'w').close(); time.sleep(2)") + process = subprocess.Popen([sys.executable, "-c", script, str(lock_path), str(marker)]) + try: + for _ in range(100): + if marker.exists(): + break + time.sleep(0.01) + assert marker.exists() and process.poll() is None + with ThreadPoolExecutor(max_workers=1) as pool: + pending = pool.submit(collector.delete_if_version, locked_key, written.etag) + time.sleep(0.1) + assert process.poll() is None and not pending.done() + assert process.wait(timeout=5) == 0 + assert pending.result(timeout=5) == "deleted" + finally: + if process.poll() is None: + process.terminate() + process.wait(timeout=5) + + def _status(url, *, method="GET", headers=None): try: with urlopen(Request(url, method=method, headers=headers or {}), timeout=5) as response: From 7e75be648f8ab8f2cfa776e941894459d224f7c9 Mon Sep 17 00:00:00 2001 From: Philip Z Date: Wed, 28 Oct 2026 05:03:53 +0800 Subject: [PATCH 37/75] Snapshot trusted Blob retention and collect completed server-stream objects --- packages/teaql-blob-reference/README.md | 6 +- .../src/teaql_blob_reference/__init__.py | 3 +- .../maintenance_service.py | 66 ++++++-- .../src/teaql_blob_reference/retention.py | 78 +++++++++ .../teaql_blob_reference/sqlite_catalog.py | 105 +++++++++++- .../teaql_blob_reference/upload_service.py | 46 ++++- .../sqlite/test_blob_catalog_graph.py | 128 ++++++++++++-- .../provider/sqlite/test_blob_maintenance.py | 77 +++++++++ tests/provider/sqlite/test_blob_module.py | 19 ++- .../sqlite/test_blob_retention_snapshot.py | 146 ++++++++++++++++ tests/provider/sqlite/test_blob_upload.py | 10 +- .../sqlite/testdata/blob-policy-cases.json | 159 ++++++++++++++++++ 12 files changed, 795 insertions(+), 48 deletions(-) create mode 100644 packages/teaql-blob-reference/src/teaql_blob_reference/retention.py create mode 100644 tests/provider/sqlite/test_blob_retention_snapshot.py create mode 100644 tests/provider/sqlite/testdata/blob-policy-cases.json diff --git a/packages/teaql-blob-reference/README.md b/packages/teaql-blob-reference/README.md index c42fe500..bb935b28 100644 --- a/packages/teaql-blob-reference/README.md +++ b/packages/teaql-blob-reference/README.md @@ -2,9 +2,11 @@ Optional Blob catalog and current-read services for TeaQL Python. Install this wheel alongside the matching `teaql` version. Catalog migration and runtime service installation are explicit host actions; this package does not install schema or expose routes on import. -`SqliteBlobUploadService` offers a host-invoked, scope-bound upload journal. The host must provide current model/action authorization and a trusted parser that consumes every stored byte and returns actual format, size, and SHA256. The service writes through the context's object-storage provider, rechecks the stored generation before confirmation, and promotes the row to `uploaded_unbound` only after full verification. The included PDF verifier in tests is a fixture, not a production format parser. Full retention/GC, other SQL backends, and production ten-format parsing remain open. +`SqliteBlobUploadService` offers a host-invoked, scope-bound upload journal. The host must provide current model/action authorization and a trusted parser that consumes every stored byte and returns actual format, size, and SHA256. The service writes through the context's object-storage provider, rechecks the stored generation before confirmation, and promotes the row to `uploaded_unbound` only after full verification. The included PDF verifier in tests is a fixture, not a production format parser. Other SQL backends and production ten-format parsing remain open. -`SqliteBlobMaintenanceService` is an optional, explicitly host-scheduled service for **expired, uncompleted, unbound server-stream uploads only**. The host supplies a current scope and async `can_maintain` policy, then calls `run_batch(context, cursor)` and may install the service through `BlobReferenceModule(..., maintenance=service)`. Run `catalog.ensure_schema()` as a separate migration before readiness; it adds a unique object-key index. Cleanup claims rows under SQLite `BEGIN IMMEDIATE`, pins the observed ETag, uses only version-conditional storage deletion, and can resume after a failed final catalog acknowledgment. `FileObjectStorage` now guards manifest publication and conditional deletion with a persistent per-key OS file lock. Its ETag includes an immutable generation token, so identical-byte replacement cannot match an older version. A matching delete removes the current data generation and manifest; older immutable generations remain for a separate collector. Keep each Blob object key unique and immutable, including across manual S3 writes. Completed unbound and old bound retention and direct-upload quiescence remain outside this service. +For retention-aware uploads, the host may implement `current_policy_on_lease(actor, lease, target)` on its upload authorization and return an immutable `BlobRetentionPolicy`. The service obtains this trusted policy on the physical SQLite begin lease and snapshots its profile, upload mode and retention durations into the catalog. It rejects direct mode on this server-stream path. Confirmation records `completed_at_ms`; replay never changes the accepted policy or completion time. Older rows remain NULL-policy after migration and are excluded from retention collection. A binding authorization must provide its own current policy on the graph lease; new bindings of unknown-policy rows fail closed. Successful binding preserves the longest accepted unreferenced duration and minimum retention deadline. The final unbind records `last_unbound_at_ms` while the row remains `available`, so its lifetime is measured from the actual final unbind. The shared policy vectors are pinned by SHA-256 in the tests. + +`SqliteBlobMaintenanceService` is an optional, explicitly host-scheduled service for expired incomplete uploads and policy-backed completed, unbound server-stream objects. It includes formerly bound objects only after the final unbind deadline and retention floor. Active bindings, unknown-policy legacy rows and direct uploads are excluded. The host supplies a current scope and async `can_maintain` policy, then calls `run_batch(context, cursor)` and may install the service through `BlobReferenceModule(..., maintenance=service)`. Run `catalog.ensure_schema()` as a separate migration before readiness; it adds a unique object-key index and nullable policy/time columns. Cleanup claims rows under SQLite `BEGIN IMMEDIATE`, pins the observed ETag, uses only version-conditional storage deletion, and can resume after a failed final catalog acknowledgment. `FileObjectStorage` guards manifest publication and conditional deletion with a persistent per-key OS file lock. Its ETag includes an immutable generation token, so identical-byte replacement cannot match an older version. A matching delete removes the current data generation and manifest; older immutable generations remain for a separate collector. Keep each Blob object key unique and immutable, including across manual S3 writes. Direct-upload quiescence, older File generations and cross-machine S3 retention remain outside this service. For actual JPEG/PNG parsing, install `teaql-blob-reference[image]` and inject `teaql_blob_reference.content_pillow.PillowBlobContentVerifier`. The image extra diff --git a/packages/teaql-blob-reference/src/teaql_blob_reference/__init__.py b/packages/teaql-blob-reference/src/teaql_blob_reference/__init__.py index 796a9f7e..59e20059 100644 --- a/packages/teaql-blob-reference/src/teaql_blob_reference/__init__.py +++ b/packages/teaql-blob-reference/src/teaql_blob_reference/__init__.py @@ -8,6 +8,7 @@ SqliteBlobMaintenanceService) from .mutation_participant import SqliteBlobMutationParticipant from .file_storage import FileObjectStorage +from .retention import BlobRetentionPolicy from .upload_service import (BlobUploadInput, BlobUploadSession, BlobUploadTarget, BlobVerifiedContent, SqliteBlobUploadService) @@ -16,4 +17,4 @@ "BlobMaintenanceOutcome", "SqliteBlobMaintenanceService", "SqliteBlobCatalog", "SqliteBlobResponseContributor", "FileObjectStorage", "BlobUploadInput", "BlobUploadSession", "BlobUploadTarget", - "BlobVerifiedContent", "SqliteBlobUploadService"] + "BlobVerifiedContent", "SqliteBlobUploadService", "BlobRetentionPolicy"] diff --git a/packages/teaql-blob-reference/src/teaql_blob_reference/maintenance_service.py b/packages/teaql-blob-reference/src/teaql_blob_reference/maintenance_service.py index da7fe24b..79b4b0be 100644 --- a/packages/teaql-blob-reference/src/teaql_blob_reference/maintenance_service.py +++ b/packages/teaql-blob-reference/src/teaql_blob_reference/maintenance_service.py @@ -15,6 +15,8 @@ from teaql.sql.types import CompiledQuery from .sqlite_catalog import SqliteBlobCatalog, _scope +from .retention import (retention_deadline, unreferenced_eligible_at, + valid_seconds, valid_time) class BlobMaintenanceAuthorization(Protocol): @@ -85,7 +87,7 @@ async def _authorize(self, actor, expected: str | None = None) -> str: def _now(self) -> int: value = self.now_ms() - if type(value) is not int or value < 0: + if not valid_time(value): raise RuntimeError("BLOB_POLICY_INVALID") return value @@ -111,7 +113,10 @@ async def _transaction(self, work): async def _row(self, sql, scope: str, key: str): try: rows = await sql.fetch_all_sql(_query( - "SELECT c.object_key,c.content_version,c.status,u.state,u.expires_ms,u.written_etag " + "SELECT c.object_key,c.content_version,c.status,c.completed_at_ms," + "c.last_unbound_at_ms,c.policy_profile,c.upload_mode," + "c.unbound_retention_seconds,c.effective_unref_seconds," + "c.retention_floor_until_ms,u.state,u.expires_ms,u.written_etag " "FROM teaql_blob_catalog c JOIN teaql_blob_upload u " "ON u.key=c.key AND u.scope=c.scope WHERE c.scope=? AND c.key=?", Value.Text(scope), Value.Text(key))) @@ -135,13 +140,41 @@ async def _row(self, sql, scope: str, key: str): raise RuntimeError("BLOB_CATALOG_FAILED") return {"key": key, "object_key": object_key, "version": version, "status": row["status"], "state": row["state"], - "expires_ms": row["expires_ms"], "written_etag": written} + "expires_ms": row["expires_ms"], "written_etag": written, + "completed_at_ms": row["completed_at_ms"], + "last_unbound_at_ms": row["last_unbound_at_ms"], + "policy_profile": row["policy_profile"], "upload_mode": row["upload_mode"], + "unbound_retention_seconds": row["unbound_retention_seconds"], + "effective_unref_seconds": row["effective_unref_seconds"], + "retention_floor_until_ms": row["retention_floor_until_ms"]} @staticmethod def _eligible(row, now): - return (row is not None and row["expires_ms"] <= now - and row["state"] in ("initiated", "written") - and row["status"] in ("initiated", "deleting")) + if row is None: + return False + if row["state"] in ("initiated", "written"): + return (row["expires_ms"] <= now and row["status"] in ("initiated", "deleting")) + if row["state"] != "completed" or row["status"] not in ( + "uploaded_unbound", "available", "deleting"): + return False + if (not _scope(row["policy_profile"]) or row["upload_mode"] != "server_stream" + or not valid_time(row["completed_at_ms"]) + or not valid_seconds(row["unbound_retention_seconds"]) + or not valid_seconds(row["effective_unref_seconds"]) + or not _version(row["version"])): + return False + last = row["last_unbound_at_ms"] + floor = row["retention_floor_until_ms"] + if last is None: + if row["status"] == "available": + return False + deadline = retention_deadline(row["completed_at_ms"], + row["unbound_retention_seconds"]) + else: + if not valid_time(last) or floor is not None and not valid_time(floor): + return False + deadline = unreferenced_eligible_at(last, row["effective_unref_seconds"], floor) + return now >= deadline async def _count(self, sql, statement, key): try: @@ -163,12 +196,12 @@ async def work(sql): row["object_key"]) if owners != 1: raise RuntimeError("BLOB_CATALOG_FAILED") - if row["status"] == "initiated": + if row["status"] != "deleting": try: affected, _ = await sql.execute_sql(_query( "UPDATE teaql_blob_catalog SET status='deleting' " - "WHERE key=? AND scope=? AND status='initiated'", - Value.Text(key), Value.Text(scope))) + "WHERE key=? AND scope=? AND status=?", + Value.Text(key), Value.Text(scope), Value.Text(row["status"]))) except Exception: raise RuntimeError("BLOB_CATALOG_FAILED") from None if affected != 1: @@ -290,11 +323,20 @@ async def run_batch(self, actor, cursor: BlobMaintenanceCursor | None = None) -> rows = await self.catalog.transport.fetch_all_sql(_query( "SELECT DISTINCT c.key FROM teaql_blob_catalog c JOIN teaql_blob_upload u " "ON u.key=c.key AND u.scope=c.scope WHERE c.scope=? AND c.key>? " - "AND u.expires_ms<=? AND u.state IN ('initiated','written') " - "AND c.status IN ('initiated','deleting') " + "AND ((u.expires_ms<=? AND u.state IN ('initiated','written') " + "AND c.status IN ('initiated','deleting')) OR " + "(u.state='completed' AND c.upload_mode='server_stream' " + "AND c.policy_profile IS NOT NULL AND c.completed_at_ms IS NOT NULL " + "AND c.content_version<>'' AND (c.status='deleting' OR " + "(c.status='uploaded_unbound' AND c.last_unbound_at_ms IS NULL " + "AND c.completed_at_ms+c.unbound_retention_seconds*1000<=?) OR " + "(c.status='available' AND c.last_unbound_at_ms IS NOT NULL " + "AND c.last_unbound_at_ms+c.effective_unref_seconds*1000<=? " + "AND (c.retention_floor_until_ms IS NULL OR c.retention_floor_until_ms<=?))))) " "AND NOT EXISTS (SELECT 1 FROM teaql_blob_binding b WHERE b.key=c.key) " "ORDER BY c.key LIMIT ?", Value.Text(scope), Value.Text(after), - Value.I64(now), Value.I64(self.batch_size))) + Value.I64(now), Value.I64(now), Value.I64(now), Value.I64(now), + Value.I64(self.batch_size))) except Exception: raise RuntimeError("BLOB_CATALOG_FAILED") from None keys = [] diff --git a/packages/teaql-blob-reference/src/teaql_blob_reference/retention.py b/packages/teaql-blob-reference/src/teaql_blob_reference/retention.py new file mode 100644 index 00000000..b1e7a577 --- /dev/null +++ b/packages/teaql-blob-reference/src/teaql_blob_reference/retention.py @@ -0,0 +1,78 @@ +"""Validated, immutable host policy and time rules for Blob retention.""" + +from dataclasses import dataclass + + +MAX_RETENTION_EPOCH_MS = 8_640_000_000_000_000 + + +def valid_time(value: object) -> bool: + return type(value) is int and 0 <= value <= MAX_RETENTION_EPOCH_MS + + +def valid_seconds(value: object) -> bool: + return type(value) is int and 0 <= value <= MAX_RETENTION_EPOCH_MS // 1000 + + +def retention_deadline(base_ms: int, seconds: int) -> int: + if not valid_time(base_ms): + raise RuntimeError("BLOB_REFERENCE_INVALID") + if not valid_seconds(seconds) or seconds > (MAX_RETENTION_EPOCH_MS - base_ms) // 1000: + raise RuntimeError("BLOB_POLICY_INVALID") + return base_ms + seconds * 1000 + + +def effective_retention(accepted: tuple[int, ...], proposed: int) -> int: + if not valid_seconds(proposed) or any(not valid_seconds(value) for value in accepted): + raise RuntimeError("BLOB_POLICY_INVALID") + return max((proposed, *accepted)) + + +def unreferenced_eligible_at(last_unbound_ms: int, seconds: int, + floor_until_ms: int | None) -> int: + deadline = retention_deadline(last_unbound_ms, seconds) + if floor_until_ms is not None: + if not valid_time(floor_until_ms): + raise RuntimeError("BLOB_REFERENCE_INVALID") + deadline = max(deadline, floor_until_ms) + return deadline + + +def require_bindable(status: str, now_ms: int, completed_ms: int | None, + unbound_seconds: int, unreferenced_eligible_ms: int | None) -> None: + if not valid_time(now_ms): + raise RuntimeError("BLOB_REFERENCE_INVALID") + if status == "initiated": + raise RuntimeError("BLOB_UPLOAD_INCOMPLETE") + if status in ("deleting", "deleted"): + raise RuntimeError("BLOB_REFERENCE_EXPIRED") + if status == "uploaded_unbound": + if completed_ms is None: + raise RuntimeError("BLOB_REFERENCE_INVALID") + if now_ms >= retention_deadline(completed_ms, unbound_seconds): + raise RuntimeError("BLOB_REFERENCE_EXPIRED") + elif status == "available": + if unreferenced_eligible_ms is not None: + if not valid_time(unreferenced_eligible_ms): + raise RuntimeError("BLOB_REFERENCE_INVALID") + if now_ms >= unreferenced_eligible_ms: + raise RuntimeError("BLOB_REFERENCE_EXPIRED") + else: + raise RuntimeError("BLOB_REFERENCE_INVALID") + + +@dataclass(frozen=True) +class BlobRetentionPolicy: + profile: str + upload_mode: str + unbound_retention_seconds: int + unreferenced_retention_seconds: int + minimum_retention_seconds: int + + def __post_init__(self): + from .sqlite_catalog import _scope + if (not _scope(self.profile) or self.upload_mode not in ("server_stream", "direct") + or not all(valid_seconds(value) for value in ( + self.unbound_retention_seconds, self.unreferenced_retention_seconds, + self.minimum_retention_seconds))): + raise RuntimeError("BLOB_POLICY_INVALID") diff --git a/packages/teaql-blob-reference/src/teaql_blob_reference/sqlite_catalog.py b/packages/teaql-blob-reference/src/teaql_blob_reference/sqlite_catalog.py index c1635592..701ab5b6 100644 --- a/packages/teaql-blob-reference/src/teaql_blob_reference/sqlite_catalog.py +++ b/packages/teaql-blob-reference/src/teaql_blob_reference/sqlite_catalog.py @@ -2,12 +2,15 @@ from dataclasses import dataclass from typing import Protocol +import time import unicodedata from teaql.core.blob_ref import BlobRef from teaql.core.value import Value from teaql.sql.types import CompiledQuery from teaql.runtime.view_preparation import AuthorizedBlobSlot +from .retention import (BlobRetentionPolicy, require_bindable, retention_deadline, + unreferenced_eligible_at, valid_seconds, valid_time) @dataclass(frozen=True) @@ -43,6 +46,11 @@ async def authorize(self, context, target: BlobBindingTarget, action: str) -> bool: ... +class BlobBindingPolicyAuthorization(Protocol): + async def current_policy_on_lease(self, context, lease, + target: BlobBindingTarget) -> BlobRetentionPolicy: ... + + def _name(value): return (isinstance(value, str) and value and value == value.strip() and len(value) <= 128 and not any(unicodedata.category(ch) == "Cc" for ch in value)) @@ -67,11 +75,12 @@ def _entity_id(value): class SqliteBlobCatalog: """SQLite-only catalog; install and schema migration are separate host actions.""" - def __init__(self, transport): + def __init__(self, transport, now_ms=None): if (transport is None or not callable(getattr(transport, "execute_sql", None)) or not callable(getattr(transport, "fetch_all_sql", None))): raise RuntimeError("BLOB_CAPABILITY_MISSING") self.transport = transport + self.now_ms = now_ms or (lambda: time.time_ns() // 1_000_000) async def ensure_schema(self): statements = ( @@ -79,6 +88,9 @@ async def ensure_schema(self): key TEXT PRIMARY KEY, scope TEXT NOT NULL, object_key TEXT NOT NULL, content_version TEXT NOT NULL, file_name TEXT NOT NULL, content_type TEXT NOT NULL, size_bytes BIGINT NOT NULL, + completed_at_ms BIGINT, last_unbound_at_ms BIGINT, + policy_profile TEXT, upload_mode TEXT, unbound_retention_seconds BIGINT, + effective_unref_seconds BIGINT, retention_floor_until_ms BIGINT, status TEXT NOT NULL CHECK (status IN ('initiated','uploaded_unbound','available','deleting','deleted'))) """, @@ -106,10 +118,37 @@ async def ensure_schema(self): await self.transport.execute_sql(CompiledQuery(sql, [])) except Exception: raise RuntimeError("BLOB_CATALOG_FAILED") from None + # Existing catalogs have no trusted policy provenance. Keep the added + # columns NULL: a migration cannot infer a host authorization decision. + columns = ( + ("completed_at_ms", "BIGINT"), ("last_unbound_at_ms", "BIGINT"), + ("policy_profile", "TEXT"), ("upload_mode", "TEXT"), + ("unbound_retention_seconds", "BIGINT"), + ("effective_unref_seconds", "BIGINT"), + ("retention_floor_until_ms", "BIGINT"), + ) + for name, kind in columns: + try: + rows = await self.transport.fetch_all_sql(CompiledQuery( + "PRAGMA table_info(teaql_blob_catalog)", [])) + if name not in {row["name"] for row in rows}: + try: + await self.transport.execute_sql(CompiledQuery( + f"ALTER TABLE teaql_blob_catalog ADD COLUMN {name} {kind}", [])) + except Exception: + # A concurrent migrator may have added the column. + rows = await self.transport.fetch_all_sql(CompiledQuery( + "PRAGMA table_info(teaql_blob_catalog)", [])) + if name not in {row["name"] for row in rows}: + raise + except Exception: + raise RuntimeError("BLOB_CATALOG_FAILED") from None async def validate_schema(self): for sql in ( - "SELECT key, scope, object_key, content_version, file_name, content_type, size_bytes, status " + "SELECT key, scope, object_key, content_version, file_name, content_type, size_bytes, " + "completed_at_ms, last_unbound_at_ms, policy_profile, upload_mode, " + "unbound_retention_seconds, effective_unref_seconds, retention_floor_until_ms, status " "FROM teaql_blob_catalog WHERE 1 = 0", "SELECT scope, entity, entity_id, field, key, version " "FROM teaql_blob_binding WHERE 1 = 0", @@ -222,13 +261,50 @@ async def bind(sql): if change.next: try: available = await sql.fetch_all_sql(CompiledQuery( - "SELECT key FROM teaql_blob_catalog WHERE key = ? AND scope = ? " + "SELECT key,status,completed_at_ms,last_unbound_at_ms,policy_profile," + "upload_mode,unbound_retention_seconds,effective_unref_seconds," + "retention_floor_until_ms FROM teaql_blob_catalog WHERE key = ? AND scope = ? " "AND status IN ('uploaded_unbound','available')", [Value.Text(change.next.key), Value.Text(scope)])) except Exception: raise RuntimeError("BLOB_CATALOG_FAILED") from None if len(available) != 1: raise RuntimeError("BLOB_BIND_DENIED") + blob = available[0] + current = getattr(authorization, "current_policy_on_lease", None) + if (not callable(current) or not _scope(blob.get("policy_profile")) + or blob.get("upload_mode") != "server_stream" + or not valid_time(blob.get("completed_at_ms")) + or not valid_seconds(blob.get("unbound_retention_seconds")) + or not valid_seconds(blob.get("effective_unref_seconds"))): + raise RuntimeError("BLOB_BIND_DENIED") + try: + policy = await current(actor, sql, target) + except Exception: + raise RuntimeError("BLOB_BIND_DENIED") from None + if not isinstance(policy, BlobRetentionPolicy) or policy.upload_mode != "server_stream": + raise RuntimeError("BLOB_BIND_DENIED") + now = self.now_ms() + if not valid_time(now): + raise RuntimeError("BLOB_POLICY_INVALID") + eligible_at = None + if blob["status"] == "available": + try: + bound = await sql.fetch_all_sql(CompiledQuery( + "SELECT 1 FROM teaql_blob_binding WHERE key = ? LIMIT 1", + [Value.Text(change.next.key)])) + except Exception: + raise RuntimeError("BLOB_CATALOG_FAILED") from None + if not bound: + last_unbound = blob.get("last_unbound_at_ms") + floor = blob.get("retention_floor_until_ms") + if not valid_time(last_unbound) or floor is not None and not valid_time(floor): + raise RuntimeError("BLOB_BIND_DENIED") + eligible_at = unreferenced_eligible_at( + last_unbound, blob["effective_unref_seconds"], floor) + require_bindable(blob["status"], now, blob["completed_at_ms"], + blob["unbound_retention_seconds"], eligible_at) + accepted_floor = retention_deadline(now, policy.minimum_retention_seconds) key = Value.Text(change.next.key) if change.next else Value.Null() try: if row: @@ -248,20 +324,37 @@ async def bind(sql): if change.next: try: promoted, _ = await sql.execute_sql(CompiledQuery( - "UPDATE teaql_blob_catalog SET status = 'available' WHERE key = ? AND scope = ? " + "UPDATE teaql_blob_catalog SET status = 'available', last_unbound_at_ms = NULL " + "WHERE key = ? AND scope = ? " "AND status IN ('uploaded_unbound','available')", [Value.Text(change.next.key), Value.Text(scope)])) except Exception: raise RuntimeError("BLOB_CATALOG_FAILED") from None if promoted != 1: raise RuntimeError("BLOB_BIND_DENIED") + try: + retained, _ = await sql.execute_sql(CompiledQuery( + "UPDATE teaql_blob_catalog SET " + "effective_unref_seconds=MAX(effective_unref_seconds,?), " + "retention_floor_until_ms=CASE WHEN retention_floor_until_ms IS NULL " + "OR retention_floor_until_ms bool: ... +class BlobUploadPolicyAuthorization(Protocol): + """Optional trusted policy lookup on the physical begin-upload SQL lease.""" + + async def current_policy_on_lease(self, context, lease: SqlTransactionTransport, + target: BlobUploadTarget) -> BlobRetentionPolicy: ... + + class BlobContentVerifier(Protocol): def verify(self, stream: BinaryIO, extension: str, maximum_bytes: int) -> BlobVerifiedContent: ... @@ -194,6 +202,22 @@ async def _transaction(self, work): with suppress(Exception): await tx.rollback_sql() + async def _policy_on_lease(self, actor, tx, target): + current = getattr(self.authorization, "current_policy_on_lease", None) + if current is None: + return None + if not callable(current): + raise RuntimeError("BLOB_POLICY_INVALID") + try: + policy = await current(actor, tx, target) + except Exception: + raise RuntimeError("BLOB_POLICY_INVALID") from None + if not isinstance(policy, BlobRetentionPolicy): + raise RuntimeError("BLOB_POLICY_INVALID") + if policy.upload_mode != "server_stream": + raise RuntimeError("BLOB_CAPABILITY_MISSING") + return policy + async def begin(self, actor, target: BlobUploadTarget, input: BlobUploadInput, replay_key: str) -> BlobUploadSession: normalized, extension = self._normalize(input) @@ -234,13 +258,20 @@ async def work(tx): object_key = "blob/" + _token() expires_ms = self.now_ms() + self.lease_seconds * 1000 await self._current(actor, target, "Begin", normalized, scope) + policy = await self._policy_on_lease(actor, tx, target) + profile = Value.Text(policy.profile) if policy else Value.Null() + mode = Value.Text(policy.upload_mode) if policy else Value.Null() + unbound = Value.I64(policy.unbound_retention_seconds) if policy else Value.Null() + unreferenced = Value.I64(policy.unreferenced_retention_seconds) if policy else Value.Null() try: await tx.execute_sql(_query( "INSERT INTO teaql_blob_catalog " - "(key,scope,object_key,content_version,file_name,content_type,size_bytes,status) " - "VALUES (?,?,?,?,?,?,?,?)", Value.Text(reference.key), Value.Text(scope), + "(key,scope,object_key,content_version,file_name,content_type,size_bytes,status," + "policy_profile,upload_mode,unbound_retention_seconds,effective_unref_seconds) " + "VALUES (?,?,?,?,?,?,?,?,?,?,?,?)", Value.Text(reference.key), Value.Text(scope), Value.Text(object_key), Value.Text(""), Value.Text(normalized.file_name), - Value.Text(""), Value.I64(0), Value.Text("initiated"))) + Value.Text(""), Value.I64(0), Value.Text("initiated"), + profile, mode, unbound, unreferenced)) await tx.execute_sql(_query( "INSERT INTO teaql_blob_upload " "(upload_id,scope,replay_key,request_hash,key,entity,entity_id,field,file_name," @@ -593,14 +624,14 @@ async def work(tx): BlobUploadInput(row["file_name"], row["declared_size"]), row["scope"]) now = self.now_ms() - if now >= row["expires_ms"]: + if not valid_time(now) or now >= row["expires_ms"]: raise RuntimeError("BLOB_UPLOAD_EXPIRED") verified = await asyncio.to_thread(verify_stored) await self._current(actor, target, "Confirm", BlobUploadInput(row["file_name"], row["declared_size"], verified.content_type), row["scope"]) now = self.now_ms() - if now >= row["expires_ms"]: + if not valid_time(now) or now >= row["expires_ms"]: raise RuntimeError("BLOB_UPLOAD_EXPIRED") try: affected, _ = await tx.execute_sql(_query( @@ -612,9 +643,10 @@ async def work(tx): raise RuntimeError("BLOB_IDEMPOTENCY_CONFLICT") affected, _ = await tx.execute_sql(_query( "UPDATE teaql_blob_catalog SET content_version=?,content_type=?,size_bytes=?, " - "status='uploaded_unbound' WHERE key=? AND scope=? AND status='initiated'", + "completed_at_ms=?,status='uploaded_unbound' " + "WHERE key=? AND scope=? AND status='initiated'", Value.Text(row["written_etag"]), Value.Text(verified.content_type), - Value.I64(verified.size_bytes), Value.Text(row["key"]), + Value.I64(verified.size_bytes), Value.I64(now), Value.Text(row["key"]), Value.Text(row["scope"]))) if affected != 1: raise RuntimeError("BLOB_IDEMPOTENCY_CONFLICT") diff --git a/tests/provider/sqlite/test_blob_catalog_graph.py b/tests/provider/sqlite/test_blob_catalog_graph.py index 6224a660..f5603fc3 100644 --- a/tests/provider/sqlite/test_blob_catalog_graph.py +++ b/tests/provider/sqlite/test_blob_catalog_graph.py @@ -22,7 +22,7 @@ ) from teaql.sql.executor import SqlDataServiceExecutor from teaql_blob_reference import ( - BlobBindingChange, BlobBindingTarget, FileObjectStorage, SqliteBlobCatalog, + BlobBindingChange, BlobBindingTarget, BlobRetentionPolicy, FileObjectStorage, SqliteBlobCatalog, SqliteBlobResponseContributor, ) @@ -38,6 +38,10 @@ def scope(self, _context): async def authorize(self, _context, _target, _previous, _next, _action): return self.allowed + async def current_policy_on_lease(self, _context, lease, _target): + assert lease is not None + return BlobRetentionPolicy("default", "server_stream", 86400, 604800, 3600) + async def fixture(tmp_path): path = str(tmp_path / "blob-catalog.db") @@ -53,7 +57,8 @@ async def fixture(tmp_path): async with aiosqlite.connect(path) as db: await db.execute("CREATE TABLE asset (id INTEGER PRIMARY KEY, version INTEGER, name TEXT)") await db.commit() - return context, path, SqliteBlobCatalog(SqliteTransport(path)) + return context, path, SqliteBlobCatalog(SqliteTransport(path), + now_ms=lambda: 1_700_000_000_000) async def all_rows(path, table): @@ -72,10 +77,12 @@ async def test_catalog_binding_same_graph_rollback_authority_tombstone_and_rebin reference = BlobRef("blob_python_catalog_1") async with aiosqlite.connect(path) as db: await db.execute("INSERT INTO teaql_blob_catalog " - "(key,scope,object_key,content_version,file_name,content_type,size_bytes,status) " - "VALUES (?,?,?,?,?,?,?,?)", + "(key,scope,object_key,content_version,file_name,content_type,size_bytes,status," + "completed_at_ms,policy_profile,upload_mode,unbound_retention_seconds,effective_unref_seconds) " + "VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?)", (reference.key, "tenant-a", "objects/1", "v1", "asset.pdf", - "application/pdf", 7, "uploaded_unbound")) + "application/pdf", 7, "uploaded_unbound", 1_700_000_000_000, + "default", "server_stream", 86400, 604800)) await db.commit() auth = Authorization() @@ -130,7 +137,7 @@ async def save(graph, asset_id, *, abort=False, catch_denial=False, foreign_acto await context.execute_graph_save( lambda graph: catalog.bind_in_graph(graph, graph.context, clear, auth), comment="clear binding") assert (await all_rows(path, "teaql_blob_binding"))[0][-2:] == (None, 2) - assert (await all_rows(path, "teaql_blob_catalog"))[0][-1] == "uploaded_unbound" + assert (await all_rows(path, "teaql_blob_catalog"))[0][-1] == "available" rebind = BlobBindingChange(BlobBindingTarget("Asset", "1", "document"), reference, 2, 3, "Update") await context.execute_graph_save( lambda graph: catalog.bind_in_graph(graph, graph.context, rebind, auth), comment="rebind") @@ -144,7 +151,7 @@ async def save(graph, asset_id, *, abort=False, catch_denial=False, foreign_acto clear = BlobBindingChange(BlobBindingTarget("Asset", "2", "document"), None, 1, 2, "Update") await context.execute_graph_save( lambda graph: catalog.bind_in_graph(graph, graph.context, clear, auth), comment="clear final binding") - assert (await all_rows(path, "teaql_blob_catalog"))[0][-1] == "uploaded_unbound" + assert (await all_rows(path, "teaql_blob_catalog"))[0][-1] == "available" @pytest.mark.asyncio @@ -161,16 +168,109 @@ async def test_catalog_rejects_old_not_null_binding_schema(tmp_path): @pytest.mark.asyncio -async def test_current_sqlite_binding_is_reread_before_view_url_signing(tmp_path): +async def test_binding_enforces_retention_deadline_and_longest_accepted_floor(tmp_path): context, path, catalog = await fixture(tmp_path) await catalog.ensure_schema() - reference = BlobRef("blob_python_read_1") + clock = [1_700_000_000_000] + catalog.now_ms = lambda: clock[0] + expired, live = BlobRef("blob_expired_policy"), BlobRef("blob_live_policy") + async with aiosqlite.connect(path) as db: + for reference, completed in ((expired, clock[0] - 86_400_000), + (live, clock[0] - 1000)): + await db.execute("INSERT INTO teaql_blob_catalog " + "(key,scope,object_key,content_version,file_name,content_type,size_bytes,status," + "completed_at_ms,policy_profile,upload_mode,unbound_retention_seconds,effective_unref_seconds) " + "VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?)", + (reference.key, "tenant-a", "objects/" + reference.key, "v1", "report.pdf", + "application/pdf", 7, "uploaded_unbound", completed, + "default", "server_stream", 86400, 3600)) + await db.commit() + + class ChangingAuthorization(Authorization): + def __init__(self): + super().__init__() + self.policy = BlobRetentionPolicy("default", "server_stream", 86400, 604800, 3600) + + async def current_policy_on_lease(self, _context, lease, _target): + assert lease is not None + return self.policy + + authority = ChangingAuthorization() + first = BlobBindingTarget("Asset", "1", "document") + second = BlobBindingTarget("Asset", "2", "document") + + async def change(target, next_ref, before, after, action): + return await context.execute_graph_save( + lambda graph: catalog.bind_in_graph(graph, graph.context, + BlobBindingChange(target, next_ref, before, after, action), authority), + comment="retention binding") + + with pytest.raises(RuntimeError, match="BLOB_REFERENCE_EXPIRED"): + await change(first, expired, 0, 1, "Create") + await change(first, live, 0, 1, "Create") + authority.policy = BlobRetentionPolicy("longer", "server_stream", 86400, 2592000, 2678400) + await change(second, live, 0, 1, "Create") + await change(first, None, 1, 2, "Update") + async with aiosqlite.connect(path) as db: + async with db.execute("SELECT effective_unref_seconds,retention_floor_until_ms," + "last_unbound_at_ms FROM teaql_blob_catalog WHERE key=?", + (live.key,)) as cursor: + seconds, floor, last_unbound = await cursor.fetchone() + assert seconds == 2592000 and floor == clock[0] + 2_678_400_000 + assert last_unbound is None + await change(second, None, 1, 2, "Update") + async with aiosqlite.connect(path) as db: + async with db.execute("SELECT status,last_unbound_at_ms FROM teaql_blob_catalog WHERE key=?", + (live.key,)) as cursor: + assert await cursor.fetchone() == ("available", clock[0]) + authority.policy = BlobRetentionPolicy("shorter", "server_stream", 86400, 3600, 0) + clock[0] = floor - 1 + await change(first, live, 2, 3, "Update") + await change(first, None, 3, 4, "Update") + clock[0] = floor + # The accepted maximum still protects this object beyond the old floor. + await change(first, live, 4, 5, "Update") + await change(first, None, 5, 6, "Update") + clock[0] += 2_592_000_000 + with pytest.raises(RuntimeError, match="BLOB_REFERENCE_EXPIRED"): + await change(first, live, 6, 7, "Update") + + +@pytest.mark.asyncio +async def test_new_binding_rejects_legacy_unknown_policy_row(tmp_path): + context, path, catalog = await fixture(tmp_path) + await catalog.ensure_schema() + reference = BlobRef("blob_legacy_unknown") async with aiosqlite.connect(path) as db: await db.execute("INSERT INTO teaql_blob_catalog " "(key,scope,object_key,content_version,file_name,content_type,size_bytes,status) " "VALUES (?,?,?,?,?,?,?,?)", + (reference.key, "tenant-a", "objects/legacy", "v1", "legacy.pdf", + "application/pdf", 5, "uploaded_unbound")) + await db.commit() + with pytest.raises(RuntimeError, match="^BLOB_BIND_DENIED$"): + await context.execute_graph_save( + lambda graph: catalog.bind_in_graph(graph, graph.context, + BlobBindingChange(BlobBindingTarget("Asset", "1", "document"), + reference, 0, 1, "Create"), Authorization()), + comment="deny legacy unknown policy") + assert await all_rows(path, "teaql_blob_binding") == [] + + +@pytest.mark.asyncio +async def test_current_sqlite_binding_is_reread_before_view_url_signing(tmp_path): + context, path, catalog = await fixture(tmp_path) + await catalog.ensure_schema() + reference = BlobRef("blob_python_read_1") + async with aiosqlite.connect(path) as db: + await db.execute("INSERT INTO teaql_blob_catalog " + "(key,scope,object_key,content_version,file_name,content_type,size_bytes,status," + "completed_at_ms,last_unbound_at_ms,policy_profile,upload_mode," + "unbound_retention_seconds,effective_unref_seconds) " + "VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?)", (reference.key, "tenant-a", "objects/read-1", "v1", "report.pdf", - "application/pdf", 7, "available")) + "application/pdf", 7, "available", 1_700_000_000_000, + 1_700_000_000_000, "default", "server_stream", 86400, 604800)) await db.commit() auth = Authorization() change = BlobBindingChange(BlobBindingTarget("Asset", "1", "document"), reference, 0, 1, "Create") @@ -253,10 +353,12 @@ async def test_authorized_sqlite_view_projects_real_file_http_bytes_and_denies_r reference = BlobRef("blob_python_real_1") async with aiosqlite.connect(path) as db: await db.execute("INSERT INTO teaql_blob_catalog " - "(key,scope,object_key,content_version,file_name,content_type,size_bytes,status) " - "VALUES (?,?,?,?,?,?,?,?)", + "(key,scope,object_key,content_version,file_name,content_type,size_bytes,status," + "completed_at_ms,policy_profile,upload_mode,unbound_retention_seconds,effective_unref_seconds) " + "VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?)", (reference.key, "tenant-a", key.value, "v1", "report.pdf", - "application/pdf", len(data), "uploaded_unbound")) + "application/pdf", len(data), "uploaded_unbound", 1_700_000_000_000, + "default", "server_stream", 86400, 604800)) await db.commit() change = BlobBindingChange(BlobBindingTarget("Asset", "1", "document"), reference, 0, 1, "Create") diff --git a/tests/provider/sqlite/test_blob_maintenance.py b/tests/provider/sqlite/test_blob_maintenance.py index dbfe2430..d29cfa7d 100644 --- a/tests/provider/sqlite/test_blob_maintenance.py +++ b/tests/provider/sqlite/test_blob_maintenance.py @@ -122,6 +122,83 @@ async def test_real_sqlite_file_maintenance_recovers_receipt_and_partial_data(tm assert await _status(path, "blob_file_partial") == "deleted" +@pytest.mark.asyncio +async def test_real_file_completed_retention_claim_and_generation_fences(tmp_path): + path = str(tmp_path / "completed-retention.db") + catalog = SqliteBlobCatalog(SqliteTransport(path)) + await catalog.ensure_schema() + provider = FileObjectStorage(tmp_path / "completed-objects", os.urandom(32)) + actor = RuntimeModule.new().into_context() + actor.with_object_storage(provider).with_object_storage_observer(lambda event: None) + store = actor.object_storage("real File completed retention") + service = SqliteBlobMaintenanceService(catalog, _MaintenanceAuthority(), 100, lambda: 10000) + content = b"%PDF-1.7\nretained\n%%EOF" + + async def seed(name, *, status="uploaded_unbound", completed=9000, + last_unbound=None, floor=None, profile="default", mode="server_stream", + bound=False): + key = ObjectKey("objects/" + name) + created = store.put_bytes(key, content, PutOptions(len(content), WriteMode.CREATE_ONLY)) + await _seed(path, name, state="completed", version=created.etag, + written_etag=created.etag, bound=bound) + async with aiosqlite.connect(path) as db: + await db.execute("UPDATE teaql_blob_catalog SET status=?,completed_at_ms=?," + "last_unbound_at_ms=?,policy_profile=?,upload_mode=?," + "unbound_retention_seconds=1,effective_unref_seconds=2," + "retention_floor_until_ms=?,content_type='application/pdf',size_bytes=? " + "WHERE key=?", (status, completed, last_unbound, profile, mode, + floor, len(content), name)) + await db.commit() + return key, created.etag + + expired, _ = await seed("blob_completed_expired") + formerly_bound, _ = await seed("blob_completed_formerly_bound", status="available", + last_unbound=7000, floor=10000) + active, _ = await seed("blob_completed_active", status="available", bound=True) + floored, _ = await seed("blob_completed_floor", status="available", + last_unbound=7000, floor=20000) + legacy, _ = await seed("blob_completed_legacy", profile=None) + direct, _ = await seed("blob_completed_direct", mode="direct") + batch = await service.run_batch(actor) + assert [(result.reference.key, result.status) for result in batch.outcomes] == [ + ("blob_completed_expired", "deleted"), + ("blob_completed_formerly_bound", "deleted")] + assert store.head(expired) is None and store.head(formerly_bound) is None + for name, key in (("blob_completed_active", active), ("blob_completed_floor", floored), + ("blob_completed_legacy", legacy), ("blob_completed_direct", direct)): + assert await _status(path, name) != "deleted" and store.head(key) is not None + + stale_key, stale_version = await seed("blob_completed_stale") + replacement = store.put_bytes(stale_key, content, PutOptions(len(content), WriteMode.REPLACE)) + assert replacement.etag != stale_version + stale = await service.run_batch(actor) + assert [(result.reference.key, result.status) for result in stale.outcomes] == [ + ("blob_completed_stale", "version_conflict")] + assert store.head(stale_key).etag == replacement.etag + assert await _status(path, "blob_completed_stale") == "deleting" + + retry_key, _ = await seed("blob_completed_retry") + async with aiosqlite.connect(path) as db: + await db.execute("""CREATE TRIGGER fail_completed_receipt BEFORE UPDATE OF status + ON teaql_blob_catalog WHEN NEW.status='deleted' AND OLD.key='blob_completed_retry' + BEGIN SELECT RAISE(ABORT,'receipt failure'); END""") + await db.commit() + failed = await service.run_batch(actor) + assert [(result.reference.key, result.status) for result in failed.outcomes] == [ + ("blob_completed_retry", "failed"), + ("blob_completed_stale", "version_conflict")] + assert await _status(path, "blob_completed_retry") == "deleting" + assert store.head(retry_key) is None + async with aiosqlite.connect(path) as db: + await db.execute("DROP TRIGGER fail_completed_receipt") + await db.commit() + recovered = await service.run_batch(actor) + assert [(result.reference.key, result.status) for result in recovered.outcomes] == [ + ("blob_completed_retry", "already_missing"), + ("blob_completed_stale", "version_conflict")] + assert await _status(path, "blob_completed_retry") == "deleted" + + @pytest.mark.asyncio async def test_expired_upload_maintenance_claim_scope_version_and_recovery(tmp_path): path = str(tmp_path / "maintenance.db") diff --git a/tests/provider/sqlite/test_blob_module.py b/tests/provider/sqlite/test_blob_module.py index e3d85100..84a2bb5a 100644 --- a/tests/provider/sqlite/test_blob_module.py +++ b/tests/provider/sqlite/test_blob_module.py @@ -1,6 +1,7 @@ """The optional Blob assembly fails closed until the owner and catalog are ready.""" import os +import time import aiosqlite import pytest @@ -16,7 +17,7 @@ from teaql.runtime import RuntimeModule, validate_capabilities from teaql.sql.executor import SqlDataServiceExecutor from teaql_blob_reference import ( - BlobReferenceModule, FileObjectStorage, SqliteBlobCatalog, + BlobReferenceModule, BlobRetentionPolicy, FileObjectStorage, SqliteBlobCatalog, SqliteBlobMaintenanceService, SqliteBlobUploadService, ) @@ -57,6 +58,10 @@ def scope(self, context): async def authorize(self, context, target, previous, next_ref, action): return self.allowed and target.entity == "Asset" and target.field == "document" + async def current_policy_on_lease(self, context, lease, target): + assert lease is not None + return BlobRetentionPolicy("default", "server_stream", 86400, 604800, 3600) + class _MaintenanceAuthority: def scope(self, context): @@ -139,10 +144,12 @@ async def test_native_mutation_binds_blob_in_same_sqlite_graph(tmp_path): await db.execute("CREATE TABLE asset (id INTEGER PRIMARY KEY, version INTEGER, document TEXT)") for key in ("blob_native_one", "blob_native_two"): await db.execute("INSERT INTO teaql_blob_catalog " - "(key,scope,object_key,content_version,file_name,content_type,size_bytes,status) " - "VALUES (?,?,?,?,?,?,?,?)", + "(key,scope,object_key,content_version,file_name,content_type,size_bytes,status," + "completed_at_ms,policy_profile,upload_mode,unbound_retention_seconds,effective_unref_seconds) " + "VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?)", (key, "tenant-a", f"objects/{key}", "v1", "report.pdf", - "application/pdf", 7, "uploaded_unbound")) + "application/pdf", 7, "uploaded_unbound", time.time_ns() // 1_000_000, + "default", "server_stream", 86400, 604800)) await db.commit() authority = _BindingAuthority() upload = SqliteBlobUploadService(catalog, _UploadAuthority(), _Verifier(), 10_000) @@ -174,7 +181,7 @@ async def work(graph): async with db.execute("SELECT key,version FROM teaql_blob_binding") as cursor: assert await cursor.fetchone() == (second.key, 2) async with db.execute("SELECT key,status FROM teaql_blob_catalog ORDER BY key") as cursor: - assert await cursor.fetchall() == [(first.key, "uploaded_unbound"), + assert await cursor.fetchall() == [(first.key, "available"), (second.key, "available")] authority.allowed = False @@ -205,7 +212,7 @@ async def work(graph): assert await cursor.fetchone() == (None, 5) async with db.execute("SELECT status FROM teaql_blob_catalog WHERE key=?", (second.key,)) as cursor: - assert await cursor.fetchone() == ("uploaded_unbound",) + assert await cursor.fetchone() == ("available",) await save(InsertCommand.new("Asset").value("id", 2), "create empty", asset_id=2) async with aiosqlite.connect(path) as db: async with db.execute("SELECT key,version FROM teaql_blob_binding WHERE entity_id='2'") as cursor: diff --git a/tests/provider/sqlite/test_blob_retention_snapshot.py b/tests/provider/sqlite/test_blob_retention_snapshot.py new file mode 100644 index 00000000..db348818 --- /dev/null +++ b/tests/provider/sqlite/test_blob_retention_snapshot.py @@ -0,0 +1,146 @@ +"""Trusted Blob retention facts on a real SQLite lease and File provider.""" + +from hashlib import sha256 +from io import BytesIO +from pathlib import Path +import json +import os + +import aiosqlite +import pytest + +from teaql.core.blob_ref import BlobFieldSpec +from teaql.provider.sqlite.transport import SqliteTransport +from teaql.runtime import RuntimeModule +from teaql.sql.types import CompiledQuery +from teaql_blob_reference import ( + BlobRetentionPolicy, BlobUploadInput, BlobUploadTarget, BlobVerifiedContent, + FileObjectStorage, SqliteBlobCatalog, SqliteBlobUploadService, +) +from teaql_blob_reference.retention import ( + effective_retention, require_bindable, retention_deadline, + unreferenced_eligible_at, +) + + +class Authority: + def __init__(self): + self.policy = BlobRetentionPolicy("default", "server_stream", 86400, 604800, 3600) + self.policy_calls = 0 + + def scope(self, _actor): + return "tenant-a" + + def field(self, _actor, _target): + return BlobFieldSpec("Asset", "document", ("pdf",), True, "default") + + async def authorize(self, _actor, _target, _action, _input): + return True + + async def current_policy_on_lease(self, _actor, lease, _target): + rows = await lease.fetch_all_sql(CompiledQuery("SELECT 1 AS alive", [])) + assert rows == [{"alive": 1}] + self.policy_calls += 1 + return self.policy + + +class Verifier: + def verify(self, stream, extension, maximum_bytes): + data = stream.read(maximum_bytes + 1) + assert extension == "pdf" and data.startswith(b"%PDF-") + assert stream.read(1) == b"" + return BlobVerifiedContent(len(data), sha256(data).hexdigest(), "application/pdf") + + +@pytest.mark.asyncio +async def test_policy_snapshotted_on_lease_completion_time_and_replay(tmp_path): + path = str(tmp_path / "catalog.db") + catalog = SqliteBlobCatalog(SqliteTransport(path)) + await catalog.ensure_schema() + await catalog.validate_schema() + authority = Authority() + clock = [1_700_000_000_000] + actor = RuntimeModule.new().into_context().with_object_storage( + FileObjectStorage(tmp_path / "private", os.urandom(32))) + actor.with_object_storage_observer(lambda _event: None) + service = SqliteBlobUploadService(catalog, authority, Verifier(), 10_000, + now_ms=lambda: clock[0]) + data = b"%PDF-1.7\nretention snapshot\n%%EOF" + target = BlobUploadTarget("Asset", "1", "document") + request = BlobUploadInput("report.pdf", len(data)) + session = await service.begin(actor, target, request, "snapshot-1") + assert authority.policy_calls == 1 + async with aiosqlite.connect(path) as db: + async with db.execute("SELECT policy_profile,upload_mode,unbound_retention_seconds," + "effective_unref_seconds,completed_at_ms,last_unbound_at_ms " + "FROM teaql_blob_catalog WHERE key=?", (session.reference.key,)) as cursor: + assert await cursor.fetchone() == ("default", "server_stream", 86400, 604800, None, None) + authority.policy = BlobRetentionPolicy("new-profile", "server_stream", 1, 1, 1) + assert (await service.begin(actor, target, request, "snapshot-1")).upload_id == session.upload_id + assert authority.policy_calls == 1 + await service.write(actor, session.upload_id, BytesIO(data)) + clock[0] += 30_000 + assert await service.complete(actor, session.upload_id) == session.reference + assert await service.complete(actor, session.upload_id) == session.reference + async with aiosqlite.connect(path) as db: + async with db.execute("SELECT policy_profile,unbound_retention_seconds," + "effective_unref_seconds,completed_at_ms,status " + "FROM teaql_blob_catalog WHERE key=?", (session.reference.key,)) as cursor: + assert await cursor.fetchone() == ("default", 86400, 604800, clock[0], "uploaded_unbound") + + +@pytest.mark.asyncio +async def test_migration_keeps_legacy_policy_unknown_and_direct_mode_rejected(tmp_path): + path = str(tmp_path / "old.db") + async with aiosqlite.connect(path) as db: + await db.execute("CREATE TABLE teaql_blob_catalog (key TEXT PRIMARY KEY, scope TEXT NOT NULL, " + "object_key TEXT NOT NULL, content_version TEXT NOT NULL, file_name TEXT NOT NULL, " + "content_type TEXT NOT NULL, size_bytes BIGINT NOT NULL, status TEXT NOT NULL)") + await db.execute("INSERT INTO teaql_blob_catalog VALUES (?,?,?,?,?,?,?,?)", + ("blob_legacy", "tenant-a", "blob/legacy", "v1", "old.pdf", "application/pdf", 10, "available")) + await db.commit() + catalog = SqliteBlobCatalog(SqliteTransport(path)) + await catalog.ensure_schema() + await catalog.ensure_schema() + await catalog.validate_schema() + async with aiosqlite.connect(path) as db: + async with db.execute("SELECT completed_at_ms,last_unbound_at_ms,policy_profile,upload_mode," + "unbound_retention_seconds,effective_unref_seconds,retention_floor_until_ms " + "FROM teaql_blob_catalog WHERE key='blob_legacy'") as cursor: + assert await cursor.fetchone() == (None,) * 7 + authority = Authority() + authority.policy = BlobRetentionPolicy("default", "direct", 1, 1, 1) + service = SqliteBlobUploadService(catalog, authority, Verifier(), 10_000) + with pytest.raises(RuntimeError, match="^BLOB_CAPABILITY_MISSING$"): + await service.begin(RuntimeModule.new().into_context(), BlobUploadTarget("Asset", "1", "document"), + BlobUploadInput("direct.pdf", 4), "direct-mode") + assert authority.policy_calls == 1 + async with aiosqlite.connect(path) as db: + async with db.execute("SELECT count(*) FROM teaql_blob_upload") as cursor: + assert await cursor.fetchone() == (0,) + + +def test_shared_retention_vectors(): + fixture = Path(__file__).with_name("testdata") / "blob-policy-cases.json" + assert sha256(fixture.read_bytes()).hexdigest() == ( + "b93cff3f8dabd5690aac5f8f2096633dcd4e158d3aa0424b9a902a16f6d3c2b6") + cases = {case["id"]: case for case in json.loads(fixture.read_text())["cases"]} + upload = cases["upload-clock-independent"] + assert upload["input"]["nowSeconds"] >= 3600 + assert retention_deadline(upload["input"]["completedAtSeconds"] * 1000, 86400) == ( + upload["expected"]["unboundEligibleAtSeconds"] * 1000) + expired = cases["unbound-deadline-rejects-bind-before-gc"] + with pytest.raises(RuntimeError, match="^BLOB_REFERENCE_EXPIRED$"): + require_bindable("uploaded_unbound", expired["input"]["nowSeconds"] * 1000, + expired["input"]["completedAtSeconds"] * 1000, + expired["input"]["unboundRetentionSeconds"], None) + maximum = cases["maximum-retention-across-fields"] + seconds = effective_retention(tuple(maximum["input"]["acceptedRetentionSeconds"]), 0) + assert seconds == maximum["expected"]["effectiveUnreferencedRetentionSeconds"] + assert unreferenced_eligible_at(maximum["input"]["lastUnboundAtSeconds"] * 1000, + seconds, maximum["input"]["retentionFloorUntilSeconds"] * 1000) == ( + maximum["expected"]["unreferencedEligibleAtSeconds"] * 1000) + shorter = cases["profile-update-does-not-shorten"] + assert effective_retention((shorter["input"]["acceptedUnreferencedRetentionSeconds"],), + shorter["input"]["currentProfileRetentionSeconds"]) == ( + shorter["expected"]["effectiveUnreferencedRetentionSeconds"]) diff --git a/tests/provider/sqlite/test_blob_upload.py b/tests/provider/sqlite/test_blob_upload.py index d28b7c44..592420bd 100644 --- a/tests/provider/sqlite/test_blob_upload.py +++ b/tests/provider/sqlite/test_blob_upload.py @@ -29,7 +29,7 @@ ) from teaql.sql.executor import SqlDataServiceExecutor from teaql_blob_reference import ( - BlobBindingChange, BlobBindingTarget, BlobUploadInput, BlobUploadTarget, + BlobBindingChange, BlobBindingTarget, BlobRetentionPolicy, BlobUploadInput, BlobUploadTarget, BlobVerifiedContent, FileObjectStorage, SqliteBlobCatalog, SqliteBlobResponseContributor, SqliteBlobUploadService, ) @@ -51,6 +51,10 @@ async def authorize(self, _context, target, action, _input): return (self.allowed and target.entity == "Asset" and target.field == "document" and action in ("Begin", "Write", "Confirm")) + async def current_policy_on_lease(self, _context, lease, _target): + assert lease is not None + return BlobRetentionPolicy("default", "server_stream", 86400, 604800, 3600) + class BindingAuthority: def scope(self, _context): @@ -59,6 +63,10 @@ def scope(self, _context): async def authorize(self, _context, target, _previous, _next, action): return target.entity == "Asset" and target.field == "document" and action == "Create" + async def current_policy_on_lease(self, _context, lease, _target): + assert lease is not None + return BlobRetentionPolicy("default", "server_stream", 86400, 604800, 3600) + class FixturePDFVerifier: def __init__(self, early=False): diff --git a/tests/provider/sqlite/testdata/blob-policy-cases.json b/tests/provider/sqlite/testdata/blob-policy-cases.json new file mode 100644 index 00000000..c8b4b794 --- /dev/null +++ b/tests/provider/sqlite/testdata/blob-policy-cases.json @@ -0,0 +1,159 @@ +{ + "schema": "teaql.blob-policy-cases/v1", + "cases": [ + { + "id": "defaults", + "input": {}, + "expected": { + "maxSizeBytes": 104857600, + "uploadSessionTtlSeconds": 3600, + "unboundRetentionSeconds": 86400, + "unreferencedRetentionSeconds": 604800, + "readUrlTtlSeconds": 600 + } + }, + { + "id": "intersections", + "input": { + "modelExtensions": [ + "pdf", + "docx", + "xlsx" + ], + "deploymentExtensions": [ + "pdf", + "docx" + ], + "profileExtensions": [ + "pdf", + "xlsx" + ] + }, + "expected": { + "allowedExtensions": [ + "pdf" + ], + "allowedContentTypes": [ + "application/pdf" + ] + } + }, + { + "id": "empty-intersection", + "input": { + "modelExtensions": [ + "pdf" + ], + "profileExtensions": [ + "docx" + ] + }, + "error": "BLOB_POLICY_INVALID" + }, + { + "id": "hard-size-limit", + "input": { + "maxSizeBytes": 104857601, + "maxUploadSizeBytes": 104857600 + }, + "error": "BLOB_POLICY_INVALID" + }, + { + "id": "url-capped-by-document", + "input": { + "requestedTtlSeconds": 600, + "documentRemainingSeconds": 120, + "storeMaximumSeconds": 90 + }, + "expected": { + "readUrlTtlSeconds": 90 + } + }, + { + "id": "upload-clock-independent", + "input": { + "beginAtSeconds": 0, + "completedAtSeconds": 3500, + "nowSeconds": 3600 + }, + "expected": { + "uploadSessionExpired": true, + "unboundEligibleAtSeconds": 89900 + } + }, + { + "id": "unbound-deadline-rejects-bind-before-gc", + "input": { + "completedAtSeconds": 0, + "unboundRetentionSeconds": 86400, + "nowSeconds": 86400 + }, + "error": "BLOB_REFERENCE_EXPIRED" + }, + { + "id": "active-reference-never-collected", + "input": { + "bindingCount": 1, + "nowSeconds": 315360000 + }, + "expected": { + "collectable": false + } + }, + { + "id": "maximum-retention-across-fields", + "input": { + "acceptedRetentionSeconds": [ + 604800, + 2592000, + 3600 + ], + "lastUnboundAtSeconds": 100, + "retentionFloorUntilSeconds": 3000000 + }, + "expected": { + "effectiveUnreferencedRetentionSeconds": 2592000, + "unreferencedEligibleAtSeconds": 3000000 + } + }, + { + "id": "profile-update-does-not-shorten", + "input": { + "acceptedUnreferencedRetentionSeconds": 2592000, + "currentProfileRetentionSeconds": 3600 + }, + "expected": { + "effectiveUnreferencedRetentionSeconds": 2592000 + } + }, + { + "id": "read-expiry-does-not-delete", + "input": { + "readUrlExpired": true, + "bindingCount": 1 + }, + "expected": { + "collectable": false, + "requiresReadAuthorization": true + } + }, + { + "id": "deleting-rejects-binding", + "input": { + "status": "deleting" + }, + "error": "BLOB_REFERENCE_EXPIRED" + }, + { + "id": "context-checked-read-is-explicit", + "input": { + "requireContextCheckedRead": true, + "storeCapabilities": [ + "presign", + "range" + ] + }, + "error": "BLOB_CAPABILITY_MISSING" + } + ] +} From 6c0a03c7be734b0f8adc8b1f9f41ee8b91263b4f Mon Sep 17 00:00:00 2001 From: Philip Z Date: Wed, 28 Oct 2026 05:04:28 +0800 Subject: [PATCH 38/75] Align secure document Blob selection fixtures with trusted retention --- .../tests/test_sqlite_document_service.py | 28 ++++++++++++++----- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/packages/teaql-secure-document-reference/tests/test_sqlite_document_service.py b/packages/teaql-secure-document-reference/tests/test_sqlite_document_service.py index 7a75d29b..b615bdfc 100644 --- a/packages/teaql-secure-document-reference/tests/test_sqlite_document_service.py +++ b/packages/teaql-secure-document-reference/tests/test_sqlite_document_service.py @@ -2,6 +2,7 @@ from dataclasses import replace from datetime import datetime, timezone +import time import aiosqlite import pytest @@ -25,7 +26,8 @@ SQLiteContextBoundDocumentService, SQLiteDocumentCommandJournal, StaticReferenceKeyProvider, SubmittedDocumentCommand, TrustedReferencePrincipal, ) -from teaql_blob_reference import BlobBindingChange, BlobBindingTarget, SqliteBlobCatalog +from teaql_blob_reference import (BlobBindingChange, BlobBindingTarget, + BlobRetentionPolicy, SqliteBlobCatalog) class OrderHost: @@ -234,6 +236,10 @@ class Authorization: def scope(self, _actor): return "merchant:7" + async def current_policy_on_lease(self, _actor, lease, _target): + assert lease is sql + return BlobRetentionPolicy("default", "server_stream", 86400, 604800, 3600) + async def authorize(self, _actor, target, _previous, next_ref, _action): return (target == BlobBindingTarget("Order", "10", "attachment") and next_ref == BlobRef.from_sql(selection.stable_key)) @@ -316,6 +322,10 @@ class Authorization: def scope(self, _actor): return "merchant:7" + async def current_policy_on_lease(self, _actor, lease, _target): + assert lease is sql + return BlobRetentionPolicy("default", "server_stream", 86400, 604800, 3600) + async def authorize(self, _actor, target, _previous, next_ref, action): return (target == BlobBindingTarget("OrderLine", selection.entity_id, "attachment") and next_ref == BlobRef.from_sql(selection.stable_key) and action == "Create") @@ -342,9 +352,11 @@ async def test_sqlite_new_child_blob_selection_commits_and_receipt_failure_rolls async with aiosqlite.connect(path) as db: for key in ("blob_C", "blob_D"): await db.execute("""INSERT INTO teaql_blob_catalog - (key,scope,object_key,content_version,file_name,content_type,size_bytes,status) - VALUES (?,?,?,?,?,?,?,'uploaded_unbound')""", - (key, "merchant:7", "objects/" + key, "v1", key + ".pdf", "application/pdf", 3)) + (key,scope,object_key,content_version,file_name,content_type,size_bytes,status, + completed_at_ms,policy_profile,upload_mode,unbound_retention_seconds,effective_unref_seconds) + VALUES (?,?,?,?,?,?,?,'uploaded_unbound',?,?,?,?,?)""", + (key, "merchant:7", "objects/" + key, "v1", key + ".pdf", "application/pdf", 3, + time.time_ns() // 1_000_000, "default", "server_stream", 86400, 604800)) await db.commit() host = ChildBlobOrderHost(catalog) service = SQLiteContextBoundDocumentService(context, old_service.transport, @@ -412,9 +424,11 @@ async def test_sqlite_document_blob_selection_shares_business_binding_and_receip async with aiosqlite.connect(path) as db: for key in ("blob_A", "blob_B"): await db.execute("""INSERT INTO teaql_blob_catalog - (key,scope,object_key,content_version,file_name,content_type,size_bytes,status) - VALUES (?,?,?,?,?,?,?,'uploaded_unbound')""", - (key, "merchant:7", "objects/" + key, "v1", key + ".pdf", "application/pdf", 3)) + (key,scope,object_key,content_version,file_name,content_type,size_bytes,status, + completed_at_ms,policy_profile,upload_mode,unbound_retention_seconds,effective_unref_seconds) + VALUES (?,?,?,?,?,?,?,'uploaded_unbound',?,?,?,?,?)""", + (key, "merchant:7", "objects/" + key, "v1", key + ".pdf", "application/pdf", 3, + time.time_ns() // 1_000_000, "default", "server_stream", 86400, 604800)) await db.commit() host = BlobOrderHost(catalog) service = SQLiteContextBoundDocumentService(context, old_service.transport, From e2b2156ae52ffd1bc0646b24dd91673c3c4d1241 Mon Sep 17 00:00:00 2001 From: Philip Z Date: Wed, 28 Oct 2026 05:31:56 +0800 Subject: [PATCH 39/75] Fence Python File generation sweeps against writers --- packages/teaql-blob-reference/README.md | 2 +- .../src/teaql_blob_reference/file_storage.py | 44 ++++++++++++++++- .../provider/sqlite/test_file_blob_storage.py | 47 +++++++++++++++++++ 3 files changed, 90 insertions(+), 3 deletions(-) diff --git a/packages/teaql-blob-reference/README.md b/packages/teaql-blob-reference/README.md index bb935b28..d44cf189 100644 --- a/packages/teaql-blob-reference/README.md +++ b/packages/teaql-blob-reference/README.md @@ -6,7 +6,7 @@ Optional Blob catalog and current-read services for TeaQL Python. Install this w For retention-aware uploads, the host may implement `current_policy_on_lease(actor, lease, target)` on its upload authorization and return an immutable `BlobRetentionPolicy`. The service obtains this trusted policy on the physical SQLite begin lease and snapshots its profile, upload mode and retention durations into the catalog. It rejects direct mode on this server-stream path. Confirmation records `completed_at_ms`; replay never changes the accepted policy or completion time. Older rows remain NULL-policy after migration and are excluded from retention collection. A binding authorization must provide its own current policy on the graph lease; new bindings of unknown-policy rows fail closed. Successful binding preserves the longest accepted unreferenced duration and minimum retention deadline. The final unbind records `last_unbound_at_ms` while the row remains `available`, so its lifetime is measured from the actual final unbind. The shared policy vectors are pinned by SHA-256 in the tests. -`SqliteBlobMaintenanceService` is an optional, explicitly host-scheduled service for expired incomplete uploads and policy-backed completed, unbound server-stream objects. It includes formerly bound objects only after the final unbind deadline and retention floor. Active bindings, unknown-policy legacy rows and direct uploads are excluded. The host supplies a current scope and async `can_maintain` policy, then calls `run_batch(context, cursor)` and may install the service through `BlobReferenceModule(..., maintenance=service)`. Run `catalog.ensure_schema()` as a separate migration before readiness; it adds a unique object-key index and nullable policy/time columns. Cleanup claims rows under SQLite `BEGIN IMMEDIATE`, pins the observed ETag, uses only version-conditional storage deletion, and can resume after a failed final catalog acknowledgment. `FileObjectStorage` guards manifest publication and conditional deletion with a persistent per-key OS file lock. Its ETag includes an immutable generation token, so identical-byte replacement cannot match an older version. A matching delete removes the current data generation and manifest; older immutable generations remain for a separate collector. Keep each Blob object key unique and immutable, including across manual S3 writes. Direct-upload quiescence, older File generations and cross-machine S3 retention remain outside this service. +`SqliteBlobMaintenanceService` is an optional, explicitly host-scheduled service for expired incomplete uploads and policy-backed completed, unbound server-stream objects. It includes formerly bound objects only after the final unbind deadline and retention floor. Active bindings, unknown-policy legacy rows and direct uploads are excluded. The host supplies a current scope and async `can_maintain` policy, then calls `run_batch(context, cursor)` and may install the service through `BlobReferenceModule(..., maintenance=service)`. Run `catalog.ensure_schema()` as a separate migration before readiness; it adds a unique object-key index and nullable policy/time columns. Cleanup claims rows under SQLite `BEGIN IMMEDIATE`, pins the observed ETag, uses only version-conditional storage deletion, and can resume after a failed final catalog acknowledgment. `FileObjectStorage` guards manifest publication and conditional deletion with a persistent per-key OS file lock. Its ETag includes an immutable generation token, so identical-byte replacement cannot match an older version. A matching delete removes the current data generation and manifest. The host may separately call `FileObjectStorage.sweep_old_generations(key)` for a known object key to remove inaccessible older immutable data generations; it waits for same-key writers and preserves the current manifest generation. Use this only after all processes using that private root run a writer-lock-capable version, and schedule/retry it separately from the catalog receipt. Keep each Blob object key unique and immutable, including across manual S3 writes. Direct-upload quiescence, a catalog-driven File sweep schedule and cross-machine S3 retention remain outside this service. For actual JPEG/PNG parsing, install `teaql-blob-reference[image]` and inject `teaql_blob_reference.content_pillow.PillowBlobContentVerifier`. The image extra diff --git a/packages/teaql-blob-reference/src/teaql_blob_reference/file_storage.py b/packages/teaql-blob-reference/src/teaql_blob_reference/file_storage.py index 56cbd3bf..2117922b 100644 --- a/packages/teaql-blob-reference/src/teaql_blob_reference/file_storage.py +++ b/packages/teaql-blob-reference/src/teaql_blob_reference/file_storage.py @@ -78,9 +78,9 @@ def _directory(self, key_hash): return self._root / key_hash[:2] / key_hash @contextmanager - def _manifest_lock(self, key_hash): + def _manifest_lock(self, key_hash, name="manifest.lock"): """Persistent OS lock: never remove the lock file while this root is in use.""" - path = self._directory(key_hash) / "manifest.lock" + path = self._directory(key_hash) / name flags = os.O_RDWR | os.O_CREAT | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NOFOLLOW", 0) fd = None locked = False @@ -176,6 +176,12 @@ def put(self, invocation, key, content, options): directory.mkdir(mode=0o700, parents=True, exist_ok=True) except OSError: raise _error(StorageErrorCode.IO) from None + # The writer lock covers the data file before its manifest is published. + # A generation sweep takes this lock first and then the manifest lock. + with self._manifest_lock(key_hash, "writer.lock"): + return self._put_under_writer_lock(key, content, options, key_hash, directory) + + def _put_under_writer_lock(self, key, content, options, key_hash, directory): version = _b64(secrets.token_bytes(24)) data_path = directory / ("data-" + version) temp_path = directory / ("manifest-" + version) @@ -297,6 +303,40 @@ def delete_if_version(self, invocation, key, content_version): except OSError: raise _error(StorageErrorCode.IO) from None + def sweep_old_generations(self, key): + """Host-scheduled cleanup of inaccessible File generations for one key. + + The caller owns the provider and must not mix pre-writer-lock process + versions on this root while sweeping. This never removes the current + manifest or its data file, even when the key was recreated. + """ + if not self._root.exists(): + return 0 + self._private_root() + key_hash = self._hash(key) + directory = self._directory(key_hash) + if not directory.exists(): + return 0 + removed = 0 + try: + with self._manifest_lock(key_hash, "writer.lock"): + with self._manifest_lock(key_hash): + manifest = self._manifest(key_hash) + current = "data-" + manifest["version"] if manifest else None + for path in directory.iterdir(): + if path.name == current or not re.fullmatch(r"data-[A-Za-z0-9_-]{32}", path.name): + continue + info = path.lstat() + if not stat.S_ISREG(info.st_mode): + raise _error(StorageErrorCode.UNAVAILABLE) + path.unlink() + removed += 1 + return removed + except StorageError: + raise + except OSError: + raise _error(StorageErrorCode.IO) from None + def _seal(self, ticket): nonce = secrets.token_bytes(12) payload = json.dumps(ticket, separators=(",", ":")).encode("utf-8") diff --git a/tests/provider/sqlite/test_file_blob_storage.py b/tests/provider/sqlite/test_file_blob_storage.py index 5cbc5925..6ef9f26e 100644 --- a/tests/provider/sqlite/test_file_blob_storage.py +++ b/tests/provider/sqlite/test_file_blob_storage.py @@ -1,6 +1,8 @@ """Real local File/HTTP behavior for the independent optional wheel.""" from concurrent.futures import ThreadPoolExecutor from datetime import timedelta +from io import BytesIO +from threading import Event from urllib.error import HTTPError from urllib.request import Request, urlopen import hashlib @@ -69,6 +71,51 @@ def test_file_conditional_delete_fences_same_bytes_and_external_process(tmp_path process.wait(timeout=5) +def test_file_generation_sweep_waits_for_writer_and_preserves_current(tmp_path): + root = tmp_path / "objects" + writer_provider = FileObjectStorage(root, os.urandom(32)) + sweep_provider = FileObjectStorage(root, os.urandom(32)) + actor = UserContext().with_object_storage(writer_provider) + writer = actor.object_storage("write File generations") + key = ObjectKey("blob/generation-sweep") + data = b"same bytes" + first = writer.put_bytes(key, data, PutOptions(len(data), WriteMode.CREATE_ONLY)) + second = writer.put_bytes(key, data, PutOptions(len(data), WriteMode.REPLACE)) + assert first.etag != second.etag + key_hash = sweep_provider._hash(key) + old_path = sweep_provider._directory(key_hash) / ("data-" + first.etag.split(".")[-1]) + assert old_path.is_file() + + reading = Event() + proceed = Event() + + class PausedSource(BytesIO): + def read(self, size=-1): + reading.set() + assert proceed.wait(5) + return super().read(size) + + with ThreadPoolExecutor(max_workers=2) as pool: + replacing = pool.submit(writer_provider.put, None, key, PausedSource(data), + PutOptions(len(data), WriteMode.REPLACE)) + assert reading.wait(5) + sweeping = pool.submit(sweep_provider.sweep_old_generations, key) + time.sleep(0.1) + assert not sweeping.done() + assert old_path.is_file() + proceed.set() + third = replacing.result(timeout=5) + assert sweeping.result(timeout=5) == 2 + + assert sweep_provider.head(None, key).etag == third.etag + assert sweep_provider.sweep_old_generations(key) == 0 + assert not old_path.exists() + current_path = sweep_provider._directory(key_hash) / ("data-" + third.etag.split(".")[-1]) + assert current_path.is_file() + assert sweep_provider.delete_if_version(None, key, third.etag) == "deleted" + assert sweep_provider.sweep_old_generations(key) == 0 + + def _status(url, *, method="GET", headers=None): try: with urlopen(Request(url, method=method, headers=headers or {}), timeout=5) as response: From fbcf8d7530fcace4a8b3dcecf58a2b2b0c562e11 Mon Sep 17 00:00:00 2001 From: Philip Z Date: Wed, 28 Oct 2026 06:13:40 +0800 Subject: [PATCH 40/75] test(blob): gate completed retention against real S3 --- .../provider/sqlite/test_blob_maintenance.py | 64 +++++++++++++++++++ 1 file changed, 64 insertions(+) diff --git a/tests/provider/sqlite/test_blob_maintenance.py b/tests/provider/sqlite/test_blob_maintenance.py index d29cfa7d..e5773b47 100644 --- a/tests/provider/sqlite/test_blob_maintenance.py +++ b/tests/provider/sqlite/test_blob_maintenance.py @@ -357,3 +357,67 @@ async def seed(suffix, content): finally: for key in keys: store.delete(key) + + +@pytest.mark.asyncio +async def test_real_s3_completed_retention_selects_only_eligible_server_stream_objects(tmp_path): + if os.environ.get("TEAQL_S3_TEST_BLOB_MAINTENANCE") != "1": + pytest.skip("explicit disposable S3 maintenance gate required") + endpoint = os.environ.get("TEAQL_S3_TEST_ENDPOINT") + bucket = os.environ.get("TEAQL_S3_TEST_BUCKET") + access = os.environ.get("TEAQL_S3_TEST_ACCESS_KEY") + secret = os.environ.get("TEAQL_S3_TEST_SECRET_KEY") + assert all((endpoint, bucket, access, secret)), "disposable S3 configuration required" + from teaql.provider.s3 import S3ObjectStorage + provider = S3ObjectStorage.connect(bucket, endpoint_url=endpoint, + aws_access_key_id=access, aws_secret_access_key=secret) + actor = RuntimeModule.new().into_context() + actor.with_object_storage(provider).with_object_storage_observer(lambda event: None) + store = actor.object_storage("real S3 completed Blob retention") + path = str(tmp_path / "s3-completed-retention.db") + catalog = SqliteBlobCatalog(SqliteTransport(path)) + await catalog.ensure_schema() + service = SqliteBlobMaintenanceService(catalog, _MaintenanceAuthority(), 10, lambda: 10000) + prefix = uuid4().hex + keys = [] + + async def seed(suffix, status="uploaded_unbound", last_unbound=None, + floor=None, profile="default", mode="server_stream", bound=False): + reference = "blob_" + prefix + "_" + suffix + key = ObjectKey("objects/" + reference) + keys.append(key) + content = ("%PDF-1.7\n" + suffix + "\n%%EOF").encode() + created = store.put_bytes(key, content, PutOptions(len(content), WriteMode.CREATE_ONLY)) + await _seed(path, reference, state="completed", version=created.etag, + written_etag=created.etag, bound=bound) + async with aiosqlite.connect(path) as db: + await db.execute("UPDATE teaql_blob_catalog SET status=?,completed_at_ms=9000," + "last_unbound_at_ms=?,policy_profile=?,upload_mode=?," + "unbound_retention_seconds=1,effective_unref_seconds=2," + "retention_floor_until_ms=?,content_type='application/pdf',size_bytes=? " + "WHERE key=?", (status, last_unbound, profile, mode, floor, + len(content), reference)) + await db.execute("UPDATE teaql_blob_upload SET declared_size=? WHERE key=?", + (len(content), reference)) + await db.commit() + return reference, key + + try: + expired = await seed("expired") + former = await seed("former", "available", 7000, 10000) + active = await seed("active", "available", bound=True) + floored = await seed("floored", "available", 7000, 20000) + legacy = await seed("legacy", profile=None) + direct = await seed("direct", mode="direct") + batch = await service.run_batch(actor) + assert [(item.reference.key, item.status) for item in batch.outcomes] == [ + (expired[0], "deleted"), (former[0], "deleted")] + for reference, key in (expired, former): + assert store.head(key) is None + assert await _status(path, reference) == "deleted" + for reference, key in (active, floored, legacy, direct): + assert store.head(key) is not None + assert await _status(path, reference) != "deleted" + finally: + for key in keys: + store.delete(key) From af99a278abce078f9401b9837cdacb40c7f7d073 Mon Sep 17 00:00:00 2001 From: Philip Z Date: Wed, 28 Oct 2026 06:33:26 +0800 Subject: [PATCH 41/75] Add optional DOCX and XLSX Blob content verifier --- packages/teaql-blob-reference/README.md | 11 ++ packages/teaql-blob-reference/pyproject.toml | 1 + .../teaql_blob_reference/content_office.py | 173 ++++++++++++++++++ .../sqlite/test_blob_content_office.py | 157 ++++++++++++++++ 4 files changed, 342 insertions(+) create mode 100644 packages/teaql-blob-reference/src/teaql_blob_reference/content_office.py create mode 100644 tests/provider/sqlite/test_blob_content_office.py diff --git a/packages/teaql-blob-reference/README.md b/packages/teaql-blob-reference/README.md index d44cf189..a1ec57d9 100644 --- a/packages/teaql-blob-reference/README.md +++ b/packages/teaql-blob-reference/README.md @@ -25,3 +25,14 @@ cross-reference parsing, a real page tree and content operators, and bounded input, pages, decoded page content and concurrent work. It rejects encrypted PDFs. This is parser validation, not malware scanning or hard CPU/process isolation. The PDF fixture verifier used in older S3 tests remains test-only. + +For actual DOCX/XLSX parsing, install `teaql-blob-reference[office]` and inject +`teaql_blob_reference.content_office.OfficeBlobContentVerifier`. The office extra +pins python-docx 1.2.0 and openpyxl 3.1.5. The adapter checks every ZIP member +and its CRC under input, decoded-byte and part-count limits, checks the expected +Office main content type and package relationship, then parses the Word body or +every Excel worksheet under an element/cell count limit. It hashes original +bytes and returns the precise Office MIME type. Construction is passive, and +`validate_ready` requires both optional parsers. The in-process parsers do not +provide hard CPU or process-memory isolation. Binary DOC/XLS, MP3 and MP4 still +need production adapters. diff --git a/packages/teaql-blob-reference/pyproject.toml b/packages/teaql-blob-reference/pyproject.toml index d87b758d..f8b4acd0 100644 --- a/packages/teaql-blob-reference/pyproject.toml +++ b/packages/teaql-blob-reference/pyproject.toml @@ -15,6 +15,7 @@ dependencies = ["teaql==0.3.1"] [project.optional-dependencies] image = ["Pillow==12.3.0"] pdf = ["pypdf==6.21.3"] +office = ["python-docx==1.2.0", "openpyxl==3.1.5"] [tool.setuptools.packages.find] where = ["src"] diff --git a/packages/teaql-blob-reference/src/teaql_blob_reference/content_office.py b/packages/teaql-blob-reference/src/teaql_blob_reference/content_office.py new file mode 100644 index 00000000..788cc29b --- /dev/null +++ b/packages/teaql-blob-reference/src/teaql_blob_reference/content_office.py @@ -0,0 +1,173 @@ +"""Optional bounded DOCX/XLSX parser for Blob upload verification.""" + +from __future__ import annotations + +from hashlib import sha256 +from io import BytesIO +from threading import BoundedSemaphore +from xml.etree import ElementTree +from zipfile import BadZipFile, ZIP_DEFLATED, ZIP_STORED, ZipFile + +from .upload_service import BlobVerifiedContent + + +_MIME = {"docx": "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + "xlsx": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"} +_REQUIRED = {"docx": "word/document.xml", "xlsx": "xl/workbook.xml"} +_MAIN_TYPE = { + "docx": "application/vnd.openxmlformats-officedocument.wordprocessingml.document.main+xml", + "xlsx": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet.main+xml", +} + + +class OfficeBlobContentVerifier: + """Host-selected Office parser; construction does no parser import or storage I/O.""" + + supported_extensions = frozenset(_MIME) + + def __init__(self, maximum_input_bytes: int = 16 * 1024 * 1024, + maximum_decoded_bytes: int = 32 * 1024 * 1024, + maximum_parts: int = 1000, maximum_elements: int = 1_000_000, + concurrent_verifications: int = 2): + if (type(maximum_input_bytes) is not int or not 1 <= maximum_input_bytes <= 100 * 1024 * 1024 + or type(maximum_decoded_bytes) is not int + or not 1 <= maximum_decoded_bytes <= 100 * 1024 * 1024 + or type(maximum_parts) is not int or not 1 <= maximum_parts <= 10000 + or type(maximum_elements) is not int or not 1 <= maximum_elements <= 2_000_000 + or type(concurrent_verifications) is not int + or not 1 <= concurrent_verifications <= 32): + raise RuntimeError("BLOB_POLICY_INVALID") + self.maximum_input_bytes = maximum_input_bytes + self.maximum_decoded_bytes = maximum_decoded_bytes + self.maximum_parts = maximum_parts + self.maximum_elements = maximum_elements + self._permits = BoundedSemaphore(concurrent_verifications) + + def validate_ready(self) -> None: + try: + from docx import Document # noqa: F401 + from openpyxl import load_workbook # noqa: F401 + except ImportError: + raise RuntimeError("BLOB_CAPABILITY_MISSING") from None + + def _check_package(self, data: bytes, extension: str) -> None: + try: + with ZipFile(BytesIO(data)) as archive: + parts = archive.infolist() + if not 1 <= len(parts) <= self.maximum_parts: + raise ValueError("Office part count") + names = set() + for part in parts: + name = part.filename + if (name in names or name.startswith("/") or "\\" in name + or any(piece in ("", ".", "..") for piece in name.split("/")) + or part.flag_bits & 1 or part.compress_type not in (ZIP_STORED, ZIP_DEFLATED)): + raise ValueError("invalid Office member") + names.add(name) + if not {"[Content_Types].xml", "_rels/.rels", _REQUIRED[extension]} <= names: + raise ValueError("missing Office document parts") + decoded = 0 + for part in parts: + if part.is_dir(): + continue + if part.file_size > self.maximum_decoded_bytes - decoded: + raise ValueError("Office decoded size") + with archive.open(part) as member: + while True: + block = member.read(65536) + if not block: + break + decoded += len(block) + if decoded > self.maximum_decoded_bytes: + raise ValueError("Office decoded size") + # Reading through EOF also verifies the member's CRC. + types = ElementTree.fromstring(archive.read("[Content_Types].xml")) + main = "/" + _REQUIRED[extension] + if not any(item.attrib.get("PartName") == main + and item.attrib.get("ContentType") == _MAIN_TYPE[extension] + for item in types): + raise ValueError("wrong Office main content type") + relationships = ElementTree.fromstring(archive.read("_rels/.rels")) + if not any(item.attrib.get("Type", "").endswith("/officeDocument") + and item.attrib.get("Target") == _REQUIRED[extension] + and item.attrib.get("TargetMode", "Internal") == "Internal" + for item in relationships): + raise ValueError("wrong Office root relationship") + except (BadZipFile, OSError, EOFError, RuntimeError) as error: + raise ValueError("invalid Office package") from error + + def _parse_docx(self, data: bytes) -> None: + from docx import Document + + document = Document(BytesIO(data)) + if document.element.body is None: + raise ValueError("missing Word body") + count = 0 + for _ in document.element.body.iter(): + count += 1 + if count > self.maximum_elements: + raise ValueError("Word element limit") + + def _parse_xlsx(self, data: bytes) -> None: + from openpyxl import load_workbook + + workbook = load_workbook(BytesIO(data), read_only=True, data_only=False, + keep_links=False) + try: + if not workbook.worksheets: + raise ValueError("missing Excel worksheet") + count = 0 + for sheet in workbook.worksheets: + sheet.reset_dimensions() # Do not trust producer-supplied used range. + for row in sheet.iter_rows(): + count += len(row) + if count > self.maximum_elements: + raise ValueError("Excel cell limit") + for cell in row: + _ = cell.value + finally: + workbook.close() + + def verify(self, stream, extension: str, maximum_bytes: int) -> BlobVerifiedContent: + if extension not in _MIME: + raise RuntimeError("BLOB_FORMAT_NOT_ALLOWED") + if type(maximum_bytes) is not int or maximum_bytes < 1: + raise RuntimeError("BLOB_POLICY_INVALID") + if not self._permits.acquire(blocking=False): + raise RuntimeError("BLOB_VERIFICATION_BUSY") + try: + maximum = min(maximum_bytes, self.maximum_input_bytes) + chunks = [] + size = 0 + digest = sha256() + while True: + try: + block = stream.read(65536) + except Exception: + raise RuntimeError("BLOB_STORE_FAILED") from None + if not isinstance(block, bytes): + raise RuntimeError("BLOB_STORE_FAILED") + if not block: + break + if len(block) > maximum - size: + raise RuntimeError("BLOB_SIZE_EXCEEDED") + chunks.append(block) + digest.update(block) + size += len(block) + data = b"".join(chunks) + try: + self.validate_ready() + self._check_package(data, extension) + if extension == "docx": + self._parse_docx(data) + else: + self._parse_xlsx(data) + except RuntimeError as error: + if str(error) == "BLOB_CAPABILITY_MISSING": + raise + raise RuntimeError("BLOB_FORMAT_NOT_ALLOWED") from None + except Exception: + raise RuntimeError("BLOB_FORMAT_NOT_ALLOWED") from None + return BlobVerifiedContent(size, digest.hexdigest(), _MIME[extension]) + finally: + self._permits.release() diff --git a/tests/provider/sqlite/test_blob_content_office.py b/tests/provider/sqlite/test_blob_content_office.py new file mode 100644 index 00000000..e83f826c --- /dev/null +++ b/tests/provider/sqlite/test_blob_content_office.py @@ -0,0 +1,157 @@ +"""Actual DOCX/XLSX parsers and SQLite/File upload promotion boundary.""" + +from hashlib import sha256 +from io import BytesIO +from threading import Event, Thread +from zipfile import ZIP_DEFLATED, ZipFile +import os + +import aiosqlite +import pytest + +docx = pytest.importorskip("docx") +openpyxl = pytest.importorskip("openpyxl") + +from teaql.core.blob_ref import BlobFieldSpec +from teaql.provider.sqlite.transport import SqliteTransport +from teaql.runtime import RuntimeModule +from teaql_blob_reference import ( + BlobUploadInput, BlobUploadTarget, FileObjectStorage, SqliteBlobCatalog, + SqliteBlobUploadService, +) +from teaql_blob_reference.content_office import OfficeBlobContentVerifier + + +def office_fixture(extension): + output = BytesIO() + if extension == "docx": + document = docx.Document() + document.add_paragraph("TeaQL Office parser") + document.save(output) + else: + workbook = openpyxl.Workbook() + workbook.active["A1"] = "TeaQL Office parser" + workbook.active["B1"] = 42 + workbook.save(output) + return output.getvalue() + + +def replace_part(data, name, replacement): + output = BytesIO() + with ZipFile(BytesIO(data)) as source, ZipFile(output, "w", ZIP_DEFLATED) as target: + for part in source.infolist(): + target.writestr(part.filename, replacement if part.filename == name + else source.read(part.filename)) + return output.getvalue() + + +@pytest.mark.parametrize("extension,mime", [ + ("docx", "application/vnd.openxmlformats-officedocument.wordprocessingml.document"), + ("xlsx", "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"), +]) +def test_real_office_structure_content_and_limits(extension, mime): + data = office_fixture(extension) + verifier = OfficeBlobContentVerifier() + verifier.validate_ready() + checked = verifier.verify(BytesIO(data), extension, len(data)) + assert (checked.size_bytes, checked.sha256, checked.content_type) == ( + len(data), sha256(data).hexdigest(), mime) + main = "word/document.xml" if extension == "docx" else "xl/workbook.xml" + wrong_type = replace_part(data, "[Content_Types].xml", + b"") + wrong_relationship = replace_part(data, "_rels/.rels", + b"") + for invalid in (data[:-12], b"PK\x03\x04not Office", b"", + replace_part(data, main, b""), wrong_type, + wrong_relationship): + with pytest.raises(RuntimeError, match="BLOB_FORMAT_NOT_ALLOWED"): + verifier.verify(BytesIO(invalid), extension, len(data) + 10000) + with pytest.raises(RuntimeError, match="BLOB_FORMAT_NOT_ALLOWED"): + verifier.verify(BytesIO(data), "pdf", len(data)) + with pytest.raises(RuntimeError, match="BLOB_SIZE_EXCEEDED"): + verifier.verify(BytesIO(data), extension, len(data) - 1) + with pytest.raises(RuntimeError, match="BLOB_FORMAT_NOT_ALLOWED"): + OfficeBlobContentVerifier(maximum_decoded_bytes=1024).verify( + BytesIO(data), extension, len(data)) + with pytest.raises(RuntimeError, match="BLOB_FORMAT_NOT_ALLOWED"): + OfficeBlobContentVerifier(maximum_elements=1).verify( + BytesIO(data), extension, len(data)) + + +def test_concurrent_admission_releases_slot(): + entered, release = Event(), Event() + data = office_fixture("xlsx") + + class PausedInput(BytesIO): + def read(self, size=-1): + entered.set() + assert release.wait(5) + return super().read(size) + + verifier = OfficeBlobContentVerifier(concurrent_verifications=1) + outcome = [] + first = Thread(target=lambda: outcome.append(verifier.verify(PausedInput(data), "xlsx", 100_000))) + first.start() + try: + assert entered.wait(5) + with pytest.raises(RuntimeError, match="BLOB_VERIFICATION_BUSY"): + verifier.verify(BytesIO(data), "xlsx", 100_000) + finally: + release.set() + first.join(5) + assert not first.is_alive() + assert outcome[0].content_type.endswith("sheet") + assert verifier.verify(BytesIO(data), "xlsx", 100_000).size_bytes == len(data) + + +class OfficeAuthority: + def scope(self, _context): + return "tenant-a" + + def field(self, _context, _target): + return BlobFieldSpec("Asset", "document", ["docx", "xlsx"], True, "default") + + async def authorize(self, _context, target, action, _input): + return target.entity == "Asset" and target.field == "document" and action in ( + "Begin", "Write", "Confirm") + + +@pytest.mark.asyncio +async def test_real_office_upload_promotes_both_formats_and_rejects_broken_copy(tmp_path): + path = str(tmp_path / "office.db") + catalog = SqliteBlobCatalog(SqliteTransport(path)) + provider = FileObjectStorage(tmp_path / "objects", os.urandom(32)) + actor = RuntimeModule.new().into_context().with_object_storage(provider) + actor.with_object_storage_observer(lambda event: None) + service = SqliteBlobUploadService(catalog, OfficeAuthority(), + OfficeBlobContentVerifier(), 100_000) + target = BlobUploadTarget("Asset", "1", "document") + try: + await catalog.ensure_schema() + for extension in ("docx", "xlsx"): + good = office_fixture(extension) + upload = await service.begin(actor, target, + BlobUploadInput("valid." + extension, len(good)), + "office-good-" + extension) + assert await service.write(actor, upload.upload_id, BytesIO(good)) == len(good) + assert await service.complete(actor, upload.upload_id) == upload.reference + async with aiosqlite.connect(path) as db: + async with db.execute( + "SELECT status,content_type,size_bytes FROM teaql_blob_catalog WHERE key=?", + (upload.reference.key,)) as cursor: + status, content_type, size = await cursor.fetchone() + assert status == "uploaded_unbound" and size == len(good) + assert content_type == OfficeBlobContentVerifier().verify( + BytesIO(good), extension, len(good)).content_type + broken = office_fixture("xlsx")[:-12] + rejected = await service.begin(actor, target, BlobUploadInput("broken.xlsx", len(broken)), + "office-bad") + await service.write(actor, rejected.upload_id, BytesIO(broken)) + with pytest.raises(RuntimeError, match="BLOB_FORMAT_NOT_ALLOWED"): + await service.complete(actor, rejected.upload_id) + async with aiosqlite.connect(path) as db: + async with db.execute("SELECT status FROM teaql_blob_catalog WHERE key=?", + (rejected.reference.key,)) as cursor: + assert (await cursor.fetchone())[0] == "initiated" + finally: + provider.close() From 40584102318a75f8c40f6a33d88004dcc92ac28c Mon Sep 17 00:00:00 2001 From: Philip Z Date: Wed, 28 Oct 2026 06:37:35 +0800 Subject: [PATCH 42/75] Verify parsed Office uploads through S3 Blob graph and view --- tests/provider/sqlite/test_blob_upload.py | 27 ++++++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/tests/provider/sqlite/test_blob_upload.py b/tests/provider/sqlite/test_blob_upload.py index 592420bd..90c3acd0 100644 --- a/tests/provider/sqlite/test_blob_upload.py +++ b/tests/provider/sqlite/test_blob_upload.py @@ -85,7 +85,8 @@ def verify(self, stream, extension, maximum_bytes): @pytest.mark.asyncio -@pytest.mark.parametrize("case", ["fixture-pdf", "real-jpeg", "real-pdf"]) +@pytest.mark.parametrize("case", ["fixture-pdf", "real-jpeg", "real-pdf", + "real-docx", "real-xlsx"]) async def test_upload_real_sqlite_s3_create_only_confirm_and_replay(tmp_path, case): if os.environ.get("TEAQL_S3_TEST_BLOB_UPLOAD") != "1": pytest.skip("explicit S3-compatible Blob upload gate required") @@ -93,6 +94,8 @@ async def test_upload_real_sqlite_s3_create_only_confirm_and_replay(tmp_path, ca pytest.skip("explicit production JPEG S3 gate required") if case == "real-pdf" and os.environ.get("TEAQL_S3_TEST_REAL_PDF") != "1": pytest.skip("explicit production PDF S3 gate required") + if case in ("real-docx", "real-xlsx") and os.environ.get("TEAQL_S3_TEST_REAL_OFFICE") != "1": + pytest.skip("explicit production Office S3 gate required") endpoint = os.environ.get("TEAQL_S3_TEST_ENDPOINT") bucket = os.environ.get("TEAQL_S3_TEST_BUCKET") access = os.environ.get("TEAQL_S3_TEST_ACCESS_KEY") @@ -124,6 +127,24 @@ async def test_upload_real_sqlite_s3_create_only_confirm_and_replay(tmp_path, ca authority, parser = UploadAuthority(("pdf",)), PypdfBlobContentVerifier() data = (Path(__file__).parents[2] / "fixtures" / "blob-sample.pdf").read_bytes() file_name, content_type = "sample.pdf", "application/pdf" + elif case in ("real-docx", "real-xlsx"): + from teaql_blob_reference.content_office import OfficeBlobContentVerifier + extension = case.removeprefix("real-") + authority, parser = UploadAuthority((extension,)), OfficeBlobContentVerifier() + output = BytesIO() + if extension == "docx": + from docx import Document + document = Document() + document.add_paragraph("TeaQL S3 Office upload") + document.save(output) + content_type = "application/vnd.openxmlformats-officedocument.wordprocessingml.document" + else: + from openpyxl import Workbook + workbook = Workbook() + workbook.active["A1"] = "TeaQL S3 Office upload" + workbook.save(output) + content_type = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" + data, file_name = output.getvalue(), "sample." + extension else: authority, parser = UploadAuthority(), FixturePDFVerifier() data = b"%PDF-1.7\nreal S3 Blob upload\n%%EOF" @@ -139,7 +160,7 @@ def verify(self, stream, extension, maximum_bytes): return self.inner.verify(stream, extension, maximum_bytes) verifier = CountingVerifier(parser) - service = SqliteBlobUploadService(catalog, authority, verifier, 10_000) + service = SqliteBlobUploadService(catalog, authority, verifier, 100_000) object_key = None try: async with aiosqlite.connect(path) as db: @@ -163,7 +184,7 @@ def verify(self, stream, extension, maximum_bytes): assert verifier.calls == 1 assert await service.write(actor, session.upload_id, BytesIO(data)) == len(data) store = actor.object_storage("verify completed S3 Blob upload") - assert store.get_bytes(object_key, 10_000) == data + assert store.get_bytes(object_key, 100_000) == data head = store.head(object_key) async with aiosqlite.connect(path) as db: async with db.execute( From b6d6afd4acbfab2ae3f71c2e3d9df4aa13764c60 Mon Sep 17 00:00:00 2001 From: Philip Z Date: Wed, 28 Oct 2026 06:41:21 +0800 Subject: [PATCH 43/75] Reject Office XML entity declarations before parsing --- packages/teaql-blob-reference/README.md | 3 ++- .../src/teaql_blob_reference/content_office.py | 7 +++++++ tests/provider/sqlite/test_blob_content_office.py | 4 +++- 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/packages/teaql-blob-reference/README.md b/packages/teaql-blob-reference/README.md index a1ec57d9..c50f97af 100644 --- a/packages/teaql-blob-reference/README.md +++ b/packages/teaql-blob-reference/README.md @@ -29,7 +29,8 @@ isolation. The PDF fixture verifier used in older S3 tests remains test-only. For actual DOCX/XLSX parsing, install `teaql-blob-reference[office]` and inject `teaql_blob_reference.content_office.OfficeBlobContentVerifier`. The office extra pins python-docx 1.2.0 and openpyxl 3.1.5. The adapter checks every ZIP member -and its CRC under input, decoded-byte and part-count limits, checks the expected +and its CRC under input, decoded-byte and part-count limits, rejects XML entity +declarations, checks the expected Office main content type and package relationship, then parses the Word body or every Excel worksheet under an element/cell count limit. It hashes original bytes and returns the precise Office MIME type. Construction is passive, and diff --git a/packages/teaql-blob-reference/src/teaql_blob_reference/content_office.py b/packages/teaql-blob-reference/src/teaql_blob_reference/content_office.py index 788cc29b..ce12ee6b 100644 --- a/packages/teaql-blob-reference/src/teaql_blob_reference/content_office.py +++ b/packages/teaql-blob-reference/src/teaql_blob_reference/content_office.py @@ -72,6 +72,8 @@ def _check_package(self, data: bytes, extension: str) -> None: continue if part.file_size > self.maximum_decoded_bytes - decoded: raise ValueError("Office decoded size") + xml_part = part.filename.endswith((".xml", ".rels")) + overlap = b"" with archive.open(part) as member: while True: block = member.read(65536) @@ -80,6 +82,11 @@ def _check_package(self, data: bytes, extension: str) -> None: decoded += len(block) if decoded > self.maximum_decoded_bytes: raise ValueError("Office decoded size") + if xml_part: + scan = (overlap + block).upper() + if b"") wrong_relationship = replace_part(data, "_rels/.rels", b"") + entity_document = replace_part(data, main, + b"]>&x;") for invalid in (data[:-12], b"PK\x03\x04not Office", b"", replace_part(data, main, b""), wrong_type, - wrong_relationship): + wrong_relationship, entity_document): with pytest.raises(RuntimeError, match="BLOB_FORMAT_NOT_ALLOWED"): verifier.verify(BytesIO(invalid), extension, len(data) + 10000) with pytest.raises(RuntimeError, match="BLOB_FORMAT_NOT_ALLOWED"): From 326dd944bfdef80659fc8a422877d5322377a35a Mon Sep 17 00:00:00 2001 From: Philip Z Date: Wed, 28 Oct 2026 06:52:37 +0800 Subject: [PATCH 44/75] Add catalog-driven File generation sweep service --- packages/teaql-blob-reference/README.md | 14 +- .../src/teaql_blob_reference/__init__.py | 5 + .../file_generation_maintenance.py | 104 +++++++++++++++ .../test_blob_file_generation_maintenance.py | 125 ++++++++++++++++++ 4 files changed, 247 insertions(+), 1 deletion(-) create mode 100644 packages/teaql-blob-reference/src/teaql_blob_reference/file_generation_maintenance.py create mode 100644 tests/provider/sqlite/test_blob_file_generation_maintenance.py diff --git a/packages/teaql-blob-reference/README.md b/packages/teaql-blob-reference/README.md index c50f97af..16a59fb9 100644 --- a/packages/teaql-blob-reference/README.md +++ b/packages/teaql-blob-reference/README.md @@ -6,7 +6,19 @@ Optional Blob catalog and current-read services for TeaQL Python. Install this w For retention-aware uploads, the host may implement `current_policy_on_lease(actor, lease, target)` on its upload authorization and return an immutable `BlobRetentionPolicy`. The service obtains this trusted policy on the physical SQLite begin lease and snapshots its profile, upload mode and retention durations into the catalog. It rejects direct mode on this server-stream path. Confirmation records `completed_at_ms`; replay never changes the accepted policy or completion time. Older rows remain NULL-policy after migration and are excluded from retention collection. A binding authorization must provide its own current policy on the graph lease; new bindings of unknown-policy rows fail closed. Successful binding preserves the longest accepted unreferenced duration and minimum retention deadline. The final unbind records `last_unbound_at_ms` while the row remains `available`, so its lifetime is measured from the actual final unbind. The shared policy vectors are pinned by SHA-256 in the tests. -`SqliteBlobMaintenanceService` is an optional, explicitly host-scheduled service for expired incomplete uploads and policy-backed completed, unbound server-stream objects. It includes formerly bound objects only after the final unbind deadline and retention floor. Active bindings, unknown-policy legacy rows and direct uploads are excluded. The host supplies a current scope and async `can_maintain` policy, then calls `run_batch(context, cursor)` and may install the service through `BlobReferenceModule(..., maintenance=service)`. Run `catalog.ensure_schema()` as a separate migration before readiness; it adds a unique object-key index and nullable policy/time columns. Cleanup claims rows under SQLite `BEGIN IMMEDIATE`, pins the observed ETag, uses only version-conditional storage deletion, and can resume after a failed final catalog acknowledgment. `FileObjectStorage` guards manifest publication and conditional deletion with a persistent per-key OS file lock. Its ETag includes an immutable generation token, so identical-byte replacement cannot match an older version. A matching delete removes the current data generation and manifest. The host may separately call `FileObjectStorage.sweep_old_generations(key)` for a known object key to remove inaccessible older immutable data generations; it waits for same-key writers and preserves the current manifest generation. Use this only after all processes using that private root run a writer-lock-capable version, and schedule/retry it separately from the catalog receipt. Keep each Blob object key unique and immutable, including across manual S3 writes. Direct-upload quiescence, a catalog-driven File sweep schedule and cross-machine S3 retention remain outside this service. +`SqliteBlobMaintenanceService` is an optional, explicitly host-scheduled service for expired incomplete uploads and policy-backed completed, unbound server-stream objects. It includes formerly bound objects only after the final unbind deadline and retention floor. Active bindings, unknown-policy legacy rows and direct uploads are excluded. The host supplies a current scope and async `can_maintain` policy, then calls `run_batch(context, cursor)` and may install the service through `BlobReferenceModule(..., maintenance=service)`. Run `catalog.ensure_schema()` as a separate migration before readiness; it adds a unique object-key index and nullable policy/time columns. Cleanup claims rows under SQLite `BEGIN IMMEDIATE`, pins the observed ETag, uses only version-conditional storage deletion, and can resume after a failed final catalog acknowledgment. `FileObjectStorage` guards manifest publication and conditional deletion with a persistent per-key OS file lock. Its ETag includes an immutable generation token, so identical-byte replacement cannot match an older version. A matching delete removes the current data generation and manifest. The host may separately call `FileObjectStorage.sweep_old_generations(key)` for a known object key to remove inaccessible older immutable data generations; it waits for same-key writers and preserves the current manifest generation. Use this only after all processes using that private root run a writer-lock-capable version, and schedule/retry it separately from the catalog receipt. Keep each Blob object key unique and immutable, including across manual S3 writes. Direct-upload quiescence and cross-machine S3 retention remain outside this service. + +`SqliteBlobFileGenerationSweepService` supplies a separate, host-scheduled +catalog-driven File sweep. After every process sharing the private File root +has upgraded to the writer-lock protocol, the host constructs it with +`all_writers_upgraded=True` and calls `run_batch(actor, provider, cursor)` +outside graph SQL. It pages only the authorized scope's `server_stream` rows, +including `deleted` rows that may retain inaccessible old data generations. +The provider must be the exact `FileObjectStorage` installed in the current +context. Each outcome reports removed generations or a sanitized failure; +restart the scan from the beginning to retry a failed row. Keep terminal +catalog rows until the File sweep has completed. The host assertion cannot +detect an older external writer on its own; no worker or timer starts at import. For actual JPEG/PNG parsing, install `teaql-blob-reference[image]` and inject `teaql_blob_reference.content_pillow.PillowBlobContentVerifier`. The image extra diff --git a/packages/teaql-blob-reference/src/teaql_blob_reference/__init__.py b/packages/teaql-blob-reference/src/teaql_blob_reference/__init__.py index 59e20059..45d873c6 100644 --- a/packages/teaql-blob-reference/src/teaql_blob_reference/__init__.py +++ b/packages/teaql-blob-reference/src/teaql_blob_reference/__init__.py @@ -8,6 +8,9 @@ SqliteBlobMaintenanceService) from .mutation_participant import SqliteBlobMutationParticipant from .file_storage import FileObjectStorage +from .file_generation_maintenance import (BlobFileGenerationSweepBatch, + BlobFileGenerationSweepOutcome, + SqliteBlobFileGenerationSweepService) from .retention import BlobRetentionPolicy from .upload_service import (BlobUploadInput, BlobUploadSession, BlobUploadTarget, BlobVerifiedContent, SqliteBlobUploadService) @@ -15,6 +18,8 @@ __all__ = ["BlobReferenceModule", "SqliteBlobMutationParticipant", "BlobBindingChange", "BlobBindingTarget", "BlobCatalogRead", "BlobMaintenanceAuthorization", "BlobMaintenanceBatch", "BlobMaintenanceCursor", "BlobMaintenanceOutcome", "SqliteBlobMaintenanceService", + "BlobFileGenerationSweepBatch", "BlobFileGenerationSweepOutcome", + "SqliteBlobFileGenerationSweepService", "SqliteBlobCatalog", "SqliteBlobResponseContributor", "FileObjectStorage", "BlobUploadInput", "BlobUploadSession", "BlobUploadTarget", "BlobVerifiedContent", "SqliteBlobUploadService", "BlobRetentionPolicy"] diff --git a/packages/teaql-blob-reference/src/teaql_blob_reference/file_generation_maintenance.py b/packages/teaql-blob-reference/src/teaql_blob_reference/file_generation_maintenance.py new file mode 100644 index 00000000..b0bd5693 --- /dev/null +++ b/packages/teaql-blob-reference/src/teaql_blob_reference/file_generation_maintenance.py @@ -0,0 +1,104 @@ +"""Explicit catalog-driven cleanup of inaccessible fenced File generations.""" + +from __future__ import annotations + +import asyncio +from dataclasses import dataclass + +from teaql.core.blob_ref import BlobRef +from teaql.core.object_storage import ObjectKey, StorageError +from teaql.core.value import Value +from teaql.sql.executor import SqlTransactionTransport +from teaql.sql.types import CompiledQuery + +from .file_storage import FileObjectStorage +from .maintenance_service import BlobMaintenanceAuthorization, BlobMaintenanceCursor +from .sqlite_catalog import SqliteBlobCatalog, _scope + + +@dataclass(frozen=True) +class BlobFileGenerationSweepOutcome: + reference: BlobRef + status: str + removed_generations: int | None = None + error: str | None = None + + +@dataclass(frozen=True) +class BlobFileGenerationSweepBatch: + outcomes: tuple[BlobFileGenerationSweepOutcome, ...] + next: BlobMaintenanceCursor | None + + +class SqliteBlobFileGenerationSweepService: + """Host-scheduled sweep; the host must first upgrade every root writer.""" + + def __init__(self, catalog: SqliteBlobCatalog, + authorization: BlobMaintenanceAuthorization, + all_writers_upgraded: bool, batch_size: int = 100): + if (not isinstance(catalog, SqliteBlobCatalog) + or not isinstance(catalog.transport, SqlTransactionTransport) + or authorization is None + or not callable(getattr(authorization, "scope", None)) + or not callable(getattr(authorization, "can_maintain", None)) + or all_writers_upgraded is not True + or type(batch_size) is not int or not 1 <= batch_size <= 1000): + raise RuntimeError("BLOB_POLICY_INVALID") + self.catalog = catalog + self.authorization = authorization + self.batch_size = batch_size + + async def _authorize(self, actor, expected: str | None = None) -> str: + if actor is None: + raise RuntimeError("BLOB_MAINTENANCE_DENIED") + try: + scope = self.authorization.scope(actor) + allowed = await self.authorization.can_maintain(actor, scope) + except Exception: + raise RuntimeError("BLOB_MAINTENANCE_DENIED") from None + if not _scope(scope) or expected is not None and scope != expected or not allowed: + raise RuntimeError("BLOB_MAINTENANCE_DENIED") + return scope + + async def run_batch(self, actor, provider: FileObjectStorage, + cursor: BlobMaintenanceCursor | None = None) -> BlobFileGenerationSweepBatch: + scope = await self._authorize(actor) + if cursor is not None and (not isinstance(cursor, BlobMaintenanceCursor) + or cursor.scope != scope): + raise RuntimeError("BLOB_MAINTENANCE_DENIED") + actor.require_no_active_graph_sql() + if (not isinstance(provider, FileObjectStorage) + or actor.get_resource("object_storage_provider") is not provider): + raise RuntimeError("BLOB_CAPABILITY_MISSING") + await self.catalog.validate_schema() + try: + rows = await self.catalog.transport.fetch_all_sql(CompiledQuery( + "SELECT key,object_key FROM teaql_blob_catalog " + "WHERE scope=? AND key>? AND upload_mode='server_stream' " + "ORDER BY key LIMIT ?", + [Value.Text(scope), Value.Text(cursor.after if cursor else ""), + Value.I64(self.batch_size)])) + except Exception: + raise RuntimeError("BLOB_CATALOG_FAILED") from None + candidates = [] + for row in rows: + try: + reference = BlobRef.from_sql(row.get("key")) + object_key = ObjectKey(row.get("object_key")) + except (ValueError, TypeError, StorageError): + raise RuntimeError("BLOB_CATALOG_FAILED") from None + candidates.append((reference, object_key)) + outcomes = [] + for reference, object_key in candidates: + await self._authorize(actor, scope) + try: + removed = await asyncio.to_thread(provider.sweep_old_generations, object_key) + if type(removed) is not int or removed < 0: + raise RuntimeError("BLOB_STORE_FAILED") + outcomes.append(BlobFileGenerationSweepOutcome(reference, "swept", removed)) + except Exception: + outcomes.append(BlobFileGenerationSweepOutcome(reference, "failed", None, + "BLOB_STORE_FAILED")) + next_cursor = (BlobMaintenanceCursor(scope, candidates[-1][0].key) + if len(candidates) == self.batch_size else None) + return BlobFileGenerationSweepBatch(tuple(outcomes), next_cursor) diff --git a/tests/provider/sqlite/test_blob_file_generation_maintenance.py b/tests/provider/sqlite/test_blob_file_generation_maintenance.py new file mode 100644 index 00000000..0a7a3918 --- /dev/null +++ b/tests/provider/sqlite/test_blob_file_generation_maintenance.py @@ -0,0 +1,125 @@ +"""Catalog-scoped scheduling for real fenced File old-generation cleanup.""" + +import os + +import aiosqlite +import pytest + +from teaql.core.object_storage import ObjectKey, PutOptions, WriteMode +from teaql.provider.sqlite.transport import SqliteTransport +from teaql.runtime import RuntimeModule +from teaql_blob_reference import ( + BlobMaintenanceCursor, FileObjectStorage, SqliteBlobCatalog, + SqliteBlobFileGenerationSweepService, +) + + +class FailOnceFileStorage(FileObjectStorage): + fail_next_sweep = False + + def sweep_old_generations(self, key): + if self.fail_next_sweep: + self.fail_next_sweep = False + raise OSError("transient File failure") + return super().sweep_old_generations(key) + + +class MaintenanceAuthority: + scope_value = "tenant-a" + allowed = True + + def scope(self, _actor): + return self.scope_value + + async def can_maintain(self, _actor, _scope): + return self.allowed + + +async def seed(path, reference, scope, object_key, version, status): + async with aiosqlite.connect(path) as db: + await db.execute("""INSERT INTO teaql_blob_catalog + (key,scope,object_key,content_version,file_name,content_type,size_bytes, + completed_at_ms,policy_profile,upload_mode,unbound_retention_seconds, + effective_unref_seconds,status) + VALUES (?,?,?,?,?,?,?,?,'default','server_stream',86400,86400,?)""", + (reference, scope, object_key.value, version, "report.pdf", "application/pdf", + 10, 1000, status)) + await db.execute("""INSERT INTO teaql_blob_upload + (upload_id,scope,replay_key,request_hash,key,entity,entity_id,field,file_name, + extension,declared_size,expires_ms,state,written_etag) + VALUES (?,?,?,?,?,'Asset','1','document','report.pdf','pdf',10,9000, + 'completed',?)""", + ("bu_" + reference, scope, "replay_" + reference, "hash_" + reference, + reference, version)) + await db.commit() + + +@pytest.mark.asyncio +async def test_catalog_file_sweep_covers_deleted_rows_preserves_current_and_retries(tmp_path): + path = str(tmp_path / "catalog.db") + catalog = SqliteBlobCatalog(SqliteTransport(path)) + await catalog.ensure_schema() + provider = FailOnceFileStorage(tmp_path / "objects", os.urandom(32)) + actor = RuntimeModule.new().into_context().with_object_storage(provider) + actor.with_object_storage_observer(lambda event: None) + store = actor.object_storage("catalog File sweep") + authority = MaintenanceAuthority() + service = SqliteBlobFileGenerationSweepService(catalog, authority, True, 1) + original = b"first data" + replacement = b"replacement data" + keys = [ObjectKey("objects/blob_a"), ObjectKey("objects/blob_b"), + ObjectKey("objects/blob_other")] + try: + assert provider is actor.get_resource("object_storage_provider") + for key, reference, scope, status in zip( + keys, ("blob_a", "blob_b", "blob_other"), + ("tenant-a", "tenant-a", "tenant-b"), + ("deleted", "uploaded_unbound", "uploaded_unbound")): + store.put_bytes(key, original, + PutOptions(len(original), WriteMode.CREATE_ONLY)) + current = store.put_bytes(key, replacement, + PutOptions(len(replacement), WriteMode.REPLACE)) + if status == "deleted": + assert store.delete_if_version(key, current.etag) == "deleted" + await seed(path, reference, scope, key, current.etag, status) + + with pytest.raises(RuntimeError, match="BLOB_POLICY_INVALID"): + SqliteBlobFileGenerationSweepService(catalog, authority, False) + authority.allowed = False + with pytest.raises(RuntimeError, match="BLOB_MAINTENANCE_DENIED"): + await service.run_batch(actor, provider) + authority.allowed = True + with pytest.raises(RuntimeError, match="BLOB_MAINTENANCE_DENIED"): + await service.run_batch(actor, provider, BlobMaintenanceCursor("tenant-b", "")) + token = actor._graph_save_guard.set({"active": True}) + try: + with pytest.raises(RuntimeError, match="BLOB_TRANSACTION_REQUIRED"): + await service.run_batch(actor, provider) + finally: + actor._graph_save_guard.reset(token) + with pytest.raises(RuntimeError, match="BLOB_CAPABILITY_MISSING"): + await service.run_batch(actor, FileObjectStorage(tmp_path / "objects", os.urandom(32))) + + provider.fail_next_sweep = True + failed = await service.run_batch(actor, provider) + assert [(entry.status, entry.error) for entry in failed.outcomes] == [ + ("failed", "BLOB_STORE_FAILED")] + outcomes = [] + cursor = None + while True: + batch = await service.run_batch(actor, provider, cursor) + outcomes.extend(batch.outcomes) + if batch.next is None: + break + cursor = batch.next + assert len(outcomes) == 2 + assert [entry.status for entry in outcomes] == ["swept", "swept"] + assert [entry.removed_generations for entry in outcomes] == [1, 1] + assert store.head(keys[0]) is None + assert store.get_bytes(keys[1], 100) == replacement + assert store.get_bytes(keys[2], 100) == replacement + assert provider.sweep_old_generations(keys[0]) == 0 + assert provider.sweep_old_generations(keys[1]) == 0 + assert provider.sweep_old_generations(keys[2]) == 1 # Other tenant was not scanned. + finally: + provider.close() From 1e8b2a261deeaf2c287740578202d930f55751bc Mon Sep 17 00:00:00 2001 From: Philip Z Date: Wed, 28 Oct 2026 07:35:31 +0800 Subject: [PATCH 45/75] Add optional bounded FFmpeg verifier for MP3 and MP4 uploads --- packages/teaql-blob-reference/README.md | 20 +- .../teaql_blob_reference/content_ffmpeg.py | 233 ++++++++++++++++++ tests/fixtures/reference-modules/sample.mp3 | Bin 0 -> 2107 bytes tests/fixtures/reference-modules/sample.mp4 | Bin 0 -> 1597 bytes .../sqlite/test_blob_content_ffmpeg.py | 144 +++++++++++ 5 files changed, 395 insertions(+), 2 deletions(-) create mode 100644 packages/teaql-blob-reference/src/teaql_blob_reference/content_ffmpeg.py create mode 100644 tests/fixtures/reference-modules/sample.mp3 create mode 100644 tests/fixtures/reference-modules/sample.mp4 create mode 100644 tests/provider/sqlite/test_blob_content_ffmpeg.py diff --git a/packages/teaql-blob-reference/README.md b/packages/teaql-blob-reference/README.md index 16a59fb9..a3673259 100644 --- a/packages/teaql-blob-reference/README.md +++ b/packages/teaql-blob-reference/README.md @@ -47,5 +47,21 @@ Office main content type and package relationship, then parses the Word body or every Excel worksheet under an element/cell count limit. It hashes original bytes and returns the precise Office MIME type. Construction is passive, and `validate_ready` requires both optional parsers. The in-process parsers do not -provide hard CPU or process-memory isolation. Binary DOC/XLS, MP3 and MP4 still -need production adapters. +provide hard CPU or process-memory isolation. Binary DOC/XLS still need +production adapters. + +For actual MP3/MP4 validation on POSIX hosts, inject +`teaql_blob_reference.content_ffmpeg.FfmpegBlobContentVerifier` with absolute +paths to trusted `ffprobe` and `ffmpeg` executables. The host installs and +patches those binaries separately; this wheel does not bundle or fetch them. +`validate_ready` checks availability when the module is activated. The adapter +walks complete MP3 frames or top-level MP4 boxes before forcing the matching +FFmpeg demuxer, probing decoded frame counts and stream properties, and fully +decoding media from a pipe. Client filenames and URLs never reach FFmpeg. It +allows only the `pipe` protocol and bounds input bytes, frame count, video +pixels, concurrent checks, child CPU time and address space, and wall time. +The accepted MP3 profile is MPEG Layer III with fixed, non-free bitrate and a +constant sample rate; MP4 must contain a supported `ftyp` brand, `moov`, +`mdat`, and at least one decodable video stream. The adapter returns the SHA-256 +of the original bytes and `audio/mpeg` or `video/mp4`. It is content validation, +not malware scanning; ffprobe/ffmpeg themselves must be kept current. diff --git a/packages/teaql-blob-reference/src/teaql_blob_reference/content_ffmpeg.py b/packages/teaql-blob-reference/src/teaql_blob_reference/content_ffmpeg.py new file mode 100644 index 00000000..1f032a03 --- /dev/null +++ b/packages/teaql-blob-reference/src/teaql_blob_reference/content_ffmpeg.py @@ -0,0 +1,233 @@ +"""Optional bounded MP3/MP4 verifier backed by host-supplied FFmpeg binaries. + +Input is passed over a pipe with only the pipe protocol enabled. The subprocess +never receives a client filename, URL, or shell command. +""" + +from __future__ import annotations + +from hashlib import sha256 +from json import loads +from os import access, name as os_name, path, X_OK +from subprocess import DEVNULL, PIPE, run +from threading import BoundedSemaphore + +from .upload_service import BlobVerifiedContent + +if os_name == "posix": + from resource import RLIMIT_AS, RLIMIT_CPU, setrlimit + + +_MIME = {"mp3": "audio/mpeg", "mp4": "video/mp4"} +_MP4_BRANDS = {b"isom", b"iso2", b"iso5", b"iso6", b"mp41", b"mp42", + b"avc1", b"M4V ", b"M4V2", b"dash"} +_MPEG1_LAYER3_BITRATES = (0, 32, 40, 48, 56, 64, 80, 96, 112, 128, + 160, 192, 224, 256, 320, 0) +_MPEG2_LAYER3_BITRATES = (0, 8, 16, 24, 32, 40, 48, 56, 64, 80, + 96, 112, 128, 144, 160, 0) + + +def _complete_mp3(data: bytes, maximum_frames: int) -> bool: + offset = 0 + if data.startswith(b"ID3"): + if len(data) < 10 or data[3] not in (2, 3, 4) or data[4] == 0xff: + return False + if any(part & 0x80 for part in data[6:10]): + return False + tag_size = (data[6] << 21) | (data[7] << 14) | (data[8] << 7) | data[9] + footer = 10 if data[3] == 4 and data[5] & 0x10 else 0 + offset = 10 + tag_size + footer + if offset > len(data): + return False + end = len(data) - (128 if len(data) >= 128 and data[-128:-125] == b"TAG" else 0) + frames = 0 + first_rate = None + while offset < end: + if end - offset < 4 or frames >= maximum_frames: + return False + header = int.from_bytes(data[offset:offset + 4], "big") + version = (header >> 19) & 3 + layer = (header >> 17) & 3 + bitrate_index = (header >> 12) & 15 + sample_index = (header >> 10) & 3 + padding = (header >> 9) & 1 + if (header >> 21 != 0x7ff or version == 1 or layer != 1 + or bitrate_index in (0, 15) or sample_index == 3): + return False + rate = (44100, 48000, 32000)[sample_index] + rate //= 1 if version == 3 else 2 if version == 2 else 4 + if first_rate is None: + first_rate = rate + elif first_rate != rate: + return False + bitrates = _MPEG1_LAYER3_BITRATES if version == 3 else _MPEG2_LAYER3_BITRATES + length = (144000 if version == 3 else 72000) * bitrates[bitrate_index] // rate + padding + if length < 4 or length > end - offset: + return False + offset += length + frames += 1 + return frames > 0 and offset == end + + +def _complete_mp4(data: bytes, maximum_boxes: int) -> bool: + offset = 0 + count = 0 + kinds = set() + while offset < len(data): + if len(data) - offset < 8 or count >= maximum_boxes: + return False + size = int.from_bytes(data[offset:offset + 4], "big") + kind = data[offset + 4:offset + 8] + header = 8 + if size == 1: + if len(data) - offset < 16: + return False + size = int.from_bytes(data[offset + 8:offset + 16], "big") + header = 16 + elif size == 0: + if kind != b"mdat": + return False + size = len(data) - offset + if size < header or size > len(data) - offset: + return False + if count == 0: + if kind != b"ftyp" or size < header + 8 or data[offset + header:offset + header + 4] not in _MP4_BRANDS: + return False + if kind in (b"ftyp", b"moov") and kind in kinds: + return False + kinds.add(kind) + offset += size + count += 1 + return {b"ftyp", b"moov", b"mdat"} <= kinds and offset == len(data) + + +class FfmpegBlobContentVerifier: + """Passive Linux/POSIX adapter; full decode occurs only during verification.""" + + supported_extensions = frozenset(_MIME) + + def __init__(self, ffprobe_path: str, ffmpeg_path: str, + maximum_input_bytes: int = 32 * 1024 * 1024, + maximum_frames: int = 100_000, maximum_video_pixels: int = 16_000_000, + concurrent_verifications: int = 2): + if (not isinstance(ffprobe_path, str) or not path.isabs(ffprobe_path) + or not isinstance(ffmpeg_path, str) or not path.isabs(ffmpeg_path) + or type(maximum_input_bytes) is not int or not 1 <= maximum_input_bytes <= 100 * 1024 * 1024 + or type(maximum_frames) is not int or not 1 <= maximum_frames <= 1_000_000 + or type(maximum_video_pixels) is not int or not 1 <= maximum_video_pixels <= 64_000_000 + or type(concurrent_verifications) is not int + or not 1 <= concurrent_verifications <= 32): + raise RuntimeError("BLOB_POLICY_INVALID") + self.ffprobe_path = ffprobe_path + self.ffmpeg_path = ffmpeg_path + self.maximum_input_bytes = maximum_input_bytes + self.maximum_frames = maximum_frames + self.maximum_video_pixels = maximum_video_pixels + self._permits = BoundedSemaphore(concurrent_verifications) + + @staticmethod + def _limits() -> None: + setrlimit(RLIMIT_CPU, (10, 10)) + setrlimit(RLIMIT_AS, (512 * 1024 * 1024, 512 * 1024 * 1024)) + + def validate_ready(self) -> None: + if os_name != "posix": + raise RuntimeError("BLOB_CAPABILITY_MISSING") + for binary, prefix in ((self.ffprobe_path, b"ffprobe version "), + (self.ffmpeg_path, b"ffmpeg version ")): + if not path.isfile(binary) or not access(binary, X_OK): + raise RuntimeError("BLOB_CAPABILITY_MISSING") + try: + completed = run((binary, "-version"), stdout=PIPE, stderr=DEVNULL, + timeout=3, check=False, env={"LC_ALL": "C"}) + except Exception: + raise RuntimeError("BLOB_CAPABILITY_MISSING") from None + if completed.returncode != 0 or not completed.stdout.startswith(prefix): + raise RuntimeError("BLOB_CAPABILITY_MISSING") + + def _run(self, binary: str, args: tuple[str, ...], data: bytes, output: bool): + try: + completed = run((binary, *args), input=data, stdout=PIPE if output else DEVNULL, + stderr=DEVNULL, timeout=15, check=False, + preexec_fn=self._limits, env={"LC_ALL": "C"}) + except Exception: + raise RuntimeError("BLOB_FORMAT_NOT_ALLOWED") from None + if completed.returncode != 0: + raise RuntimeError("BLOB_FORMAT_NOT_ALLOWED") + return completed.stdout if output else b"" + + def verify(self, stream, extension: str, maximum_bytes: int) -> BlobVerifiedContent: + if extension not in _MIME: + raise RuntimeError("BLOB_FORMAT_NOT_ALLOWED") + if type(maximum_bytes) is not int or maximum_bytes < 1: + raise RuntimeError("BLOB_POLICY_INVALID") + if not self._permits.acquire(blocking=False): + raise RuntimeError("BLOB_VERIFICATION_BUSY") + try: + maximum = min(maximum_bytes, self.maximum_input_bytes) + chunks = [] + digest = sha256() + size = 0 + while True: + try: + block = stream.read(65536) + except Exception: + raise RuntimeError("BLOB_STORE_FAILED") from None + if not isinstance(block, bytes): + raise RuntimeError("BLOB_STORE_FAILED") + if not block: + break + if len(block) > maximum - size: + raise RuntimeError("BLOB_SIZE_EXCEEDED") + chunks.append(block) + digest.update(block) + size += len(block) + data = b"".join(chunks) + if extension == "mp3": + valid = _complete_mp3(data, self.maximum_frames) + demuxer = "mp3" + else: + valid = _complete_mp4(data, 1000) + demuxer = "mov" + if not valid: + raise RuntimeError("BLOB_FORMAT_NOT_ALLOWED") + self.validate_ready() + probe = self._run(self.ffprobe_path, ( + "-v", "error", "-protocol_whitelist", "pipe", "-f", demuxer, + "-count_frames", "-show_entries", + "stream=codec_type,codec_name,nb_read_frames,width,height,sample_rate:format=format_name", + "-of", "json", "-i", "pipe:0"), data, True) + if len(probe) > 65536: + raise RuntimeError("BLOB_FORMAT_NOT_ALLOWED") + try: + result = loads(probe) + streams = result["streams"] + if not isinstance(streams, list) or not 1 <= len(streams) <= 8: + raise ValueError("invalid streams") + if extension == "mp3": + if (result["format"]["format_name"] != "mp3" or len(streams) != 1 + or streams[0]["codec_type"] != "audio" + or streams[0]["codec_name"] != "mp3" + or not 8000 <= int(streams[0]["sample_rate"]) <= 192000 + or not 1 <= int(streams[0]["nb_read_frames"]) <= self.maximum_frames): + raise ValueError("invalid MP3") + maps = ("-map", "0:a:0") + else: + videos = [item for item in streams if item["codec_type"] == "video"] + if ("mp4" not in result["format"]["format_name"] + or not 1 <= len(videos) <= 4 + or any(item["codec_type"] not in ("video", "audio") for item in streams) + or any(not 1 <= int(item["width"]) * int(item["height"]) <= self.maximum_video_pixels + or not 1 <= int(item["nb_read_frames"]) <= self.maximum_frames + for item in videos)): + raise ValueError("invalid MP4") + maps = ("-map", "0:v", "-map", "0:a?") + except Exception: + raise RuntimeError("BLOB_FORMAT_NOT_ALLOWED") from None + self._run(self.ffmpeg_path, ( + "-nostdin", "-v", "error", "-xerror", "-err_detect", "explode", + "-threads", "1", "-protocol_whitelist", "pipe", "-f", demuxer, + "-i", "pipe:0", *maps, "-f", "null", "-"), data, False) + return BlobVerifiedContent(size, digest.hexdigest(), _MIME[extension]) + finally: + self._permits.release() diff --git a/tests/fixtures/reference-modules/sample.mp3 b/tests/fixtures/reference-modules/sample.mp3 new file mode 100644 index 0000000000000000000000000000000000000000..99129a8c092ecc703843fd12d33f19f656cf7979 GIT binary patch literal 2107 zcmeZtF=k-^0p*b3U{@f`&%nU!lUSB!YN2Otre|ni0G5Ri|9?9iK;lA}o_T5cKo(FR z11AFm2Oon00(c=n3Ifz2zzhV~gaBs{;28q^M*xt`Ku0D69cgF`;=q6k^BD+}gRehu z%_lJX|L=e!49o`@7?%LK3JeS;4Gat{K90Vw#vt#75)a7HL%iuktNi~v=tyicntzDa xjjV7Gd1&FNhmd0i#vY9!SV&Np2IZmAB!@Ljhh37x>Pz|yQZLET-|bj!1OR5+t<3-c literal 0 HcmV?d00001 diff --git a/tests/fixtures/reference-modules/sample.mp4 b/tests/fixtures/reference-modules/sample.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..40d9e28d7b27c68aec1a6ff5f19d7a365377c201 GIT binary patch literal 1597 zcmZuxO^6&t6t3A#LXaFhEEr@16^$U0neCpLovbS@8NI9CA?b?7^EyjYRO`N%4?FKtY2S7rcasD1y1}#f;zU+1Xv!JhuA1_r34E`gw1M zF~<9TY;vbd#^xAlthN}YVQIP{WAkCD^^h@El*3FyACdV#;M`*kMt_I$9|!O;f9;U@ zBx?@mc-vbsK=2(usP~*dLY!5*cLbZ7v&bxP%wHp%?=QwKDCr%%MO8QtO+p#dx^-^j z=ThbZ7%%1VyT1tdG<)+d6l2Xy7PhwV8Rk;WY|_`^>kU!KLJ_C&UYS>Y7z@L)E?RFz zM92j>jV(&6h(kj0GuyV{Q@io1^GN~wTj$-ZA;h^uO@k&mT62nU#@Pl;S|tBxO!33- z*~tV>7gg6Al07wfz@FKJ{$ls_;=~#TgW+>>{_cnBW`mu|{(AWRbKm{5w@3L;8{rHz z?LliH@pk`-?YoT8VrQMNpC4w;LEBG|y?|R$TS#X4i@?$;bbx_h+Z4O8A0tKM^Aq$L zEYwOhfe_T17tqPJ^rYV^Rjny@{%HHqax09X7e0_?KAn*lKJ%Iio|B?ZrUSA!#IV1- z(z?`Zg+T!Sbl)lrz}#;oSkQM)ot#WQ|MkkB-~7J2@%6_Sf8o2&|NUsx?sa*Sr`jqW z_Bz~lf?mLvWzrL(hrmV)gr%$3uDsUl^2^uP!IUaRU|pMWq57V;gP_xF2kj7G=Dk^6 zTH4v!Y0>!ULR2knHP+!(3&##TmQD?k!Hyv#>R##AvxMr=j3siLsQ)6D8p z#7GUfx2h;|2Ww@tB2y0|-72F1M-cLcu2j?xFNPuS3+H3w2Du@|w7`~$b-(YFkD6`n zGm8u=RA@aA8JzeKb48xwEHfbBm96(k^HO*^M_zel3jq@Fq}aA%9H+W8!oyGTMELc| zE0h4y3gYywD3!xwC9xU9&SlhwEu=7XizH5R;b@cGs`_3#D&Nd}f|xd{iZ``EbP5`z z4%8U6qxN#JZ^rmdh{AxoR8=b7_E9&WV_HE^YL&~7ZqxYSx)ir*l-fQo@z15`T5zIv zrxoxmx=a+bFi%^C3fEeUqTb5@oL465^4y?F7zp$bwZs Date: Wed, 28 Oct 2026 07:40:06 +0800 Subject: [PATCH 46/75] Reject audio-only MP4 and gate real media S3 presentation --- packages/teaql-blob-reference/README.md | 2 +- .../src/teaql_blob_reference/content_ffmpeg.py | 4 +++- .../provider/sqlite/test_blob_content_ffmpeg.py | 12 ++++++++++++ tests/provider/sqlite/test_blob_upload.py | 17 ++++++++++++++++- 4 files changed, 32 insertions(+), 3 deletions(-) diff --git a/packages/teaql-blob-reference/README.md b/packages/teaql-blob-reference/README.md index a3673259..44a7d651 100644 --- a/packages/teaql-blob-reference/README.md +++ b/packages/teaql-blob-reference/README.md @@ -62,6 +62,6 @@ allows only the `pipe` protocol and bounds input bytes, frame count, video pixels, concurrent checks, child CPU time and address space, and wall time. The accepted MP3 profile is MPEG Layer III with fixed, non-free bitrate and a constant sample rate; MP4 must contain a supported `ftyp` brand, `moov`, -`mdat`, and at least one decodable video stream. The adapter returns the SHA-256 +`mdat`, and at least one decodable non-cover-art video stream. The adapter returns the SHA-256 of the original bytes and `audio/mpeg` or `video/mp4`. It is content validation, not malware scanning; ffprobe/ffmpeg themselves must be kept current. diff --git a/packages/teaql-blob-reference/src/teaql_blob_reference/content_ffmpeg.py b/packages/teaql-blob-reference/src/teaql_blob_reference/content_ffmpeg.py index 1f032a03..0997d5da 100644 --- a/packages/teaql-blob-reference/src/teaql_blob_reference/content_ffmpeg.py +++ b/packages/teaql-blob-reference/src/teaql_blob_reference/content_ffmpeg.py @@ -195,7 +195,7 @@ def verify(self, stream, extension: str, maximum_bytes: int) -> BlobVerifiedCont probe = self._run(self.ffprobe_path, ( "-v", "error", "-protocol_whitelist", "pipe", "-f", demuxer, "-count_frames", "-show_entries", - "stream=codec_type,codec_name,nb_read_frames,width,height,sample_rate:format=format_name", + "stream=codec_type,codec_name,nb_read_frames,width,height,sample_rate:stream_disposition=attached_pic:format=format_name", "-of", "json", "-i", "pipe:0"), data, True) if len(probe) > 65536: raise RuntimeError("BLOB_FORMAT_NOT_ALLOWED") @@ -217,6 +217,8 @@ def verify(self, stream, extension: str, maximum_bytes: int) -> BlobVerifiedCont if ("mp4" not in result["format"]["format_name"] or not 1 <= len(videos) <= 4 or any(item["codec_type"] not in ("video", "audio") for item in streams) + or any(item.get("disposition", {}).get("attached_pic", 0) != 0 + for item in videos) or any(not 1 <= int(item["width"]) * int(item["height"]) <= self.maximum_video_pixels or not 1 <= int(item["nb_read_frames"]) <= self.maximum_frames for item in videos)): diff --git a/tests/provider/sqlite/test_blob_content_ffmpeg.py b/tests/provider/sqlite/test_blob_content_ffmpeg.py index e4df2598..7dfa5326 100644 --- a/tests/provider/sqlite/test_blob_content_ffmpeg.py +++ b/tests/provider/sqlite/test_blob_content_ffmpeg.py @@ -4,6 +4,7 @@ from io import BytesIO from pathlib import Path from shutil import which +from subprocess import DEVNULL, PIPE, run from threading import Event, Thread import os @@ -94,6 +95,17 @@ def read(self, size=-1): assert result[0].content_type == "audio/mpeg" +def test_audio_only_mp4_is_not_a_video(verifier): + # Remux the shared MP3 into a structurally valid MP4 with no video track. + media = run((verifier.ffmpeg_path, "-nostdin", "-v", "error", "-f", "mp3", + "-i", "pipe:0", "-vn", "-c:a", "copy", "-movflags", + "frag_keyframe+empty_moov", "-f", "mp4", "pipe:1"), + input=media_fixture("mp3"), stdout=PIPE, stderr=DEVNULL, + timeout=10, check=True).stdout + with pytest.raises(RuntimeError, match="BLOB_FORMAT_NOT_ALLOWED"): + verifier.verify(BytesIO(media), "mp4", len(media)) + + class MediaAuthority: def scope(self, _context): return "tenant-a" diff --git a/tests/provider/sqlite/test_blob_upload.py b/tests/provider/sqlite/test_blob_upload.py index 90c3acd0..ef451988 100644 --- a/tests/provider/sqlite/test_blob_upload.py +++ b/tests/provider/sqlite/test_blob_upload.py @@ -4,6 +4,7 @@ from hashlib import sha256 from io import BytesIO from pathlib import Path +from shutil import which from threading import Event from urllib.request import Request, urlopen from uuid import uuid4 @@ -86,7 +87,7 @@ def verify(self, stream, extension, maximum_bytes): @pytest.mark.asyncio @pytest.mark.parametrize("case", ["fixture-pdf", "real-jpeg", "real-pdf", - "real-docx", "real-xlsx"]) + "real-docx", "real-xlsx", "real-mp3", "real-mp4"]) async def test_upload_real_sqlite_s3_create_only_confirm_and_replay(tmp_path, case): if os.environ.get("TEAQL_S3_TEST_BLOB_UPLOAD") != "1": pytest.skip("explicit S3-compatible Blob upload gate required") @@ -96,6 +97,8 @@ async def test_upload_real_sqlite_s3_create_only_confirm_and_replay(tmp_path, ca pytest.skip("explicit production PDF S3 gate required") if case in ("real-docx", "real-xlsx") and os.environ.get("TEAQL_S3_TEST_REAL_OFFICE") != "1": pytest.skip("explicit production Office S3 gate required") + if case in ("real-mp3", "real-mp4") and os.environ.get("TEAQL_S3_TEST_REAL_MEDIA") != "1": + pytest.skip("explicit production media S3 gate required") endpoint = os.environ.get("TEAQL_S3_TEST_ENDPOINT") bucket = os.environ.get("TEAQL_S3_TEST_BUCKET") access = os.environ.get("TEAQL_S3_TEST_ACCESS_KEY") @@ -145,6 +148,18 @@ async def test_upload_real_sqlite_s3_create_only_confirm_and_replay(tmp_path, ca workbook.save(output) content_type = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" data, file_name = output.getvalue(), "sample." + extension + elif case in ("real-mp3", "real-mp4"): + from teaql_blob_reference.content_ffmpeg import FfmpegBlobContentVerifier + ffprobe, ffmpeg = which("ffprobe"), which("ffmpeg") + if not ffprobe or not ffmpeg: + pytest.skip("host FFmpeg command-line tools required") + extension = case.removeprefix("real-") + authority = UploadAuthority((extension,)) + parser = FfmpegBlobContentVerifier(ffprobe, ffmpeg) + data = (Path(__file__).parents[2] / "fixtures" / "reference-modules" / + ("sample." + extension)).read_bytes() + file_name = "sample." + extension + content_type = "audio/mpeg" if extension == "mp3" else "video/mp4" else: authority, parser = UploadAuthority(), FixturePDFVerifier() data = b"%PDF-1.7\nreal S3 Blob upload\n%%EOF" From f466b029788c9e5de6572b666b1863e9cffb8e05 Mon Sep 17 00:00:00 2001 From: Philip Z Date: Wed, 28 Oct 2026 09:03:08 +0800 Subject: [PATCH 47/75] feat: compose optional Blob content verifiers by format --- packages/teaql-blob-reference/README.md | 7 +++ .../src/teaql_blob_reference/__init__.py | 4 +- .../teaql_blob_reference/content_composite.py | 42 ++++++++++++++++++ .../sqlite/test_blob_content_composite.py | 43 +++++++++++++++++++ 4 files changed, 95 insertions(+), 1 deletion(-) create mode 100644 packages/teaql-blob-reference/src/teaql_blob_reference/content_composite.py create mode 100644 tests/provider/sqlite/test_blob_content_composite.py diff --git a/packages/teaql-blob-reference/README.md b/packages/teaql-blob-reference/README.md index 44a7d651..c435d9e5 100644 --- a/packages/teaql-blob-reference/README.md +++ b/packages/teaql-blob-reference/README.md @@ -4,6 +4,13 @@ Optional Blob catalog and current-read services for TeaQL Python. Install this w `SqliteBlobUploadService` offers a host-invoked, scope-bound upload journal. The host must provide current model/action authorization and a trusted parser that consumes every stored byte and returns actual format, size, and SHA256. The service writes through the context's object-storage provider, rechecks the stored generation before confirmation, and promotes the row to `uploaded_unbound` only after full verification. The included PDF verifier in tests is a fixture, not a production format parser. Other SQL backends and production ten-format parsing remain open. +To use several optional parsers in one upload service, pass a +`CompositeBlobContentVerifier` with explicit extension-to-parser routes. It +copies and validates the route table at construction, calls each distinct +parser's readiness check once, and rejects unregistered formats before reading +stored bytes. It does not import optional decoder packages or add formats by +itself; install only the image/PDF/Office/media extras the host selects. + For retention-aware uploads, the host may implement `current_policy_on_lease(actor, lease, target)` on its upload authorization and return an immutable `BlobRetentionPolicy`. The service obtains this trusted policy on the physical SQLite begin lease and snapshots its profile, upload mode and retention durations into the catalog. It rejects direct mode on this server-stream path. Confirmation records `completed_at_ms`; replay never changes the accepted policy or completion time. Older rows remain NULL-policy after migration and are excluded from retention collection. A binding authorization must provide its own current policy on the graph lease; new bindings of unknown-policy rows fail closed. Successful binding preserves the longest accepted unreferenced duration and minimum retention deadline. The final unbind records `last_unbound_at_ms` while the row remains `available`, so its lifetime is measured from the actual final unbind. The shared policy vectors are pinned by SHA-256 in the tests. `SqliteBlobMaintenanceService` is an optional, explicitly host-scheduled service for expired incomplete uploads and policy-backed completed, unbound server-stream objects. It includes formerly bound objects only after the final unbind deadline and retention floor. Active bindings, unknown-policy legacy rows and direct uploads are excluded. The host supplies a current scope and async `can_maintain` policy, then calls `run_batch(context, cursor)` and may install the service through `BlobReferenceModule(..., maintenance=service)`. Run `catalog.ensure_schema()` as a separate migration before readiness; it adds a unique object-key index and nullable policy/time columns. Cleanup claims rows under SQLite `BEGIN IMMEDIATE`, pins the observed ETag, uses only version-conditional storage deletion, and can resume after a failed final catalog acknowledgment. `FileObjectStorage` guards manifest publication and conditional deletion with a persistent per-key OS file lock. Its ETag includes an immutable generation token, so identical-byte replacement cannot match an older version. A matching delete removes the current data generation and manifest. The host may separately call `FileObjectStorage.sweep_old_generations(key)` for a known object key to remove inaccessible older immutable data generations; it waits for same-key writers and preserves the current manifest generation. Use this only after all processes using that private root run a writer-lock-capable version, and schedule/retry it separately from the catalog receipt. Keep each Blob object key unique and immutable, including across manual S3 writes. Direct-upload quiescence and cross-machine S3 retention remain outside this service. diff --git a/packages/teaql-blob-reference/src/teaql_blob_reference/__init__.py b/packages/teaql-blob-reference/src/teaql_blob_reference/__init__.py index 45d873c6..3d7c1ba1 100644 --- a/packages/teaql-blob-reference/src/teaql_blob_reference/__init__.py +++ b/packages/teaql-blob-reference/src/teaql_blob_reference/__init__.py @@ -12,6 +12,7 @@ BlobFileGenerationSweepOutcome, SqliteBlobFileGenerationSweepService) from .retention import BlobRetentionPolicy +from .content_composite import CompositeBlobContentVerifier from .upload_service import (BlobUploadInput, BlobUploadSession, BlobUploadTarget, BlobVerifiedContent, SqliteBlobUploadService) @@ -22,4 +23,5 @@ "SqliteBlobFileGenerationSweepService", "SqliteBlobCatalog", "SqliteBlobResponseContributor", "FileObjectStorage", "BlobUploadInput", "BlobUploadSession", "BlobUploadTarget", - "BlobVerifiedContent", "SqliteBlobUploadService", "BlobRetentionPolicy"] + "BlobVerifiedContent", "SqliteBlobUploadService", "BlobRetentionPolicy", + "CompositeBlobContentVerifier"] diff --git a/packages/teaql-blob-reference/src/teaql_blob_reference/content_composite.py b/packages/teaql-blob-reference/src/teaql_blob_reference/content_composite.py new file mode 100644 index 00000000..85c291be --- /dev/null +++ b/packages/teaql-blob-reference/src/teaql_blob_reference/content_composite.py @@ -0,0 +1,42 @@ +"""Passive host-selected routes for independently optional Blob parsers.""" + +from __future__ import annotations + +from collections.abc import Mapping + +from .upload_service import BlobContentVerifier, BlobVerifiedContent + + +class CompositeBlobContentVerifier: + """Connect multiple parsers to the single trusted upload verifier SPI.""" + + def __init__(self, routes: Mapping[str, BlobContentVerifier]): + if not isinstance(routes, Mapping): + raise RuntimeError("BLOB_POLICY_INVALID") + selected = dict(routes) + if (not 1 <= len(selected) <= 32 + or any(not isinstance(extension, str) or not 1 <= len(extension) <= 16 + or not extension.isascii() or not extension.isalnum() + or extension != extension.lower() or verifier is None + or not callable(getattr(verifier, "verify", None)) + for extension, verifier in selected.items())): + raise RuntimeError("BLOB_POLICY_INVALID") + self._routes = selected + self.supported_extensions = frozenset(self._routes) + + def validate_ready(self) -> None: + seen: set[int] = set() + for verifier in self._routes.values(): + identity = id(verifier) + if identity in seen: + continue + seen.add(identity) + ready = getattr(verifier, "validate_ready", None) + if callable(ready): + ready() + + def verify(self, stream, extension: str, maximum_bytes: int) -> BlobVerifiedContent: + verifier = self._routes.get(extension) + if verifier is None: + raise RuntimeError("BLOB_FORMAT_NOT_ALLOWED") + return verifier.verify(stream, extension, maximum_bytes) diff --git a/tests/provider/sqlite/test_blob_content_composite.py b/tests/provider/sqlite/test_blob_content_composite.py new file mode 100644 index 00000000..8b779ed3 --- /dev/null +++ b/tests/provider/sqlite/test_blob_content_composite.py @@ -0,0 +1,43 @@ +"""One upload verifier dispatches real optional image and media parsers.""" + +from hashlib import sha256 +from io import BytesIO +from pathlib import Path +from shutil import which + +import pytest + +from teaql_blob_reference import CompositeBlobContentVerifier +from teaql_blob_reference.content_ffmpeg import FfmpegBlobContentVerifier +from teaql_blob_reference.content_pillow import PillowBlobContentVerifier + + +class Unreadable: + def read(self, *_): + raise AssertionError("unregistered route read source") + + +def test_real_image_and_media_share_trusted_upload_verifier(): + ffprobe, ffmpeg = which("ffprobe"), which("ffmpeg") + if not ffprobe or not ffmpeg: + pytest.skip("host FFmpeg tools required") + image = PillowBlobContentVerifier() + media = FfmpegBlobContentVerifier(ffprobe, ffmpeg) + routes = {"jpg": image, "jpeg": image, "png": image, "mp3": media, "mp4": media} + verifier = CompositeBlobContentVerifier(routes) + routes.clear() + assert verifier.supported_extensions == {"jpg", "jpeg", "png", "mp3", "mp4"} + verifier.validate_ready() + fixture = Path(__file__).resolve().parents[2] / "fixtures" + for extension, path, mime in ( + ("jpg", fixture / "blob-sample.jpg", "image/jpeg"), + ("mp3", fixture / "reference-modules" / "sample.mp3", "audio/mpeg"), + ): + data = path.read_bytes() + result = verifier.verify(BytesIO(data), extension, len(data)) + assert (result.size_bytes, result.sha256, result.content_type) == ( + len(data), sha256(data).hexdigest(), mime) + with pytest.raises(RuntimeError, match="BLOB_FORMAT_NOT_ALLOWED"): + verifier.verify(Unreadable(), "docx", 1024) + with pytest.raises(RuntimeError, match="BLOB_POLICY_INVALID"): + CompositeBlobContentVerifier({"PDF": image}) From 9c546e13e379a1ee8b867f46fd08e93a7ab0109a Mon Sep 17 00:00:00 2001 From: Philip Z Date: Wed, 28 Oct 2026 12:04:09 +0800 Subject: [PATCH 48/75] Add sandboxed DOC and XLS verifier to Python Blob wheel --- .gitattributes | 2 + packages/teaql-blob-reference/README.md | 15 +- .../content_legacy_office.py | 192 ++++++++++++++++++ tests/fixtures/reference-modules/sample.doc | Bin 0 -> 9728 bytes tests/fixtures/reference-modules/sample.xls | Bin 0 -> 5632 bytes .../sqlite/test_blob_content_legacy_office.py | 161 +++++++++++++++ tests/provider/sqlite/test_blob_upload.py | 22 +- 7 files changed, 389 insertions(+), 3 deletions(-) create mode 100644 packages/teaql-blob-reference/src/teaql_blob_reference/content_legacy_office.py create mode 100644 tests/fixtures/reference-modules/sample.doc create mode 100644 tests/fixtures/reference-modules/sample.xls create mode 100644 tests/provider/sqlite/test_blob_content_legacy_office.py diff --git a/.gitattributes b/.gitattributes index d4a292f1..a5a4a7b3 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1 +1,3 @@ test-vectors/*.tsv whitespace=-blank-at-eol +tests/fixtures/reference-modules/*.doc binary +tests/fixtures/reference-modules/*.xls binary diff --git a/packages/teaql-blob-reference/README.md b/packages/teaql-blob-reference/README.md index c435d9e5..84e2426d 100644 --- a/packages/teaql-blob-reference/README.md +++ b/packages/teaql-blob-reference/README.md @@ -54,8 +54,19 @@ Office main content type and package relationship, then parses the Word body or every Excel worksheet under an element/cell count limit. It hashes original bytes and returns the precise Office MIME type. Construction is passive, and `validate_ready` requires both optional parsers. The in-process parsers do not -provide hard CPU or process-memory isolation. Binary DOC/XLS still need -production adapters. +provide hard CPU or process-memory isolation. + +For binary DOC/XLS on Linux, inject +`teaql_blob_reference.content_legacy_office.LegacyOfficeBlobContentVerifier` +with absolute paths to patched LibreOffice (`soffice`), `bwrap`, `prlimit` and +an owner-only (`0700`) canonical scratch directory. Install the `office` extra +for its converted OOXML verifier. The adapter checks original CFB header and +sector layout, converts each file in a network/PID-isolated bubblewrap +namespace with read-only system mounts and only one private writable scratch, +then fully validates the DOCX/XLSX output. CPU, address-space, file-size, +wall-time, input and concurrency limits apply. It returns original DOC/XLS +SHA, size and MIME, so storage keeps the original bytes. This is bounded +decodability verification, not malware or macro detection. For actual MP3/MP4 validation on POSIX hosts, inject `teaql_blob_reference.content_ffmpeg.FfmpegBlobContentVerifier` with absolute diff --git a/packages/teaql-blob-reference/src/teaql_blob_reference/content_legacy_office.py b/packages/teaql-blob-reference/src/teaql_blob_reference/content_legacy_office.py new file mode 100644 index 00000000..1be708a7 --- /dev/null +++ b/packages/teaql-blob-reference/src/teaql_blob_reference/content_legacy_office.py @@ -0,0 +1,192 @@ +"""Optional Linux sandboxed binary DOC/XLS verifier for Blob uploads.""" + +from __future__ import annotations + +from hashlib import sha256 +from io import BytesIO +from pathlib import Path +from tempfile import TemporaryDirectory +from threading import BoundedSemaphore +import os +import signal +import stat +import subprocess +import sys + +from .content_office import OfficeBlobContentVerifier +from .upload_service import BlobVerifiedContent + + +_MIME = {"doc": "application/msword", "xls": "application/vnd.ms-excel"} +_MAGIC = bytes.fromhex("d0cf11e0a1b11ae1") + + +def _u16(data: bytes, offset: int) -> int: + return int.from_bytes(data[offset:offset + 2], "little") + + +def _u32(data: bytes, offset: int) -> int: + return int.from_bytes(data[offset:offset + 4], "little") + + +def _valid_container(data: bytes) -> bool: + if (len(data) < 1024 or not data.startswith(_MAGIC) + or _u16(data, 28) != 0xfffe or _u16(data, 32) != 6): + return False + major, shift = _u16(data, 26), _u16(data, 30) + if not ((major, shift) in ((3, 9), (4, 12))): + return False + sector_size = 1 << shift + if len(data) % sector_size or len(data) < 2 * sector_size: + return False + sectors = len(data) // sector_size - 1 + fat_count, directory = _u32(data, 44), _u32(data, 48) + mini_fat, mini_count = _u32(data, 60), _u32(data, 64) + difat, difat_count = _u32(data, 68), _u32(data, 72) + return (0 < fat_count <= sectors and directory < sectors + and _u32(data, 56) == 4096 and mini_count <= sectors + and (not mini_count or mini_fat < sectors) + and difat_count <= sectors and (not difat_count or difat < sectors)) + + +class LegacyOfficeBlobContentVerifier: + """Host-selected, passive DOC/XLS adapter; the host provisions sandbox tools.""" + + supported_extensions = frozenset(_MIME) + + def __init__(self, soffice_path: str, bwrap_path: str, prlimit_path: str, + scratch_root: str | Path, maximum_input_bytes: int = 32 * 1024 * 1024, + maximum_converted_bytes: int = 10 * 1024 * 1024, + concurrent_verifications: int = 2): + paths = (soffice_path, bwrap_path, prlimit_path, str(scratch_root)) + if (any(not isinstance(path, str) or not os.path.isabs(path) for path in paths) + or os.path.normpath(paths[3]) != paths[3] + or type(maximum_input_bytes) is not int + or not 1 <= maximum_input_bytes <= 100 * 1024 * 1024 + or type(maximum_converted_bytes) is not int + or not 1 <= maximum_converted_bytes <= 10 * 1024 * 1024 + or type(concurrent_verifications) is not int + or not 1 <= concurrent_verifications <= 32): + raise RuntimeError("BLOB_POLICY_INVALID") + self.soffice_path, self.bwrap_path, self.prlimit_path = paths[:3] + self.scratch_root = Path(paths[3]) + self.maximum_input_bytes = maximum_input_bytes + self.maximum_converted_bytes = maximum_converted_bytes + self._permits = BoundedSemaphore(concurrent_verifications) + self._package = OfficeBlobContentVerifier( + maximum_input_bytes=maximum_converted_bytes, + maximum_decoded_bytes=100 * 1024 * 1024, + maximum_parts=10000, maximum_elements=2_000_000, + concurrent_verifications=concurrent_verifications) + + def validate_ready(self) -> None: + if sys.platform != "linux": + raise RuntimeError("BLOB_CAPABILITY_MISSING") + try: + for executable in (self.soffice_path, self.bwrap_path, self.prlimit_path): + if not Path(executable).is_file() or not os.access(executable, os.X_OK): + raise ValueError("missing converter") + root = self.scratch_root.lstat() + if (not stat.S_ISDIR(root.st_mode) or stat.S_IMODE(root.st_mode) != 0o700 + or root.st_uid != os.geteuid() + or self.scratch_root.resolve() != self.scratch_root): + raise ValueError("unsafe scratch root") + self._package.validate_ready() + except Exception: + raise RuntimeError("BLOB_CAPABILITY_MISSING") from None + + def _convert(self, data: bytes, extension: str) -> bytes: + try: + with TemporaryDirectory(prefix="legacy-office-", dir=self.scratch_root) as name: + job = Path(name) + output, profile = job / "output", job / "profile" + output.mkdir(mode=0o700) + profile.mkdir(mode=0o700) + source = job / ("source." + extension) + descriptor = os.open(source, os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o600) + with os.fdopen(descriptor, "wb") as writer: + writer.write(data) + args = [self.prlimit_path, "--cpu=20:20", + "--as=1073741824:1073741824", "--fsize=10485760:10485760", + "--", self.bwrap_path, + "--unshare-net", "--unshare-pid", "--die-with-parent", + "--ro-bind", "/usr", "/usr", "--ro-bind", "/lib", "/lib", + "--ro-bind", "/lib64", "/lib64", "--symlink", "usr/bin", "/bin", + "--ro-bind", "/etc", "/etc", "--dev", "/dev", "--proc", "/proc", + "--dir", "/tmp", "--bind", name, name, "--clearenv", + "--setenv", "LC_ALL", "C", "--setenv", "PATH", "/usr/bin:/bin", + "--setenv", "HOME", name, "--setenv", "TMPDIR", name, + "--", self.soffice_path, "-env:UserInstallation=" + profile.as_uri(), + "--headless", "--nodefault", "--norestore", "--nolockcheck", + "--convert-to", extension + "x", "--outdir", str(output), str(source)] + try: + process = subprocess.Popen(args, stdin=subprocess.DEVNULL, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, cwd="/", + env={"LC_ALL": "C", "PATH": "/usr/bin:/bin"}, + start_new_session=True) + except OSError: + raise RuntimeError("BLOB_CAPABILITY_MISSING") from None + try: + code = process.wait(timeout=30) + except subprocess.TimeoutExpired: + try: + os.killpg(process.pid, signal.SIGKILL) + except ProcessLookupError: + pass + process.wait() + raise RuntimeError("BLOB_FORMAT_NOT_ALLOWED") from None + if code: + raise RuntimeError("BLOB_FORMAT_NOT_ALLOWED") + entries = list(output.iterdir()) + expected = output / ("source." + extension + "x") + if len(entries) != 1 or entries[0] != expected: + raise RuntimeError("BLOB_FORMAT_NOT_ALLOWED") + info = expected.lstat() + if (not stat.S_ISREG(info.st_mode) or not 1 <= info.st_size <= + self.maximum_converted_bytes): + raise RuntimeError("BLOB_FORMAT_NOT_ALLOWED") + converted = expected.read_bytes() + if len(converted) != info.st_size: + raise RuntimeError("BLOB_FORMAT_NOT_ALLOWED") + return converted + except RuntimeError: + raise + except Exception: + raise RuntimeError("BLOB_STORE_FAILED") from None + + def verify(self, stream, extension: str, maximum_bytes: int) -> BlobVerifiedContent: + if extension not in _MIME: + raise RuntimeError("BLOB_FORMAT_NOT_ALLOWED") + if type(maximum_bytes) is not int or maximum_bytes < 1: + raise RuntimeError("BLOB_POLICY_INVALID") + if not self._permits.acquire(blocking=False): + raise RuntimeError("BLOB_VERIFICATION_BUSY") + try: + self.validate_ready() + maximum = min(maximum_bytes, self.maximum_input_bytes) + chunks, size = [], 0 + while True: + try: + block = stream.read(65536) + except Exception: + raise RuntimeError("BLOB_STORE_FAILED") from None + if not isinstance(block, bytes): + raise RuntimeError("BLOB_STORE_FAILED") + if not block: + break + if len(block) > maximum - size: + raise RuntimeError("BLOB_SIZE_EXCEEDED") + chunks.append(block) + size += len(block) + data = b"".join(chunks) + if not _valid_container(data): + raise RuntimeError("BLOB_FORMAT_NOT_ALLOWED") + converted = self._convert(data, extension) + try: + self._package.verify(BytesIO(converted), extension + "x", len(converted)) + except Exception: + raise RuntimeError("BLOB_FORMAT_NOT_ALLOWED") from None + return BlobVerifiedContent(size, sha256(data).hexdigest(), _MIME[extension]) + finally: + self._permits.release() diff --git a/tests/fixtures/reference-modules/sample.doc b/tests/fixtures/reference-modules/sample.doc new file mode 100644 index 0000000000000000000000000000000000000000..0979c3cdc8182d7d8d7e7cdeb175f70e9d397d96 GIT binary patch literal 9728 zcmeI2%WqUw9LLX{PNy>!%5*wbEGR=63-V~82o&+rmI7sK--hYnZ^>!1_X!xOLpHo_+8g3a(GY=NiX zX?O;ng>LA9Uf2qKuno4u4(NxSkbqsV8wOwx48joXg?+Fe4#0D85Dvlf@B+LDhv5hu zg=3)hzLfKC_TJqOS=6M40bY^=Ae;c; z^rSr}4D452UuM`|6)4=!gZ;j}%lxzXo7bFzO8sJgj?hwxO1z6;arLZ14%;)5lMsD} z|H}Goy$9&645QkZNt)BV);zW@ZdS&v9?Y%7^sY36i5jyPwk8&uvHp;M0)`TPlkWEi zC*fG4)+Bj%ynnN1$l`9hnOb5QpC$XL!+J)MIbu?bt1lBXpBgq@yk?>o&zV5n;_Dbp z1P~lR@M1HRs56eB#n^FgSmnr`Ls-?QIcJa1y|iS+9-qT#JTelsEK20NyZ8YqW2Pf) zSrIEn*pIEI%mlfl-bS8}C4$&{E9JIY-Ne>m@kw)r6e&tg6EmOhxQV){e)Fhb#8$}0 zprx#8|Iwh;X1A~tUn_;37=6mm}$*ovzo?m(a*}GES1> z3^;`(OQUWXCj&S^O~=XSc5;kiN--%qL&H_+Hth-^sf-l8)(18T`K8A&uLOgp;xx4G z^jOwz$~i^7v>@rDuCLqrP4WRH?RqoNqBG0w&JwjN)_(QZ1o`rH>a}+HEp^J7(N4cN zot1Cr@eOy<&H?n822E@kbJo_Kdfv5O^r-Sn=+AvMTbtd|3tQjIKw|~ZGjIIB$8c&! z@lJ#WQ7s27{xnp*Ykvf{#=2RT*O(ZIl&tFRKm(DrcaXVA)`Iaa=CPA*yC!cJk#9#& zvEyK_oRLHZboISY|GqwWD*VNVL9=T4w^tIT791URT|HqJjVO{#Q0sc!xBbnzXs^mT}=M`+#zcGnPS)t6y;< zZ{s(DwaQ*OKyhvZJmz&JS;#%j%BoUVk96IgsZH{|l%x8(Cnf7RhELM+O6$;&lP}qF zUgWE;f9fARaJ3xJpXEwI7a6+y*PTxOO^tgy`ELmW?! zbJEs_KWx`TYhT@4d2=llw7-p_-E*_r8Io^LA6*s5KaPCwEiA7GyW?{|D?qvHfd6|wdrltZPIMS+6OCIvu#?n(ApRy7Afe><|eM$O+tSB z0e`lseGn`teek6qg}jL34?=xVFnRMqQG`Mj9};~k`mhy63QgAUn|n7|*KD#Yp;E~# zbI#m3bMDNUnKS3y{q~!>+4CQ7nUeyzPyDiw4M{~F-vDjqp2H$O(Bv&-v)LRQxRWefudVi3rfZvc`P^7w%kq4Zud= zX5dZ0n}N3gw*a>Ssdt;#leR$b@Sbn;`lXeuQ`f@-Arn!o2+C(0zhcSP+CPVk_$V-= zdGQl?CuD@V(P<~+*7bq|YJ+VsH>Z)L`#>3T4%`~~P`RILyHB$ZLNT%SqT`L^xG$!* zpV78i+Zwcne5!MPP|j}T<(H7pzWS!O34+Fc)W3w-l9Yl`$gS0AQ=nR=c)mX53^}8A za#r`vl>DLl0g4o+E%IrY@_+Z(vNE6ET28(gpKo1G{{PFLf=?|j&x4#l+gv|n3i+)? z_|YQ#T}AkzfAG)xu&2_JWgkxawXaK8XkVWWXum1FLHmYuF#ELT?rG%zBnvfm>X@Ax zNjh&}ZyNZ`7$#7p=Zqo0$djzlnQmQ9MTV7_TbOykoXW>NQk7u^{{nY=Q%)MF_~x_` zLmQK(E3EOT`6g<=iJETyiL!r9w(EJ}of(6Pp*~z2pR}nyX+8&bVizc5-DfQdc=9&>K;znL(y($U?ORcHb-{f9ogR~trA&P zuFD>9tVq`>Rq&d&5g4;a;c~~$Jh}z+srE~g?Uyngt$Fl$X~xmohvo4%DN!S8;DDq=_i1R}ebdP%9UZ1;0uc`I(0?f$7=0JyLa#mM-UgZv;pqFVBlKc{JY@1Lh!>BhVM4^F$WOV)j#?@8Z`N&)vB-eD-bFAHDj) zLGXeFyc6KDx^X>3rdH0@Vtk0KgXe*@{Qgt_7wn Date: Wed, 28 Oct 2026 14:25:25 +0800 Subject: [PATCH 49/75] Support Unicode filenames in signed object URLs --- src/teaql/core/object_storage.py | 18 +++++++++++++++--- tests/runtime/test_object_storage.py | 10 ++++++++++ 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/src/teaql/core/object_storage.py b/src/teaql/core/object_storage.py index d5137367..13cd51ba 100644 --- a/src/teaql/core/object_storage.py +++ b/src/teaql/core/object_storage.py @@ -4,6 +4,7 @@ from enum import Enum from types import MappingProxyType from typing import BinaryIO, Mapping, Optional +from urllib.parse import quote import unicodedata MAX_PUT_SIZE = 5 * 1024**3 @@ -116,14 +117,25 @@ def __post_init__(self): if self.disposition not in (None, "inline", "attachment"): raise StorageError(StorageErrorCode.INVALID_INPUT) if self.filename is not None: - _header(self.filename) - if self.disposition is None or any(c in self.filename for c in '/\\"'): + try: + valid_length = 0 < len(self.filename.encode("utf-8")) <= 1024 + except (AttributeError, UnicodeEncodeError): + valid_length = False + if (self.disposition is None or not valid_length or + any(c in '/\\"' or ord(c) < 32 or 127 <= ord(c) <= 159 + for c in self.filename)): raise StorageError(StorageErrorCode.INVALID_INPUT) def content_disposition(self): if self.disposition is None: return None - return self.disposition + (f'; filename="{self.filename}"' if self.filename is not None else "") + if self.filename is None: + return self.disposition + if all(ord(c) <= 126 for c in self.filename): + return f'{self.disposition}; filename="{self.filename}"' + fallback = "".join(c if ord(c) <= 126 else "_" for c in self.filename) + return (f'{self.disposition}; filename="{fallback}"; ' + f"filename*=UTF-8''{quote(self.filename, safe='')}") def __repr__(self): return "ReadUrlOptions[details=masked]" diff --git a/tests/runtime/test_object_storage.py b/tests/runtime/test_object_storage.py index bb38fd0c..907b1372 100644 --- a/tests/runtime/test_object_storage.py +++ b/tests/runtime/test_object_storage.py @@ -99,6 +99,16 @@ def test_object_storage_options_and_private_representations(): for seconds in (0, -1, 0.5, 604801): assert_error(StorageErrorCode.INVALID_INPUT, lambda: ReadUrlOptions(timedelta(seconds=seconds))) assert_error(StorageErrorCode.INVALID_INPUT, lambda: ReadUrlOptions(filename="bad\nname", disposition="attachment")) + assert_error(StorageErrorCode.INVALID_INPUT, lambda: ReadUrlOptions(filename="bad\r\nname", disposition="attachment")) + assert_error(StorageErrorCode.INVALID_INPUT, lambda: ReadUrlOptions(filename="bad/name", disposition="attachment")) + assert_error(StorageErrorCode.INVALID_INPUT, lambda: ReadUrlOptions(filename="\ud800", disposition="attachment")) + assert ReadUrlOptions(disposition="inline", filename="sample.jpg").content_disposition() == ( + 'inline; filename="sample.jpg"') + unicode_disposition = ReadUrlOptions( + disposition="inline", filename="报告 & 📄\u2028mid\u2029x.jpg").content_disposition() + assert unicode_disposition.startswith('inline; filename="__ & _') + assert "filename*=UTF-8''%E6%8A%A5%E5%91%8A%20%26%20%F0%9F%93%84" in unicode_disposition + assert "\r" not in unicode_disposition and "\n" not in unicode_disposition assert "CANARY" not in repr(ObjectKey("PRIVATE-KEY-CANARY")) assert "CANARY" not in repr(ObjectReadUrl("PRIVATE-URL-CANARY", None)) options = PutOptions(0, WriteMode.CREATE_ONLY, metadata={"tag": "PRIVATE-METADATA-CANARY"}) From 0fa02653260d7b81dc1d9df12ea2a35c31925dd6 Mon Sep 17 00:00:00 2001 From: Philip Z Date: Wed, 28 Oct 2026 15:42:03 +0800 Subject: [PATCH 50/75] Canonicalize document selection digest with legacy replay --- .../sqlite_document_service.py | 19 +++++++++++++++++-- .../tests/test_sqlite_document_service.py | 13 +++++++++++++ 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/packages/teaql-secure-document-reference/src/teaql_secure_document_reference/sqlite_document_service.py b/packages/teaql-secure-document-reference/src/teaql_secure_document_reference/sqlite_document_service.py index 236351f4..39c386c2 100644 --- a/packages/teaql-secure-document-reference/src/teaql_secure_document_reference/sqlite_document_service.py +++ b/packages/teaql-secure-document-reference/src/teaql_secure_document_reference/sqlite_document_service.py @@ -138,7 +138,11 @@ def _prepared(command: SubmittedDocumentCommand) -> tuple[str, SubmittedDocument # Keep the original digest for receipts written without selections. parts = [command.business_id, command.document_token, command.command_id, copied] if selections is not None: - parts.append([selection.__dict__ for selection in selections]) + parts.append([{"canonicalField": selection.canonical_field, + "kind": selection.kind, + "targetEntity": selection.target_entity, + "targetKey": selection.target_key, + "token": selection.token} for selection in selections]) encoded = json.dumps(parts, ensure_ascii=False, separators=(",", ":"), sort_keys=True).encode("utf-8") if len(encoded) > _MAX_BYTES: @@ -147,6 +151,15 @@ def _prepared(command: SubmittedDocumentCommand) -> tuple[str, SubmittedDocument command.business_id, command.document_token, command.command_id, copied, selections) +def _legacy_selection_hash(command: SubmittedDocumentCommand) -> str: + """Recognize receipts written before the shared selection-key encoding.""" + parts = [command.business_id, command.document_token, command.command_id, + command.body, [selection.__dict__ for selection in command.blob_selections or ()]] + encoded = json.dumps(parts, ensure_ascii=False, separators=(",", ":"), + sort_keys=True).encode("utf-8") + return hashlib.sha256(encoded).hexdigest() + + def _receipt(value: DocumentCommandReceipt) -> DocumentCommandReceipt: if (not isinstance(value, DocumentCommandReceipt) or type(value.previous_revision) is not int or @@ -232,7 +245,9 @@ async def current_actor(): _invalid("DOCUMENT_CATALOG_FAILED") if rows: row = rows[0] - if row.get("request_hash") != request_hash: + if (row.get("request_hash") != request_hash and + (frozen.blob_selections is None or + row.get("request_hash") != _legacy_selection_hash(frozen))): _invalid("DOCUMENT_REPLAY_CONFLICT") try: found = _receipt(DocumentCommandReceipt(row["previous_revision"], diff --git a/packages/teaql-secure-document-reference/tests/test_sqlite_document_service.py b/packages/teaql-secure-document-reference/tests/test_sqlite_document_service.py index b615bdfc..7d5105b9 100644 --- a/packages/teaql-secure-document-reference/tests/test_sqlite_document_service.py +++ b/packages/teaql-secure-document-reference/tests/test_sqlite_document_service.py @@ -2,6 +2,8 @@ from dataclasses import replace from datetime import datetime, timezone +import hashlib +import json import time import aiosqlite @@ -462,6 +464,17 @@ def selection(opened, command_id, key): assert await receipts(path) == 1 replayed = await service.accept(context, "Order", "edit-order", command) assert replayed.replayed and host.mutations == 1 + legacy_parts = [command.business_id, command.document_token, command.command_id, + command.body, [item.__dict__ for item in command.blob_selections]] + legacy_hash = hashlib.sha256(json.dumps(legacy_parts, ensure_ascii=False, + separators=(",", ":"), sort_keys=True).encode("utf-8")).hexdigest() + async with aiosqlite.connect(path) as db: + async with db.execute("SELECT request_hash FROM teaql_document_command_receipt") as cursor: + assert (await cursor.fetchone())[0] != legacy_hash + await db.execute("UPDATE teaql_document_command_receipt SET request_hash=?", (legacy_hash,)) + await db.commit() + old_replay = await service.accept(context, "Order", "edit-order", command) + assert old_replay.replayed and host.mutations == 1 with pytest.raises(RuntimeError, match="DOCUMENT_REPLAY_CONFLICT"): await service.accept(context, "Order", "edit-order", replace(command, blob_selections=None)) oversized = DocumentBlobSelectionSubmission("tqs1." + "A" * (256000 - 5), From 11d539e12aec5679da8f68e44ea8fb73a1166afa Mon Sep 17 00:00:00 2001 From: Philip Z Date: Wed, 28 Oct 2026 19:43:24 +0800 Subject: [PATCH 51/75] Recheck Blob upload authorization on SQLite transaction lease --- packages/teaql-blob-reference/README.md | 2 + .../src/teaql_blob_reference/__init__.py | 5 +- .../teaql_blob_reference/upload_service.py | 50 +++++++++---- tests/provider/sqlite/test_blob_upload.py | 70 ++++++++++++++++++- 4 files changed, 111 insertions(+), 16 deletions(-) diff --git a/packages/teaql-blob-reference/README.md b/packages/teaql-blob-reference/README.md index 84e2426d..50323da2 100644 --- a/packages/teaql-blob-reference/README.md +++ b/packages/teaql-blob-reference/README.md @@ -4,6 +4,8 @@ Optional Blob catalog and current-read services for TeaQL Python. Install this w `SqliteBlobUploadService` offers a host-invoked, scope-bound upload journal. The host must provide current model/action authorization and a trusted parser that consumes every stored byte and returns actual format, size, and SHA256. The service writes through the context's object-storage provider, rechecks the stored generation before confirmation, and promotes the row to `uploaded_unbound` only after full verification. The included PDF verifier in tests is a fixture, not a production format parser. Other SQL backends and production ten-format parsing remain open. +For permissions backed by mutable business rows, the host may implement `current_on_lease(actor, lease, target, action, input)` and return `BlobUploadAuthorizationState(scope, field, allowed)`. Begin, Write, and Confirm then recheck that state on the same physical SQLite transaction as the upload journal; a missing row, revoked permission, malformed state, or scope/field mismatch denies the operation. The ordinary `scope`, `field`, and `authorize` callbacks remain the preflight path and the compatibility path for hosts without this optional hook. + To use several optional parsers in one upload service, pass a `CompositeBlobContentVerifier` with explicit extension-to-parser routes. It copies and validates the route table at construction, calls each distinct diff --git a/packages/teaql-blob-reference/src/teaql_blob_reference/__init__.py b/packages/teaql-blob-reference/src/teaql_blob_reference/__init__.py index 3d7c1ba1..02dcda1f 100644 --- a/packages/teaql-blob-reference/src/teaql_blob_reference/__init__.py +++ b/packages/teaql-blob-reference/src/teaql_blob_reference/__init__.py @@ -13,7 +13,8 @@ SqliteBlobFileGenerationSweepService) from .retention import BlobRetentionPolicy from .content_composite import CompositeBlobContentVerifier -from .upload_service import (BlobUploadInput, BlobUploadSession, BlobUploadTarget, +from .upload_service import (BlobUploadAuthorizationState, BlobUploadInput, + BlobUploadSession, BlobUploadTarget, BlobVerifiedContent, SqliteBlobUploadService) __all__ = ["BlobReferenceModule", "SqliteBlobMutationParticipant", "BlobBindingChange", "BlobBindingTarget", "BlobCatalogRead", @@ -22,6 +23,6 @@ "BlobFileGenerationSweepBatch", "BlobFileGenerationSweepOutcome", "SqliteBlobFileGenerationSweepService", "SqliteBlobCatalog", "SqliteBlobResponseContributor", "FileObjectStorage", - "BlobUploadInput", "BlobUploadSession", "BlobUploadTarget", + "BlobUploadAuthorizationState", "BlobUploadInput", "BlobUploadSession", "BlobUploadTarget", "BlobVerifiedContent", "SqliteBlobUploadService", "BlobRetentionPolicy", "CompositeBlobContentVerifier"] diff --git a/packages/teaql-blob-reference/src/teaql_blob_reference/upload_service.py b/packages/teaql-blob-reference/src/teaql_blob_reference/upload_service.py index 95d533a3..f87c6ad8 100644 --- a/packages/teaql-blob-reference/src/teaql_blob_reference/upload_service.py +++ b/packages/teaql-blob-reference/src/teaql_blob_reference/upload_service.py @@ -44,6 +44,15 @@ class BlobUploadInput: declared_content_type: str = "" +@dataclass(frozen=True) +class BlobUploadAuthorizationState: + """Current scope, field and permission read on the upload's SQL lease.""" + + scope: str + field: BlobFieldSpec + allowed: bool + + @dataclass(frozen=True) class BlobVerifiedContent: size_bytes: int @@ -75,6 +84,14 @@ async def current_policy_on_lease(self, context, lease: SqlTransactionTransport, target: BlobUploadTarget) -> BlobRetentionPolicy: ... +class BlobUploadLeaseAuthorization(Protocol): + """Optional host authorization on the physical upload transaction.""" + + async def current_on_lease(self, context, lease: SqlTransactionTransport, + target: BlobUploadTarget, action: str, + input: BlobUploadInput) -> BlobUploadAuthorizationState: ... + + class BlobContentVerifier(Protocol): def verify(self, stream: BinaryIO, extension: str, maximum_bytes: int) -> BlobVerifiedContent: ... @@ -163,20 +180,27 @@ def _normalize(self, input: BlobUploadInput): raise RuntimeError("BLOB_FORMAT_NOT_ALLOWED") return BlobUploadInput(name, input.declared_size, declared), extension - async def _current(self, actor, target, action, input, expected_scope=None): + async def _current(self, actor, target, action, input, expected_scope=None, lease=None): if (actor is None or not isinstance(target, BlobUploadTarget) or not _name(target.entity) or not _name(target.field) or target.entity_id != "" and not _entity_id(target.entity_id)): raise RuntimeError("BLOB_REFERENCE_INVALID") try: - scope = self.authorization.scope(actor) - field = self.authorization.field(actor, target) - allowed = await self.authorization.authorize(actor, target, action, input) + current_on_lease = getattr(self.authorization, "current_on_lease", None) + if lease is not None and current_on_lease is not None: + state = await current_on_lease(actor, lease, target, action, input) + if not isinstance(state, BlobUploadAuthorizationState): + raise TypeError("invalid upload authorization state") + scope, field, allowed = state.scope, state.field, state.allowed + else: + scope = self.authorization.scope(actor) + field = self.authorization.field(actor, target) + allowed = await self.authorization.authorize(actor, target, action, input) except Exception: raise RuntimeError("BLOB_UPLOAD_DENIED") from None if (not _scope(scope) or expected_scope is not None and scope != expected_scope or not isinstance(field, BlobFieldSpec) or field.entity != target.entity - or field.field != target.field or not allowed): + or field.field != target.field or type(allowed) is not bool or not allowed): raise RuntimeError("BLOB_UPLOAD_DENIED") extension = input.file_name.rsplit(".", 1)[-1].lower() if extension not in field.allowed_extensions: @@ -242,7 +266,7 @@ async def work(tx): row = rows[0] if row.get("request_hash") != request_hash: raise RuntimeError("BLOB_IDEMPOTENCY_CONFLICT") - await self._current(actor, target, "Begin", normalized, scope) + await self._current(actor, target, "Begin", normalized, scope, tx) try: expires_ms = row["expires_ms"] if type(expires_ms) is not int: @@ -257,7 +281,7 @@ async def work(tx): upload_id = "bu_" + _token() object_key = "blob/" + _token() expires_ms = self.now_ms() + self.lease_seconds * 1000 - await self._current(actor, target, "Begin", normalized, scope) + await self._current(actor, target, "Begin", normalized, scope, tx) policy = await self._policy_on_lease(actor, tx, target) profile = Value.Text(policy.profile) if policy else Value.Null() mode = Value.Text(policy.upload_mode) if policy else Value.Null() @@ -380,7 +404,7 @@ async def work(tx): raise RuntimeError("BLOB_IDEMPOTENCY_CONFLICT") await self._current(actor, target, "Write", BlobUploadInput(row["file_name"], row["declared_size"]), - row["scope"]) + row["scope"], tx) if self.now_ms() >= row["expires_ms"]: raise RuntimeError("BLOB_UPLOAD_EXPIRED") @@ -424,7 +448,7 @@ def verify_replay(): count = await asyncio.to_thread(verify_replay) await self._current(actor, target, "Write", BlobUploadInput(row["file_name"], row["declared_size"]), - row["scope"]) + row["scope"], tx) if self.now_ms() >= row["expires_ms"]: raise RuntimeError("BLOB_UPLOAD_EXPIRED") return count @@ -432,7 +456,7 @@ def verify_replay(): raise RuntimeError("BLOB_IDEMPOTENCY_CONFLICT") await self._current(actor, target, "Write", BlobUploadInput(row["file_name"], row["declared_size"]), - row["scope"]) + row["scope"], tx) if self.now_ms() >= row["expires_ms"]: raise RuntimeError("BLOB_UPLOAD_EXPIRED") store = actor.object_storage("write Blob upload") @@ -482,7 +506,7 @@ def recover_existing(): raise RuntimeError("BLOB_STORE_FAILED") from None await self._current(actor, target, "Write", BlobUploadInput(row["file_name"], row["declared_size"]), - row["scope"]) + row["scope"], tx) now = self.now_ms() if now >= row["expires_ms"]: raise RuntimeError("BLOB_UPLOAD_EXPIRED") @@ -622,14 +646,14 @@ async def work(tx): raise RuntimeError("BLOB_IDEMPOTENCY_CONFLICT") await self._current(actor, target, "Confirm", BlobUploadInput(row["file_name"], row["declared_size"]), - row["scope"]) + row["scope"], tx) now = self.now_ms() if not valid_time(now) or now >= row["expires_ms"]: raise RuntimeError("BLOB_UPLOAD_EXPIRED") verified = await asyncio.to_thread(verify_stored) await self._current(actor, target, "Confirm", BlobUploadInput(row["file_name"], row["declared_size"], - verified.content_type), row["scope"]) + verified.content_type), row["scope"], tx) now = self.now_ms() if not valid_time(now) or now >= row["expires_ms"]: raise RuntimeError("BLOB_UPLOAD_EXPIRED") diff --git a/tests/provider/sqlite/test_blob_upload.py b/tests/provider/sqlite/test_blob_upload.py index 142e9c57..c55c1ee3 100644 --- a/tests/provider/sqlite/test_blob_upload.py +++ b/tests/provider/sqlite/test_blob_upload.py @@ -29,8 +29,10 @@ ViewPreparationError, ViewPreparationService, ) from teaql.sql.executor import SqlDataServiceExecutor +from teaql.sql.types import CompiledQuery from teaql_blob_reference import ( - BlobBindingChange, BlobBindingTarget, BlobRetentionPolicy, BlobUploadInput, BlobUploadTarget, + BlobBindingChange, BlobBindingTarget, BlobRetentionPolicy, BlobUploadAuthorizationState, + BlobUploadInput, BlobUploadTarget, BlobVerifiedContent, FileObjectStorage, SqliteBlobCatalog, SqliteBlobResponseContributor, SqliteBlobUploadService, ) @@ -290,6 +292,72 @@ async def authorize(self, _context, _result, _options): provider.close() +@pytest.mark.asyncio +async def test_upload_rechecks_current_permission_on_physical_sql_lease(tmp_path): + path = str(tmp_path / "lease-authority.db") + catalog = SqliteBlobCatalog(SqliteTransport(path)) + await catalog.ensure_schema() + async with aiosqlite.connect(path) as db: + await db.execute("CREATE TABLE upload_permission (id INTEGER PRIMARY KEY, allowed INTEGER)") + await db.execute("INSERT INTO upload_permission VALUES (1,0)") + await db.commit() + + class LeaseAuthority(UploadAuthority): + def __init__(self): + super().__init__() + self.seen = [] + + async def current_on_lease(self, actor, lease, target, action, _input): + self.seen.append(action) + rows = await lease.fetch_all_sql(CompiledQuery( + "SELECT allowed FROM upload_permission WHERE id=1", [])) + assert len(rows) == 1 + return BlobUploadAuthorizationState(self.scope(actor), self.field(actor, target), + rows[0]["allowed"] == 1) + + async def set_allowed(value): + async with aiosqlite.connect(path) as db: + await db.execute("UPDATE upload_permission SET allowed=? WHERE id=1", (value,)) + await db.commit() + + authority = LeaseAuthority() + verifier = FixturePDFVerifier() + service = SqliteBlobUploadService(catalog, authority, verifier, 10_000) + actor = RuntimeModule.new().into_context().with_object_storage( + FileObjectStorage(tmp_path / "private", os.urandom(32))) + actor.with_object_storage_observer(lambda event: None) + target = BlobUploadTarget("Asset", "1", "document") + data = b"%PDF-1.7\nlease permission\n%%EOF" + input = BlobUploadInput("report.pdf", len(data), "application/pdf") + + with pytest.raises(RuntimeError, match="BLOB_UPLOAD_DENIED"): + await service.begin(actor, target, input, "lease-denied") + async with aiosqlite.connect(path) as db: + async with db.execute("SELECT COUNT(*) FROM teaql_blob_upload") as cursor: + assert (await cursor.fetchone())[0] == 0 + + await set_allowed(1) + session = await service.begin(actor, target, input, "lease-allowed") + await set_allowed(0) + source = BytesIO(data) + with pytest.raises(RuntimeError, match="BLOB_UPLOAD_DENIED"): + await service.write(actor, session.upload_id, source) + async with aiosqlite.connect(path) as db: + async with db.execute("SELECT state FROM teaql_blob_upload WHERE upload_id=?", + (session.upload_id,)) as cursor: + assert (await cursor.fetchone())[0] == "initiated" + await set_allowed(1) + assert await service.write(actor, session.upload_id, BytesIO(data)) == len(data) + await set_allowed(0) + with pytest.raises(RuntimeError, match="BLOB_UPLOAD_DENIED"): + await service.complete(actor, session.upload_id) + assert verifier.calls == 0 + await set_allowed(1) + assert await service.complete(actor, session.upload_id) == session.reference + assert verifier.calls == 1 + assert {"Begin", "Write", "Confirm"}.issubset(authority.seen) + + @pytest.mark.asyncio async def test_expired_replay_key_cannot_recover_upload_session(tmp_path): catalog = SqliteBlobCatalog(SqliteTransport(str(tmp_path / "expired.db"))) From 5e6bdf19d7092523f8c49928085a376674d640c0 Mon Sep 17 00:00:00 2001 From: Philip Z Date: Wed, 28 Oct 2026 20:45:49 +0800 Subject: [PATCH 52/75] Recheck completed upload replay on SQLite lease --- .../teaql_blob_reference/upload_service.py | 66 +++++++++++------- tests/provider/sqlite/test_blob_upload.py | 69 +++++++++++++++++++ 2 files changed, 111 insertions(+), 24 deletions(-) diff --git a/packages/teaql-blob-reference/src/teaql_blob_reference/upload_service.py b/packages/teaql-blob-reference/src/teaql_blob_reference/upload_service.py index f87c6ad8..4634e642 100644 --- a/packages/teaql-blob-reference/src/teaql_blob_reference/upload_service.py +++ b/packages/teaql-blob-reference/src/teaql_blob_reference/upload_service.py @@ -540,42 +540,60 @@ async def complete(self, actor, upload_id: str) -> BlobRef: re.fullmatch(r"[0-9a-f]{64}", row["written_sha256"]) is None or not row["written_etag"]): raise RuntimeError("BLOB_UPLOAD_INCOMPLETE") - try: - rows = await self.catalog.transport.fetch_all_sql(_query( - "SELECT object_key,content_version,content_type,size_bytes,status " - "FROM teaql_blob_catalog WHERE key=? AND scope=?", - Value.Text(row["key"]), Value.Text(row["scope"]))) - if len(rows) != 1: + target = BlobUploadTarget(row["entity"], row["entity_id"], row["field"]) + mime = blob_mime_for_extension(row["extension"]) + input = BlobUploadInput(row["file_name"], row["declared_size"], mime) + + async def replay_work(tx): + try: + current = await tx.fetch_all_sql(_query( + "SELECT state,expires_ms,key,written_sha256,written_etag " + "FROM teaql_blob_upload WHERE upload_id=? AND scope=?", + Value.Text(upload_id), Value.Text(row["scope"]))) + rows = await tx.fetch_all_sql(_query( + "SELECT object_key,content_version,content_type,size_bytes,status " + "FROM teaql_blob_catalog WHERE key=? AND scope=?", + Value.Text(row["key"]), Value.Text(row["scope"]))) + except Exception: + raise RuntimeError("BLOB_CATALOG_FAILED") from None + if (len(current) != 1 or current[0]["state"] != "completed" or + current[0]["expires_ms"] != row["expires_ms"] or + current[0]["key"] != row["key"] or + current[0]["written_sha256"] != row["written_sha256"] or + current[0]["written_etag"] != row["written_etag"] or + len(rows) != 1): raise RuntimeError("BLOB_UPLOAD_INCOMPLETE") catalog = rows[0] - mime = blob_mime_for_extension(row["extension"]) if (catalog["status"] not in ("uploaded_unbound", "available") or catalog["content_version"] != row["written_etag"] or catalog["content_type"] != mime or type(catalog["size_bytes"]) is not int or catalog["size_bytes"] != row["declared_size"]): raise RuntimeError("BLOB_UPLOAD_INCOMPLETE") - key = ObjectKey(catalog["object_key"]) - except Exception: - raise RuntimeError("BLOB_UPLOAD_INCOMPLETE") from None - - def recheck_stored(): try: - metadata = actor.object_storage("recheck completed Blob upload").head(key) - if (metadata is None or metadata.size != row["declared_size"] or - metadata.etag != row["written_etag"]): - raise RuntimeError("BLOB_UPLOAD_INCOMPLETE") + key = ObjectKey(catalog["object_key"]) except Exception: raise RuntimeError("BLOB_UPLOAD_INCOMPLETE") from None + await self._current(actor, target, "Confirm", input, row["scope"], tx) + if self.now_ms() >= row["expires_ms"]: + raise RuntimeError("BLOB_UPLOAD_EXPIRED") - await asyncio.to_thread(recheck_stored) - target = BlobUploadTarget(row["entity"], row["entity_id"], row["field"]) - await self._current(actor, target, "Confirm", - BlobUploadInput(row["file_name"], row["declared_size"], mime), - row["scope"]) - if self.now_ms() >= row["expires_ms"]: - raise RuntimeError("BLOB_UPLOAD_EXPIRED") - return BlobRef(row["key"]) + def recheck_stored(): + try: + metadata = actor.object_storage("recheck completed Blob upload").head(key) + if (metadata is None or metadata.size != row["declared_size"] or + metadata.etag != row["written_etag"]): + raise RuntimeError("BLOB_UPLOAD_INCOMPLETE") + except Exception: + raise RuntimeError("BLOB_UPLOAD_INCOMPLETE") from None + + await asyncio.to_thread(recheck_stored) + await self._current(actor, target, "Confirm", input, row["scope"], tx) + if self.now_ms() >= row["expires_ms"]: + raise RuntimeError("BLOB_UPLOAD_EXPIRED") + return BlobRef(row["key"]) + + return await self._transaction(replay_work) if row["state"] != "written" or not row["written_sha256"] or not row["written_etag"]: raise RuntimeError("BLOB_UPLOAD_INCOMPLETE") key = await self._object_key(actor, row) diff --git a/tests/provider/sqlite/test_blob_upload.py b/tests/provider/sqlite/test_blob_upload.py index c55c1ee3..293c9f6a 100644 --- a/tests/provider/sqlite/test_blob_upload.py +++ b/tests/provider/sqlite/test_blob_upload.py @@ -355,6 +355,13 @@ async def set_allowed(value): await set_allowed(1) assert await service.complete(actor, session.upload_id) == session.reference assert verifier.calls == 1 + await set_allowed(0) + with pytest.raises(RuntimeError, match="BLOB_UPLOAD_DENIED"): + await service.complete(actor, session.upload_id) + assert verifier.calls == 1 + await set_allowed(1) + assert await service.complete(actor, session.upload_id) == session.reference + assert verifier.calls == 1 assert {"Begin", "Write", "Confirm"}.issubset(authority.seen) @@ -465,6 +472,68 @@ def verify(self, stream, extension, maximum_bytes): assert verifier.calls == 1 +@pytest.mark.asyncio +async def test_confirm_serializes_permission_revocation_during_parser_io(tmp_path): + path = str(tmp_path / "confirm-revocation.db") + catalog = SqliteBlobCatalog(SqliteTransport(path)) + await catalog.ensure_schema() + async with aiosqlite.connect(path) as db: + await db.execute("CREATE TABLE upload_permission (id INTEGER PRIMARY KEY, allowed INTEGER)") + await db.execute("INSERT INTO upload_permission VALUES (1,1)") + await db.commit() + entered, release = Event(), Event() + + class LeaseAuthority(UploadAuthority): + async def current_on_lease(self, actor, lease, target, action, _input): + rows = await lease.fetch_all_sql(CompiledQuery( + "SELECT allowed FROM upload_permission WHERE id=1", [])) + return BlobUploadAuthorizationState(self.scope(actor), self.field(actor, target), + len(rows) == 1 and rows[0]["allowed"] == 1) + + class BlockingVerifier(FixturePDFVerifier): + def verify(self, stream, extension, maximum_bytes): + result = super().verify(stream, extension, maximum_bytes) + entered.set() + if not release.wait(5): + raise AssertionError("test did not release parser") + return result + + actor = RuntimeModule.new().into_context().with_object_storage( + FileObjectStorage(tmp_path / "private", os.urandom(32))) + actor.with_object_storage_observer(lambda event: None) + verifier = BlockingVerifier() + service = SqliteBlobUploadService(catalog, LeaseAuthority(), verifier, 10_000) + data = b"%PDF-1.7\nconcurrent revocation\n%%EOF" + session = await service.begin(actor, BlobUploadTarget("Asset", "1", "document"), + BlobUploadInput("report.pdf", len(data)), "revoke-request") + assert await service.write(actor, session.upload_id, BytesIO(data)) == len(data) + + async def revoke(): + async with aiosqlite.connect(path) as db: + await db.execute("UPDATE upload_permission SET allowed=0 WHERE id=1") + await db.commit() + + confirmation = asyncio.create_task(service.complete(actor, session.upload_id)) + revocation = None + try: + assert await asyncio.to_thread(entered.wait, 5) + revocation = asyncio.create_task(revoke()) + await asyncio.sleep(0.05) + assert not revocation.done(), "revocation committed during the parser's SQL owner" + finally: + release.set() + assert await confirmation == session.reference + assert revocation is not None + await revocation + with pytest.raises(RuntimeError, match="BLOB_UPLOAD_DENIED"): + await service.complete(actor, session.upload_id) + assert verifier.calls == 1 + async with aiosqlite.connect(path) as db: + async with db.execute("SELECT state FROM teaql_blob_upload WHERE upload_id=?", + (session.upload_id,)) as cursor: + assert (await cursor.fetchone())[0] == "completed" + + @pytest.mark.asyncio async def test_upload_write_commit_unknown_replays_only_same_bytes_and_generation(tmp_path): path = str(tmp_path / "commit-unknown.db") From 2d156dea6e4d70c64aeb80144c08b445b46f1046 Mon Sep 17 00:00:00 2001 From: Philip Z Date: Wed, 28 Oct 2026 22:30:16 +0800 Subject: [PATCH 53/75] test(blob): cover allocated ledger binding and outer rollback --- tests/provider/sqlite/test_blob_module.py | 59 ++++++++++++++++++++++- 1 file changed, 58 insertions(+), 1 deletion(-) diff --git a/tests/provider/sqlite/test_blob_module.py b/tests/provider/sqlite/test_blob_module.py index 84a2bb5a..2440495a 100644 --- a/tests/provider/sqlite/test_blob_module.py +++ b/tests/provider/sqlite/test_blob_module.py @@ -7,7 +7,7 @@ import pytest from teaql.core.blob_ref import BlobFieldSpec, BlobRef -from teaql.core.entity import EntityKey +from teaql.core.entity import EntityKey, EntityRoot from teaql.core.meta import EntityDescriptor, PropertyDescriptor from teaql.core.mutation import DeleteCommand, InsertCommand, UpdateCommand from teaql.core.value import DataType @@ -219,4 +219,61 @@ async def work(graph): assert await cursor.fetchone() == (None, 1) await save(DeleteCommand.new("Asset", 2).expected_version(1).hard_delete(), "delete empty", asset_id=2) + + ledger_ref = BlobRef("blob_native_ledger") + async with aiosqlite.connect(path) as db: + await db.execute("INSERT INTO teaql_blob_catalog " + "(key,scope,object_key,content_version,file_name,content_type,size_bytes,status," + "completed_at_ms,policy_profile,upload_mode,unbound_retention_seconds,effective_unref_seconds) " + "VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?)", + (ledger_ref.key, "tenant-a", "objects/ledger", "v1", "ledger.pdf", + "application/pdf", 7, "uploaded_unbound", time.time_ns() // 1_000_000, + "default", "server_stream", 86400, 604800)) + await db.commit() + + async def save_allocated_ledger(abort): + ledger = EntityRoot() + provisional = EntityKey("Asset", -1) + ledger.mark_as_new(provisional) + ledger.set(provisional, "document", ledger_ref) + + async def work(graph): + allocated = await graph.transaction.next_id("Asset") + assert type(allocated) is int and allocated > 0 + actual = EntityKey("Asset", allocated) + ledger.rekey(provisional, actual) + command = (InsertCommand.new("Asset").value("id", allocated) + .value("document", ledger_ref)) + result = await graph.transaction.mutate(graph.context, + graph.request(command, graph.scope(actual), ledger, actual)) + assert getattr(result.generated_values["id"], "val") == allocated + assert ledger.get(actual, "document") == ledger_ref + if abort: + raise RuntimeError("outer owner rejected") + return allocated + + return await context.execute_graph_save(work, comment="save allocated ledger Blob") + + with pytest.raises(RuntimeError, match="outer owner rejected"): + await save_allocated_ledger(True) + async with aiosqlite.connect(path) as db: + async with db.execute("SELECT COUNT(*) FROM asset WHERE document=?", + (ledger_ref.key,)) as cursor: + assert await cursor.fetchone() == (0,) + async with db.execute("SELECT COUNT(*) FROM teaql_blob_binding WHERE key=?", + (ledger_ref.key,)) as cursor: + assert await cursor.fetchone() == (0,) + async with db.execute("SELECT status FROM teaql_blob_catalog WHERE key=?", + (ledger_ref.key,)) as cursor: + assert await cursor.fetchone() == ("uploaded_unbound",) + allocated = await save_allocated_ledger(False) + async with aiosqlite.connect(path) as db: + async with db.execute("SELECT document FROM asset WHERE id=?", (allocated,)) as cursor: + assert await cursor.fetchone() == (ledger_ref.key,) + async with db.execute("SELECT key FROM teaql_blob_binding WHERE entity_id=?", + (str(allocated),)) as cursor: + assert await cursor.fetchone() == (ledger_ref.key,) + async with db.execute("SELECT status FROM teaql_blob_catalog WHERE key=?", + (ledger_ref.key,)) as cursor: + assert await cursor.fetchone() == ("available",) store.close() From 0a8d386f2b42adf98734c6a455cd0036501f03c3 Mon Sep 17 00:00:00 2001 From: Philip Z Date: Wed, 28 Oct 2026 23:27:16 +0800 Subject: [PATCH 54/75] Build and verify optional reference Python release artifacts --- .github/workflows/release.yml | 10 +- scripts/verify-reference-release-artifacts.py | 116 ++++++++++++++++++ scripts/verify-reference-release-consumers.py | 77 ++++++++++++ 3 files changed, 202 insertions(+), 1 deletion(-) create mode 100644 scripts/verify-reference-release-artifacts.py create mode 100644 scripts/verify-reference-release-consumers.py diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index eea7a2c6..8db7224e 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -16,10 +16,18 @@ jobs: - run: python -m pip install --upgrade build - run: python -m pip install --editable ".[test,telemetry]" - run: python -m pip install --no-deps --editable ./packages/teaql-blob-reference + - run: python -m pip install --no-deps --editable ./packages/teaql-secure-document-reference - name: Verify tag matches package version run: test "${GITHUB_REF_NAME#python-v}" = "$(python -c 'import importlib.metadata; print(importlib.metadata.version("teaql"))')" - run: python -m pytest - - run: python -m build + - run: python -m pytest packages/teaql-secure-document-reference/tests + - run: python -m build --outdir dist . + - run: python -m build --outdir dist ./packages/teaql-blob-reference + - run: python -m build --outdir dist ./packages/teaql-secure-document-reference + - name: Verify exact release wheels and source distributions + run: python scripts/verify-reference-release-artifacts.py dist + - name: Run four isolated consumers from exact wheels + run: python scripts/verify-reference-release-consumers.py dist - uses: actions/upload-artifact@v4 with: name: python-distributions diff --git a/scripts/verify-reference-release-artifacts.py b/scripts/verify-reference-release-artifacts.py new file mode 100644 index 00000000..97cf35f2 --- /dev/null +++ b/scripts/verify-reference-release-artifacts.py @@ -0,0 +1,116 @@ +#!/usr/bin/env python3 +"""Check the exact three-module wheel/sdist set before artifact upload.""" + +from email.parser import Parser +from pathlib import Path +import sys +import tarfile +from zipfile import ZipFile + + +ROOT = Path(__file__).resolve().parents[1] +if len(sys.argv) != 2 or not Path(sys.argv[1]).is_dir(): + raise SystemExit("Usage: verify-reference-release-artifacts.py ") +DIST = Path(sys.argv[1]).resolve() + +try: + import tomllib +except ImportError: # Local Python 3.10; release CI uses Python 3.12. + import tomli as tomllib + +projects = { + "teaql": (ROOT, "teaql"), + "teaql-blob-reference": (ROOT / "packages/teaql-blob-reference", "teaql_blob_reference"), + "teaql-secure-document-reference": ( + ROOT / "packages/teaql-secure-document-reference", "teaql_secure_document_reference" + ), +} +versions = {} +for name, (source, _) in projects.items(): + metadata = tomllib.loads((source / "pyproject.toml").read_text())['project'] + if metadata['name'] != name: + raise SystemExit(f"Unexpected project name in {source}") + versions[name] = metadata['version'] +if len(set(versions.values())) != 1: + raise SystemExit("Reference module versions differ from the base package") + +artifacts = sorted(DIST.iterdir()) +wheels = [p for p in artifacts if p.name.endswith(".whl")] +sdists = [p for p in artifacts if p.name.endswith(".tar.gz")] +if len(artifacts) != 6 or len(wheels) != 3 or len(sdists) != 3: + raise SystemExit("Expected exactly three wheels and three source distributions") + + +def identity(raw: bytes) -> tuple[str, str]: + data = Parser().parsestr(raw.decode("utf-8")) + return data.get("Name", ""), data.get("Version", "") + + +def source_files(source: Path, package: str) -> dict[str, bytes]: + root = source / "src" / package + return { + str(path.relative_to(source / "src")): path.read_bytes() + for path in root.rglob("*") + if path.is_file() and "__pycache__" not in path.parts and path.suffix != ".pyc" + } + + +seen_wheels = set() +for wheel in wheels: + with ZipFile(wheel) as archive: + metadata_files = [entry for entry in archive.namelist() + if entry.endswith(".dist-info/METADATA")] + if len(metadata_files) != 1: + raise SystemExit(f"Unexpected wheel metadata in {wheel.name}") + name, version = identity(archive.read(metadata_files[0])) + if name not in projects or version != versions[name] or name in seen_wheels: + raise SystemExit(f"Unexpected wheel identity in {wheel.name}: {name} {version}") + expected_name = name.replace("-", "_") + f"-{version}-py3-none-any.whl" + if wheel.name != expected_name: + raise SystemExit(f"Unexpected wheel filename: {wheel.name}") + source, package = projects[name] + entries = set(archive.namelist()) + for other in {item[1] for item in projects.values()} - {package}: + if any(entry.startswith(other + "/") for entry in entries): + raise SystemExit(f"Other module leaked into {wheel.name}: {other}") + files = source_files(source, package) + if not files or any(path not in entries or archive.read(path) != data + for path, data in files.items()): + raise SystemExit(f"Source bytes missing or changed in {wheel.name}") + seen_wheels.add(name) + print(f"PASS exact {name} {version} wheel, {len(files)} source files") + +seen_sdists = set() +for sdist in sdists: + with tarfile.open(sdist, "r:gz") as archive: + entries = {entry.name: entry for entry in archive.getmembers() if entry.isfile()} + metadata_files = [path for path in entries if path.endswith("/PKG-INFO") + and path.count("/") == 1] + if len(metadata_files) != 1: + raise SystemExit(f"Unexpected sdist metadata in {sdist.name}") + extracted = archive.extractfile(entries[metadata_files[0]]) + if extracted is None: + raise SystemExit(f"Missing sdist metadata bytes in {sdist.name}") + name, version = identity(extracted.read()) + if name not in projects or version != versions[name] or name in seen_sdists: + raise SystemExit(f"Unexpected sdist identity in {sdist.name}: {name} {version}") + source, package = projects[name] + root = metadata_files[0].split("/", 1)[0] + if sdist.name not in (f"{name}-{version}.tar.gz", + f"{name.replace('-', '_')}-{version}.tar.gz"): + raise SystemExit(f"Unexpected sdist filename: {sdist.name}") + for other in {item[1] for item in projects.values()} - {package}: + if any(path.startswith(f"{root}/src/{other}/") for path in entries): + raise SystemExit(f"Other module leaked into {sdist.name}: {other}") + files = source_files(source, package) + for path, data in files.items(): + member = entries.get(f"{root}/src/{path}") + content = None if member is None else archive.extractfile(member) + if content is None or content.read() != data: + raise SystemExit(f"Source bytes missing or changed in {sdist.name}: {path}") + seen_sdists.add(name) + print(f"PASS exact {name} {version} sdist, {len(files)} source files") + +if seen_wheels != set(projects) or seen_sdists != set(projects): + raise SystemExit("Missing reference module distribution") +print("PASS exact three-package Python release set: 3 wheels and 3 sdists") diff --git a/scripts/verify-reference-release-consumers.py b/scripts/verify-reference-release-consumers.py new file mode 100644 index 00000000..0e3df1a4 --- /dev/null +++ b/scripts/verify-reference-release-consumers.py @@ -0,0 +1,77 @@ +#!/usr/bin/env python3 +"""Import exact release wheels in four isolated TeaQL package installations.""" + +from pathlib import Path +import json +import subprocess +import sys +import tempfile + + +if len(sys.argv) != 2 or not Path(sys.argv[1]).is_dir(): + raise SystemExit("Usage: verify-reference-release-consumers.py ") +dist = Path(sys.argv[1]).resolve() +wheels = { + "base": list(dist.glob("teaql-*-py3-none-any.whl")), + "blob": list(dist.glob("teaql_blob_reference-*-py3-none-any.whl")), + "secure": list(dist.glob("teaql_secure_document_reference-*-py3-none-any.whl")), +} +if any(len(files) != 1 for files in wheels.values()): + raise SystemExit("Expected exactly one wheel per reference package") +variants = { + "base": ("base",), + "secure-only": ("base", "secure"), + "blob-only": ("base", "blob"), + "combined": ("base", "blob", "secure"), +} +third_party = [path for path in sys.path + if "site-packages" in path and Path(path).is_dir()] + +consumer = r''' +import importlib.util +import json +from pathlib import Path +import sys + +target = Path(sys.argv[1]).resolve() +third_party = json.loads(sys.argv[2]) +expected_blob = sys.argv[3] == "1" +expected_secure = sys.argv[4] == "1" +sys.path = [str(target)] + [path for path in sys.path if "site-packages" not in path] + third_party + +from teaql.runtime import RuntimeModule +assert RuntimeModule.__name__ == "RuntimeModule" +assert Path(importlib.util.find_spec("teaql").origin).resolve().is_relative_to(target) +for module, expected, symbol in ( + ("teaql_blob_reference", expected_blob, "BlobReferenceModule"), + ("teaql_secure_document_reference", expected_secure, "SecureDocumentReferenceModule"), +): + spec = importlib.util.find_spec(module) + assert (spec is not None) == expected, (module, expected) + if expected: + package = __import__(module) + assert getattr(package, symbol).__name__ == symbol + assert Path(spec.origin).resolve().is_relative_to(target) +print("PASS isolated wheel consumer") +''' + +with tempfile.TemporaryDirectory(prefix="teaql-python-release-consumers-") as scratch: + for variant, packages in variants.items(): + target = Path(scratch) / variant + target.mkdir() + command = [sys.executable, "-m", "pip", "install", "--no-index", "--no-deps", + "--disable-pip-version-check", "--target", str(target)] + command.extend(str(wheels[package][0]) for package in packages) + installed = subprocess.run(command, capture_output=True, text=True) + if installed.returncode: + raise SystemExit(f"{variant}: exact wheel install failed: " + + (installed.stdout + installed.stderr)[-900:]) + checked = subprocess.run( + [sys.executable, "-S", "-c", consumer, str(target), json.dumps(third_party), + "1" if "blob" in packages else "0", "1" if "secure" in packages else "0"], + cwd=scratch, capture_output=True, text=True, + ) + if checked.returncode or "PASS isolated wheel consumer" not in checked.stdout: + raise SystemExit(f"{variant}: isolated wheel import failed: " + + (checked.stdout + checked.stderr)[-900:]) + print(f"PASS {variant} exact-wheel consumer") From 565a1e199d8992ee4d1d7ed1d21efcd4b0d656d4 Mon Sep 17 00:00:00 2001 From: Philip Z Date: Wed, 28 Oct 2026 23:36:28 +0800 Subject: [PATCH 55/75] Verify same-host signed S3 URL refresh after expiry --- tests/runtime/test_object_storage.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/runtime/test_object_storage.py b/tests/runtime/test_object_storage.py index 907b1372..4e5cafd8 100644 --- a/tests/runtime/test_object_storage.py +++ b/tests/runtime/test_object_storage.py @@ -209,6 +209,10 @@ def test_minio_real_signed_range_and_expiry(): with pytest.raises(HTTPError) as expired: urlopen(ticket.url, timeout=5) assert expired.value.code == 403 + refreshed = storage.get_read_url(key, ReadUrlOptions(timedelta(seconds=30), "attachment", "report.pdf")) + assert refreshed.url != ticket.url and refreshed.expires_at > datetime.now(timezone.utc) + with urlopen(Request(refreshed.url, headers={"Range": "bytes=0-7"}), timeout=5) as response: + assert response.status == 206 and response.read() == content[:8] if os.environ.get("TEAQL_S3_TEST_CREATE_ONLY") == "1": create = PutOptions(len(content), WriteMode.CREATE_ONLY, "application/pdf") created = storage.put_bytes(create_key, content, create) From 2aaf2951c98dd178d6dbc1d56c133bcfa3ea68a3 Mon Sep 17 00:00:00 2001 From: Philip Z Date: Thu, 29 Oct 2026 00:55:42 +0800 Subject: [PATCH 56/75] Verify release wheels with isolated dependency installs --- .github/workflows/release.yml | 2 + .../verify-reference-release-dependencies.py | 149 ++++++++++++++++++ 2 files changed, 151 insertions(+) create mode 100644 scripts/verify-reference-release-dependencies.py diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 8db7224e..e994ca0d 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -28,6 +28,8 @@ jobs: run: python scripts/verify-reference-release-artifacts.py dist - name: Run four isolated consumers from exact wheels run: python scripts/verify-reference-release-consumers.py dist + - name: Resolve and install exact wheels with declared dependencies in clean environments + run: python scripts/verify-reference-release-dependencies.py dist - uses: actions/upload-artifact@v4 with: name: python-distributions diff --git a/scripts/verify-reference-release-dependencies.py b/scripts/verify-reference-release-dependencies.py new file mode 100644 index 00000000..f33c863a --- /dev/null +++ b/scripts/verify-reference-release-dependencies.py @@ -0,0 +1,149 @@ +#!/usr/bin/env python3 +"""Install the exact reference release wheels and declared extras in clean venvs.""" + +from pathlib import Path +import hashlib +import os +import subprocess +import sys +import tempfile + + +if len(sys.argv) != 2 or not Path(sys.argv[1]).is_dir(): + raise SystemExit("Usage: verify-reference-release-dependencies.py ") +DIST = Path(sys.argv[1]).resolve() + + +def run(args, *, cwd, env, timeout=900): + result = subprocess.run(args, cwd=cwd, env=env, capture_output=True, text=True, + timeout=timeout) + if result.returncode: + raise RuntimeError(" ".join(map(str, args)) + " failed:\n" + + (result.stdout + result.stderr)[-5000:]) + return result.stdout + result.stderr + + +def exact_wheels(): + names = { + "base": "teaql-", + "blob": "teaql_blob_reference-", + "secure": "teaql_secure_document_reference-", + } + result = {} + for name, prefix in names.items(): + found = [path for path in DIST.glob("*.whl") + if path.name.startswith(prefix) and path.name.endswith("-py3-none-any.whl")] + if len(found) != 1: + raise RuntimeError("expected exactly one " + name + " release wheel") + result[name] = found[0] + return result + + +CHECK = r''' +from importlib import metadata, util +from pathlib import Path +import sys + +root = Path(sys.prefix).resolve() +variant = sys.argv[1] +blob = variant in ("blob-only", "combined") +secure = variant in ("secure-only", "combined") +for distribution, expected in ( + ("teaql", True), + ("teaql-blob-reference", blob), + ("teaql-secure-document-reference", secure), +): + try: + installed = metadata.distribution(distribution) + except metadata.PackageNotFoundError: + installed = None + assert (installed is not None) == expected, (variant, distribution) + if installed is not None: + assert Path(installed.locate_file("")).resolve().is_relative_to(root), distribution +from teaql.runtime import RuntimeModule +assert RuntimeModule.__name__ == "RuntimeModule" +assert (util.find_spec("teaql_blob_reference") is not None) == blob +assert (util.find_spec("teaql_secure_document_reference") is not None) == secure +if blob: + from teaql_blob_reference import BlobReferenceModule + from teaql_blob_reference.content_pillow import PillowBlobContentVerifier + from teaql_blob_reference.content_pypdf import PypdfBlobContentVerifier + from teaql_blob_reference.content_office import OfficeBlobContentVerifier + from teaql.provider.s3 import S3ObjectStorage + import boto3 + assert BlobReferenceModule.__name__ == "BlobReferenceModule" + assert S3ObjectStorage.__name__ == "S3ObjectStorage" + assert boto3.__version__ + PillowBlobContentVerifier().validate_ready() + PypdfBlobContentVerifier().validate_ready() + OfficeBlobContentVerifier().validate_ready() +else: + assert util.find_spec("boto3") is None + assert util.find_spec("PIL") is None +if secure: + from teaql_secure_document_reference import SecureDocumentReferenceModule + assert SecureDocumentReferenceModule.__name__ == "SecureDocumentReferenceModule" +if variant == "combined": + from teaql.runtime.opentelemetry import OpenTelemetryRuntimeTelemetry + import opentelemetry.sdk + assert OpenTelemetryRuntimeTelemetry.__name__ == "OpenTelemetryRuntimeTelemetry" +print("PASS isolated dependency consumer " + variant) +''' + + +def main(): + wheels = exact_wheels() + with tempfile.TemporaryDirectory(prefix="teaql-python-release-dependencies-") as temporary: + work = Path(temporary) + wheelhouse = work / "wheelhouse" + wheelhouse.mkdir() + environment = dict(os.environ, PIP_CACHE_DIR=str(work / "pip-cache"), + PIP_DISABLE_PIP_VERSION_CHECK="1", PIP_NO_INPUT="1", + PIP_PROGRESS_BAR="off", PYTHONNOUSERSITE="1") + environment.pop("PYTHONPATH", None) + inputs = [ + str(wheels["base"]) + "[object-storage-s3,telemetry]", + str(wheels["blob"]) + "[image,pdf,office]", + str(wheels["secure"]), + ] + run([sys.executable, "-m", "pip", "download", "--only-binary=:all:", + "--dest", str(wheelhouse), *inputs], cwd=work, env=environment) + for name, candidate in wheels.items(): + copy = wheelhouse / candidate.name + if not copy.is_file() or copy.read_bytes() != candidate.read_bytes(): + raise RuntimeError(name + " candidate wheel not retained exactly") + print("PASS exact " + name + " wheel_sha256=" + + hashlib.sha256(candidate.read_bytes()).hexdigest(), flush=True) + resolved = sorted(wheelhouse.glob("*.whl")) + manifest = "\n".join( + path.name + " " + hashlib.sha256(path.read_bytes()).hexdigest() + for path in resolved + ).encode() + print("PASS resolved wheelhouse count=" + str(len(resolved)) + + " manifest_sha256=" + hashlib.sha256(manifest).hexdigest(), flush=True) + cases = { + "base": (str(wheels["base"]),), + "secure-only": (str(wheels["base"]), str(wheels["secure"])), + "blob-only": (str(wheels["base"]) + "[object-storage-s3]", + str(wheels["blob"]) + "[image,pdf,office]"), + "combined": (str(wheels["base"]) + "[object-storage-s3,telemetry]", + str(wheels["blob"]) + "[image,pdf,office]", str(wheels["secure"])), + } + for name, packages in cases.items(): + venv = work / name + run([sys.executable, "-m", "venv", str(venv)], cwd=work, env=environment) + python = venv / "bin/python" + run([str(python), "-I", "-m", "pip", "install", "--no-index", + "--find-links", str(wheelhouse), "--only-binary=:all:", + "--no-cache-dir", *packages], cwd=work, env=environment) + run([str(python), "-I", "-m", "pip", "check"], cwd=work, env=environment) + output = run([str(python), "-I", "-c", CHECK, name], + cwd=work, env=environment) + if "PASS isolated dependency consumer " + name not in output: + raise RuntimeError(name + " application did not run") + print("PASS isolated dependency consumer " + name, flush=True) + print("PASS exact Python release-wheel dependency closure 4/4", flush=True) + + +if __name__ == "__main__": + main() From af8cee377a1e6efe501cc4791af8f6167a4ec07e Mon Sep 17 00:00:00 2001 From: Philip Z Date: Thu, 29 Oct 2026 01:13:48 +0800 Subject: [PATCH 57/75] Guard expired direct uploads from Blob maintenance --- .../maintenance_service.py | 9 ++++- .../provider/sqlite/test_blob_maintenance.py | 40 +++++++++++++++++++ 2 files changed, 47 insertions(+), 2 deletions(-) diff --git a/packages/teaql-blob-reference/src/teaql_blob_reference/maintenance_service.py b/packages/teaql-blob-reference/src/teaql_blob_reference/maintenance_service.py index 79b4b0be..3d2a0789 100644 --- a/packages/teaql-blob-reference/src/teaql_blob_reference/maintenance_service.py +++ b/packages/teaql-blob-reference/src/teaql_blob_reference/maintenance_service.py @@ -153,7 +153,11 @@ def _eligible(row, now): if row is None: return False if row["state"] in ("initiated", "written"): - return (row["expires_ms"] <= now and row["status"] in ("initiated", "deleting")) + # NULL is a pre-upload-mode server-stream session; explicit direct + # uploads need independent quiescence proof before object deletion. + return (row["upload_mode"] in (None, "server_stream") + and row["expires_ms"] <= now + and row["status"] in ("initiated", "deleting")) if row["state"] != "completed" or row["status"] not in ( "uploaded_unbound", "available", "deleting"): return False @@ -324,7 +328,8 @@ async def run_batch(self, actor, cursor: BlobMaintenanceCursor | None = None) -> "SELECT DISTINCT c.key FROM teaql_blob_catalog c JOIN teaql_blob_upload u " "ON u.key=c.key AND u.scope=c.scope WHERE c.scope=? AND c.key>? " "AND ((u.expires_ms<=? AND u.state IN ('initiated','written') " - "AND c.status IN ('initiated','deleting')) OR " + "AND c.status IN ('initiated','deleting') " + "AND (c.upload_mode IS NULL OR c.upload_mode='server_stream')) OR " "(u.state='completed' AND c.upload_mode='server_stream' " "AND c.policy_profile IS NOT NULL AND c.completed_at_ms IS NOT NULL " "AND c.content_version<>'' AND (c.status='deleting' OR " diff --git a/tests/provider/sqlite/test_blob_maintenance.py b/tests/provider/sqlite/test_blob_maintenance.py index e5773b47..a466b7c0 100644 --- a/tests/provider/sqlite/test_blob_maintenance.py +++ b/tests/provider/sqlite/test_blob_maintenance.py @@ -73,6 +73,46 @@ async def _status(path, key): return (await cursor.fetchone())[0] +@pytest.mark.asyncio +async def test_expired_direct_uploads_are_not_collected_without_quiescence(tmp_path): + path = str(tmp_path / "direct-maintenance.db") + catalog = SqliteBlobCatalog(SqliteTransport(path)) + await catalog.ensure_schema() + provider = FileObjectStorage(tmp_path / "direct-objects", os.urandom(32)) + actor = RuntimeModule.new().into_context() + actor.with_object_storage(provider).with_object_storage_observer(lambda event: None) + store = actor.object_storage("direct upload maintenance guard") + service = SqliteBlobMaintenanceService(catalog, _MaintenanceAuthority(), 10, lambda: 10000) + for name, state, status in ( + ("blob_direct_initiated", "initiated", "initiated"), + ("blob_direct_written", "written", "initiated"), + ("blob_direct_deleting", "initiated", "deleting"), + ): + key = ObjectKey("objects/" + name) + created = store.put_bytes(key, b"%PDF-1.7\n", PutOptions(9, WriteMode.CREATE_ONLY)) + await _seed(path, name, state=state, + written_etag=created.etag if state == "written" else None) + async with aiosqlite.connect(path) as db: + await db.execute("UPDATE teaql_blob_catalog SET upload_mode=?,status=? WHERE key=?", + ("direct", status, name)) + await db.commit() + legacy = ObjectKey("objects/blob_legacy_stream") + store.put_bytes(legacy, b"%PDF-1.7\n", PutOptions(9, WriteMode.CREATE_ONLY)) + await _seed(path, "blob_legacy_stream", state="initiated") + + batch = await service.run_batch(actor) + assert [(item.reference.key, item.status) for item in batch.outcomes] == [ + ("blob_legacy_stream", "deleted")] + assert store.head(legacy) is None + for name, _, status in ( + ("blob_direct_initiated", "initiated", "initiated"), + ("blob_direct_written", "written", "initiated"), + ("blob_direct_deleting", "initiated", "deleting"), + ): + assert await _status(path, name) == status + assert store.head(ObjectKey("objects/" + name)) is not None + + @pytest.mark.asyncio async def test_real_sqlite_file_maintenance_recovers_receipt_and_partial_data(tmp_path): path = str(tmp_path / "file-maintenance.db") From 262acc24c9817b52662194e4339ac53453bd2063 Mon Sep 17 00:00:00 2001 From: Philip Z Date: Thu, 29 Oct 2026 01:32:22 +0800 Subject: [PATCH 58/75] test(blob): verify committed Begin reply loss replay in SQLite --- tests/provider/sqlite/test_blob_upload.py | 62 +++++++++++++++++++++++ 1 file changed, 62 insertions(+) diff --git a/tests/provider/sqlite/test_blob_upload.py b/tests/provider/sqlite/test_blob_upload.py index 293c9f6a..21f967bb 100644 --- a/tests/provider/sqlite/test_blob_upload.py +++ b/tests/provider/sqlite/test_blob_upload.py @@ -87,6 +87,68 @@ def verify(self, stream, extension, maximum_bytes): return BlobVerifiedContent(len(data), sha256(data).hexdigest(), "application/pdf") +@pytest.mark.asyncio +async def test_upload_begin_commit_unknown_replays_original_live_session(tmp_path): + path = str(tmp_path / "begin-reply-lost.db") + catalog = SqliteBlobCatalog(SqliteTransport(path)) + await catalog.ensure_schema() + actor = RuntimeModule.new().into_context() + authority = UploadAuthority() + now = [10000] + service = SqliteBlobUploadService(catalog, authority, FixturePDFVerifier(), 1000, + now_ms=lambda: now[0]) + target = BlobUploadTarget("Asset", "1", "document") + upload = BlobUploadInput("report.pdf", 25) + begin_sql = catalog.transport.begin_sql + + class CommitThenFail: + def __init__(self, tx): + self.tx = tx + + def __getattr__(self, name): + return getattr(self.tx, name) + + async def commit_sql(self): + await self.tx.commit_sql() + raise OSError("PRIVATE-BEGIN-REPLY-CANARY") + + async def uncertain_begin(): + return CommitThenFail(await begin_sql()) + + catalog.transport.begin_sql = uncertain_begin + try: + with pytest.raises(RuntimeError, match="^BLOB_CATALOG_FAILED$") as failure: + await service.begin(actor, target, upload, "lost-begin") + assert "CANARY" not in str(failure.value) + finally: + catalog.transport.begin_sql = begin_sql + + async with aiosqlite.connect(path) as db: + async with db.execute("""SELECT u.upload_id,u.key,u.expires_ms,c.status,c.upload_mode + FROM teaql_blob_upload u JOIN teaql_blob_catalog c ON c.key=u.key + WHERE u.scope=? AND u.replay_key=?""", ("tenant-a", "lost-begin")) as cursor: + rows = await cursor.fetchall() + assert len(rows) == 1 + upload_id, key, expires_ms, status, mode = rows[0] + assert status == "initiated" and mode == "server_stream" + + authority.allowed = False + with pytest.raises(RuntimeError, match="^BLOB_UPLOAD_DENIED$"): + await service.begin(actor, target, upload, "lost-begin") + authority.allowed = True + with pytest.raises(RuntimeError, match="^BLOB_IDEMPOTENCY_CONFLICT$"): + await service.begin(actor, target, BlobUploadInput("changed.pdf", 25), "lost-begin") + replay = await service.begin(actor, target, upload, "lost-begin") + assert (replay.upload_id, replay.reference.key, + int(replay.expires_at.timestamp() * 1000)) == (upload_id, key, expires_ms) + now[0] = expires_ms + with pytest.raises(RuntimeError, match="^BLOB_UPLOAD_EXPIRED$"): + await service.begin(actor, target, upload, "lost-begin") + async with aiosqlite.connect(path) as db: + async with db.execute("SELECT COUNT(*) FROM teaql_blob_upload") as cursor: + assert (await cursor.fetchone())[0] == 1 + + @pytest.mark.asyncio @pytest.mark.parametrize("case", ["fixture-pdf", "real-jpeg", "real-pdf", "real-docx", "real-xlsx", "real-doc", "real-xls", From 345ef19a4fab1dc8c88b719ff8369fbd653aad55 Mon Sep 17 00:00:00 2001 From: Philip Z Date: Thu, 29 Oct 2026 02:00:51 +0800 Subject: [PATCH 59/75] test(blob): verify Confirm rolls back on storage close failure --- tests/provider/sqlite/test_blob_upload.py | 47 ++++++++++++++++++++++- 1 file changed, 46 insertions(+), 1 deletion(-) diff --git a/tests/provider/sqlite/test_blob_upload.py b/tests/provider/sqlite/test_blob_upload.py index 21f967bb..ddff4eea 100644 --- a/tests/provider/sqlite/test_blob_upload.py +++ b/tests/provider/sqlite/test_blob_upload.py @@ -18,7 +18,7 @@ from teaql.core.entity import EntityKey from teaql.core.meta import EntityDescriptor, PropertyDescriptor from teaql.core.mutation import InsertCommand -from teaql.core.object_storage import ObjectKey, PutOptions, WriteMode +from teaql.core.object_storage import ObjectContent, ObjectKey, PutOptions, WriteMode from teaql.core.value import DataType from teaql.provider.sqlite import SimpleSchemaProvider from teaql.provider.sqlite.dialect import SqliteDialect @@ -149,6 +149,51 @@ async def uncertain_begin(): assert (await cursor.fetchone())[0] == 1 +@pytest.mark.asyncio +async def test_upload_confirm_close_failure_rolls_back_and_retries(tmp_path): + class CloseFailStream: + def __init__(self, source): + self.source = source + + def read(self, size=-1): + return self.source.read(size) + + def close(self): + self.source.close() + raise OSError("PRIVATE-OBJECT-CLOSE-CANARY") + + class CloseFailStore(FileObjectStorage): + fail_close = True + + def get(self, invocation, key): + content = super().get(invocation, key) + if self.fail_close: + return ObjectContent(content.metadata, CloseFailStream(content.stream)) + return content + + path = str(tmp_path / "confirm-close-failure.db") + catalog = SqliteBlobCatalog(SqliteTransport(path)) + await catalog.ensure_schema() + store = CloseFailStore(tmp_path / "private", os.urandom(32)) + actor = RuntimeModule.new().into_context().with_object_storage(store) + actor.with_object_storage_observer(lambda event: None) + service = SqliteBlobUploadService(catalog, UploadAuthority(), FixturePDFVerifier(), 10_000) + target = BlobUploadTarget("Asset", "1", "document") + data = b"%PDF-1.7\nclose failure must not promote\n%%EOF" + session = await service.begin(actor, target, BlobUploadInput("report.pdf", len(data)), "close-failure") + assert await service.write(actor, session.upload_id, BytesIO(data)) == len(data) + with pytest.raises(RuntimeError, match="^BLOB_UPLOAD_INCOMPLETE$") as failure: + await service.complete(actor, session.upload_id) + assert "CANARY" not in str(failure.value) + async with aiosqlite.connect(path) as db: + async with db.execute("""SELECT u.state,c.status FROM teaql_blob_upload u + JOIN teaql_blob_catalog c ON c.key=u.key WHERE u.upload_id=?""", + (session.upload_id,)) as cursor: + assert await cursor.fetchone() == ("written", "initiated") + store.fail_close = False + assert await service.complete(actor, session.upload_id) == session.reference + + @pytest.mark.asyncio @pytest.mark.parametrize("case", ["fixture-pdf", "real-jpeg", "real-pdf", "real-docx", "real-xlsx", "real-doc", "real-xls", From 27219a58625ff1ee8d06c23da850a02aa28e6e55 Mon Sep 17 00:00:00 2001 From: Philip Z Date: Thu, 29 Oct 2026 02:30:46 +0800 Subject: [PATCH 60/75] fix(blob): close upload source once before SQL promotion --- .../teaql_blob_reference/upload_service.py | 23 ++++++++-- tests/provider/sqlite/test_blob_upload.py | 44 +++++++++++++++++++ 2 files changed, 64 insertions(+), 3 deletions(-) diff --git a/packages/teaql-blob-reference/src/teaql_blob_reference/upload_service.py b/packages/teaql-blob-reference/src/teaql_blob_reference/upload_service.py index 4634e642..192d1a6a 100644 --- a/packages/teaql-blob-reference/src/teaql_blob_reference/upload_service.py +++ b/packages/teaql-blob-reference/src/teaql_blob_reference/upload_service.py @@ -135,7 +135,9 @@ def read(self, size=-1): return block def close(self): - self.stream.close() + # ObjectStorage.put closes this borrowed reader. The upload service + # owns the caller stream through conflict recovery and SQL promotion. + pass class _ReplayMismatch(Exception): @@ -361,6 +363,18 @@ async def _object_key(self, actor, row): async def write(self, actor, upload_id: str, source: BinaryIO) -> int: if source is None or not callable(getattr(source, "read", None)): raise RuntimeError("BLOB_UPLOAD_INCOMPLETE") + source_closed = False + + def close_source(): + nonlocal source_closed + if source_closed: + return + source_closed = True + try: + source.close() + except Exception: + raise RuntimeError("BLOB_STORE_FAILED") from None + try: row = await self._row(actor, upload_id, "Write") key = await self._object_key(actor, row) @@ -451,6 +465,7 @@ def verify_replay(): row["scope"], tx) if self.now_ms() >= row["expires_ms"]: raise RuntimeError("BLOB_UPLOAD_EXPIRED") + close_source() return count if state != "initiated" or catalog[0]["status"] != "initiated": raise RuntimeError("BLOB_IDEMPOTENCY_CONFLICT") @@ -513,6 +528,7 @@ def recover_existing(): if (not reader.eof or reader.count != row["declared_size"] or metadata.size != reader.count or not metadata.etag): raise RuntimeError("BLOB_UPLOAD_INCOMPLETE") + close_source() digest = reader.digest.hexdigest() try: affected, _ = await tx.execute_sql(_query( @@ -530,8 +546,9 @@ def recover_existing(): return reader.count return await self._transaction(work) finally: - with suppress(Exception): - source.close() + if not source_closed: + with suppress(Exception): + source.close() async def complete(self, actor, upload_id: str) -> BlobRef: row = await self._row(actor, upload_id, "Confirm") diff --git a/tests/provider/sqlite/test_blob_upload.py b/tests/provider/sqlite/test_blob_upload.py index ddff4eea..4eb01d12 100644 --- a/tests/provider/sqlite/test_blob_upload.py +++ b/tests/provider/sqlite/test_blob_upload.py @@ -149,6 +149,50 @@ async def uncertain_begin(): assert (await cursor.fetchone())[0] == 1 +@pytest.mark.asyncio +async def test_upload_write_closes_source_once_before_promotion_and_replay(tmp_path): + class CountedSource(BytesIO): + def __init__(self, data, *, fail_close=False): + super().__init__(data) + self.close_count = 0 + self.fail_close = fail_close + + def close(self): + self.close_count += 1 + super().close() + if self.fail_close: + raise OSError("PRIVATE-UPLOAD-SOURCE-CLOSE-CANARY") + if self.close_count > 1: + raise OSError("PRIVATE-DOUBLE-CLOSE-CANARY") + + path = str(tmp_path / "write-source-close.db") + catalog = SqliteBlobCatalog(SqliteTransport(path)) + await catalog.ensure_schema() + actor = RuntimeModule.new().into_context().with_object_storage( + FileObjectStorage(tmp_path / "private", os.urandom(32))) + actor.with_object_storage_observer(lambda event: None) + service = SqliteBlobUploadService(catalog, UploadAuthority(), FixturePDFVerifier(), 10_000) + data = b"%PDF-1.7\nsource close before promotion\n%%EOF" + session = await service.begin(actor, BlobUploadTarget("Asset", "1", "document"), + BlobUploadInput("report.pdf", len(data)), "source-close") + failed = CountedSource(data, fail_close=True) + with pytest.raises(RuntimeError, match="^BLOB_STORE_FAILED$"): + await service.write(actor, session.upload_id, failed) + assert failed.close_count == 1 + async with aiosqlite.connect(path) as db: + async with db.execute("""SELECT u.state,c.status FROM teaql_blob_upload u + JOIN teaql_blob_catalog c ON c.key=u.key WHERE u.upload_id=?""", + (session.upload_id,)) as cursor: + assert await cursor.fetchone() == ("initiated", "initiated") + retry = CountedSource(data) + assert await service.write(actor, session.upload_id, retry) == len(data) + assert retry.close_count == 1 + assert await service.complete(actor, session.upload_id) == session.reference + replay = CountedSource(data) + assert await service.write(actor, session.upload_id, replay) == len(data) + assert replay.close_count == 1 + + @pytest.mark.asyncio async def test_upload_confirm_close_failure_rolls_back_and_retries(tmp_path): class CloseFailStream: From 7e0e1f87ceed87d871b4caa97c5083551496aaaf Mon Sep 17 00:00:00 2001 From: Philip Z Date: Thu, 29 Oct 2026 04:15:46 +0800 Subject: [PATCH 61/75] Cover ten-format S3 graph-to-retention lifecycle --- tests/provider/sqlite/test_blob_upload.py | 103 +++++++++++++++++++--- 1 file changed, 89 insertions(+), 14 deletions(-) diff --git a/tests/provider/sqlite/test_blob_upload.py b/tests/provider/sqlite/test_blob_upload.py index 4eb01d12..cb13ff90 100644 --- a/tests/provider/sqlite/test_blob_upload.py +++ b/tests/provider/sqlite/test_blob_upload.py @@ -10,6 +10,7 @@ from uuid import uuid4 import asyncio import os +import time import aiosqlite import pytest @@ -17,7 +18,7 @@ from teaql.core.blob_ref import BlobFieldSpec from teaql.core.entity import EntityKey from teaql.core.meta import EntityDescriptor, PropertyDescriptor -from teaql.core.mutation import InsertCommand +from teaql.core.mutation import InsertCommand, UpdateCommand from teaql.core.object_storage import ObjectContent, ObjectKey, PutOptions, WriteMode from teaql.core.value import DataType from teaql.provider.sqlite import SimpleSchemaProvider @@ -34,7 +35,7 @@ BlobBindingChange, BlobBindingTarget, BlobRetentionPolicy, BlobUploadAuthorizationState, BlobUploadInput, BlobUploadTarget, BlobVerifiedContent, FileObjectStorage, SqliteBlobCatalog, SqliteBlobResponseContributor, - SqliteBlobUploadService, + SqliteBlobMaintenanceService, SqliteBlobUploadService, ) @@ -239,14 +240,14 @@ def get(self, invocation, key): @pytest.mark.asyncio -@pytest.mark.parametrize("case", ["fixture-pdf", "real-jpeg", "real-pdf", +@pytest.mark.parametrize("case", ["fixture-pdf", "real-jpg", "real-jpeg", "real-png", "real-pdf", "real-docx", "real-xlsx", "real-doc", "real-xls", "real-mp3", "real-mp4"]) async def test_upload_real_sqlite_s3_create_only_confirm_and_replay(tmp_path, case): if os.environ.get("TEAQL_S3_TEST_BLOB_UPLOAD") != "1": pytest.skip("explicit S3-compatible Blob upload gate required") - if case == "real-jpeg" and os.environ.get("TEAQL_S3_TEST_REAL_IMAGE") != "1": - pytest.skip("explicit production JPEG S3 gate required") + if case in ("real-jpg", "real-jpeg", "real-png") and os.environ.get("TEAQL_S3_TEST_REAL_IMAGE") != "1": + pytest.skip("explicit production image S3 gate required") if case == "real-pdf" and os.environ.get("TEAQL_S3_TEST_REAL_PDF") != "1": pytest.skip("explicit production PDF S3 gate required") if case in ("real-docx", "real-xlsx") and os.environ.get("TEAQL_S3_TEST_REAL_OFFICE") != "1": @@ -269,18 +270,23 @@ async def test_upload_real_sqlite_s3_create_only_confirm_and_replay(tmp_path, ca descriptor.property(PropertyDescriptor("id", DataType.I64).is_id()) descriptor.property(PropertyDescriptor("version", DataType.I64).is_version()) descriptor.property(PropertyDescriptor("name", DataType.Text)) + descriptor.property(PropertyDescriptor("document", DataType.Text)) schema = SimpleSchemaProvider() schema.register_entity(descriptor) actor = RuntimeModule.new().entity(descriptor).into_context() actor.insert_resource("dataService", SqlDataServiceExecutor( SqliteDialect(), SqliteTransport(path), schema)) actor.with_object_storage(provider).with_object_storage_observer(lambda event: None) - catalog = SqliteBlobCatalog(SqliteTransport(path)) - if case == "real-jpeg": + clock = [time.time_ns() // 1_000_000] + catalog = SqliteBlobCatalog(SqliteTransport(path), now_ms=lambda: clock[0]) + if case in ("real-jpg", "real-jpeg", "real-png"): from teaql_blob_reference.content_pillow import PillowBlobContentVerifier - authority, parser = UploadAuthority(("jpg",)), PillowBlobContentVerifier() - data = (Path(__file__).parents[2] / "fixtures" / "blob-sample.jpg").read_bytes() - file_name, content_type = "sample.jpg", "image/jpeg" + extension = case.removeprefix("real-") + authority, parser = UploadAuthority((extension,)), PillowBlobContentVerifier() + fixture = "png" if extension == "png" else "jpg" + data = (Path(__file__).parents[2] / "fixtures" / ("blob-sample." + fixture)).read_bytes() + file_name = "sample." + extension + content_type = "image/png" if extension == "png" else "image/jpeg" elif case == "real-pdf": from teaql_blob_reference.content_pypdf import PypdfBlobContentVerifier authority, parser = UploadAuthority(("pdf",)), PypdfBlobContentVerifier() @@ -348,11 +354,12 @@ def verify(self, stream, extension, maximum_bytes): return self.inner.verify(stream, extension, maximum_bytes) verifier = CountingVerifier(parser) - service = SqliteBlobUploadService(catalog, authority, verifier, 100_000) + service = SqliteBlobUploadService(catalog, authority, verifier, 100_000, + now_ms=lambda: clock[0]) object_key = None try: async with aiosqlite.connect(path) as db: - await db.execute("CREATE TABLE asset (id INTEGER PRIMARY KEY, version INTEGER, name TEXT)") + await db.execute("CREATE TABLE asset (id INTEGER PRIMARY KEY, version INTEGER, name TEXT, document TEXT)") await db.commit() await catalog.ensure_schema() session = await service.begin(actor, BlobUploadTarget("Asset", "1", "document"), @@ -383,18 +390,31 @@ def verify(self, stream, extension, maximum_bytes): assert (status, mime, size, version) == ( "uploaded_unbound", content_type, len(data), head.etag) + class LifecycleBindingAuthority(BindingAuthority): + async def authorize(self, _context, target, previous, next_ref, action): + return (target == BlobBindingTarget("Asset", "1", "document") + and ((action == "Create" and previous is None + and next_ref == session.reference) + or (action == "Update" and previous == session.reference + and next_ref is None))) + + binding_authority = LifecycleBindingAuthority() + async def bind(graph): command = (InsertCommand.new("Asset").value("id", 1) - .value("version", 1).value("name", "document")) + .value("version", 1).value("name", "document") + .value("document", session.reference.key)) await graph.transaction.mutate(graph.context, graph.request(command, graph.scope(EntityKey("Asset", 1)))) assert await catalog.bind_in_graph(graph, graph.context, BlobBindingChange(BlobBindingTarget("Asset", "1", "document"), session.reference, 0, 1, "Create"), - BindingAuthority()) == 1 + binding_authority) == 1 await actor.execute_graph_save(bind, comment="bind S3 Blob in business graph") async with aiosqlite.connect(path) as db: + async with db.execute("SELECT document,version FROM asset WHERE id=1") as cursor: + assert await cursor.fetchone() == (session.reference.key, 1) async with db.execute("SELECT status FROM teaql_blob_catalog WHERE key=?", (session.reference.key,)) as cursor: assert (await cursor.fetchone())[0] == "available" @@ -435,6 +455,61 @@ async def authorize(self, _context, _result, _options): ViewOptions("ordinary", "committed", datetime.now(timezone.utc) + timedelta(seconds=60))) assert (stale.value.code, stale.value.business_state) == ("BLOB_READ_DENIED", "committed") + if os.environ.get("TEAQL_S3_TEST_END_TO_END_GC") == "1": + async def unbind(graph): + command = (UpdateCommand.new("Asset", 1).expected_version(1) + .value("document", None)) + await graph.transaction.mutate(graph.context, + graph.request(command, graph.scope(EntityKey("Asset", 1)))) + assert await catalog.bind_in_graph(graph, graph.context, + BlobBindingChange(BlobBindingTarget("Asset", "1", "document"), + None, 1, 2, "Update"), binding_authority) == 2 + + await actor.execute_graph_save(unbind, comment="unbind S3 Blob in business graph") + async with aiosqlite.connect(path) as db: + async with db.execute("SELECT document,version FROM asset WHERE id=1") as cursor: + assert await cursor.fetchone() == (None, 2) + async with db.execute("SELECT key,version FROM teaql_blob_binding WHERE " + "scope='tenant-a' AND entity='Asset' AND entity_id='1' " + "AND field='document'") as cursor: + assert await cursor.fetchone() == (None, 2) + async with db.execute( + "SELECT status,last_unbound_at_ms,effective_unref_seconds," + "retention_floor_until_ms FROM teaql_blob_catalog WHERE key=?", + (session.reference.key,) + ) as cursor: + status, last_unbound, effective, floor = await cursor.fetchone() + assert status == "available" and last_unbound == clock[0] + assert effective == 604800 and floor >= clock[0] + 3600 * 1000 + authorizer.version = 1 + with pytest.raises(ViewPreparationError) as unbound_view: + await ViewPreparationService().prepare(actor, None, options) + assert unbound_view.value.code == "BLOB_READ_DENIED" + + class MaintenanceAuthority: + def scope(self, _context): + return "tenant-a" + + async def can_maintain(self, _context, scope): + return scope == "tenant-a" + + maintenance = SqliteBlobMaintenanceService( + catalog, MaintenanceAuthority(), 10, now_ms=lambda: clock[0]) + deadline = max(last_unbound + effective * 1000, floor) + clock[0] = deadline - 1 + assert (await maintenance.run_batch(actor)).outcomes == () + assert store.head(object_key) is not None + clock[0] = deadline + collected = await maintenance.run_batch(actor) + assert [(item.reference, item.status) for item in collected.outcomes] == [ + (session.reference, "deleted")] + assert store.head(object_key) is None + async with aiosqlite.connect(path) as db: + async with db.execute("SELECT status FROM teaql_blob_catalog WHERE key=?", + (session.reference.key,)) as cursor: + assert (await cursor.fetchone())[0] == "deleted" + print("PASS Python " + case + + " S3 upload/confirm/graph bind/authorized view/unbind/retention GC") finally: try: if object_key is not None: From 3bc7c0ad00d7dfb997ba359977e87680d178a272 Mon Sep 17 00:00:00 2001 From: Philip Z Date: Thu, 29 Oct 2026 05:03:48 +0800 Subject: [PATCH 62/75] test(s3): verify native Blob module owner rollback and retention --- tests/provider/sqlite/test_blob_upload.py | 62 ++++++++++++++++------- 1 file changed, 43 insertions(+), 19 deletions(-) diff --git a/tests/provider/sqlite/test_blob_upload.py b/tests/provider/sqlite/test_blob_upload.py index cb13ff90..58c81ff2 100644 --- a/tests/provider/sqlite/test_blob_upload.py +++ b/tests/provider/sqlite/test_blob_upload.py @@ -32,9 +32,10 @@ from teaql.sql.executor import SqlDataServiceExecutor from teaql.sql.types import CompiledQuery from teaql_blob_reference import ( - BlobBindingChange, BlobBindingTarget, BlobRetentionPolicy, BlobUploadAuthorizationState, + BlobBindingChange, BlobBindingTarget, BlobReferenceModule, BlobRetentionPolicy, + BlobUploadAuthorizationState, BlobUploadInput, BlobUploadTarget, - BlobVerifiedContent, FileObjectStorage, SqliteBlobCatalog, SqliteBlobResponseContributor, + BlobVerifiedContent, FileObjectStorage, SqliteBlobCatalog, SqliteBlobMaintenanceService, SqliteBlobUploadService, ) @@ -270,7 +271,10 @@ async def test_upload_real_sqlite_s3_create_only_confirm_and_replay(tmp_path, ca descriptor.property(PropertyDescriptor("id", DataType.I64).is_id()) descriptor.property(PropertyDescriptor("version", DataType.I64).is_version()) descriptor.property(PropertyDescriptor("name", DataType.Text)) - descriptor.property(PropertyDescriptor("document", DataType.Text)) + descriptor.property(PropertyDescriptor("document", DataType.Text).blob( + BlobFieldSpec("Asset", "document", + ["doc", "docx", "jpeg", "jpg", "mp3", "mp4", "pdf", "png", "xls", "xlsx"], + True, "default"))) schema = SimpleSchemaProvider() schema.register_entity(descriptor) actor = RuntimeModule.new().entity(descriptor).into_context() @@ -400,16 +404,46 @@ async def authorize(self, _context, target, previous, next_ref, action): binding_authority = LifecycleBindingAuthority() + class ReadAuthorization: + allowed = True + async def authorize(self, _context, _slot): + return self.allowed + + read_permission = ReadAuthorization() + module = BlobReferenceModule(catalog, service, read_permission, binding_authority, + maximum_url_seconds=3) + module.install(actor) + await module.ready(actor) + + async def rejected_bind(graph): + command = (InsertCommand.new("Asset").value("id", 1) + .value("version", 1).value("name", "rolled back") + .value("document", session.reference)) + await graph.transaction.mutate(graph.context, + graph.request(command, graph.scope(EntityKey("Asset", 1)))) + raise RuntimeError("S3 outer owner rejected") + + with pytest.raises(RuntimeError, match="S3 outer owner rejected"): + await actor.execute_graph_save(rejected_bind, comment="reject S3 Blob outer owner") + async with aiosqlite.connect(path) as db: + async with db.execute("SELECT COUNT(*) FROM asset WHERE id=1") as cursor: + assert await cursor.fetchone() == (0,) + async with db.execute("SELECT COUNT(*) FROM teaql_blob_binding WHERE " + "scope='tenant-a' AND entity='Asset' AND entity_id='1' " + "AND field='document'") as cursor: + assert await cursor.fetchone() == (0,) + async with db.execute("SELECT status FROM teaql_blob_catalog WHERE key=?", + (session.reference.key,)) as cursor: + assert await cursor.fetchone() == ("uploaded_unbound",) + assert store.head(object_key) is not None + print("PASS Python " + case + " native module outer owner rollback") + async def bind(graph): command = (InsertCommand.new("Asset").value("id", 1) .value("version", 1).value("name", "document") - .value("document", session.reference.key)) + .value("document", session.reference)) await graph.transaction.mutate(graph.context, graph.request(command, graph.scope(EntityKey("Asset", 1)))) - assert await catalog.bind_in_graph(graph, graph.context, - BlobBindingChange(BlobBindingTarget("Asset", "1", "document"), - session.reference, 0, 1, "Create"), - binding_authority) == 1 await actor.execute_graph_save(bind, comment="bind S3 Blob in business graph") async with aiosqlite.connect(path) as db: @@ -419,11 +453,6 @@ async def bind(graph): (session.reference.key,)) as cursor: assert (await cursor.fetchone())[0] == "available" - class ReadAuthorization: - allowed = True - async def authorize(self, _context, _slot): - return self.allowed - class Authorizer: version = 1 async def authorize(self, _context, _result, _options): @@ -431,10 +460,8 @@ async def authorize(self, _context, _result, _options): "tenant-a", self.version, session.reference) return AuthorizedResponseGraph({"document": session.reference}, (slot,)) - read_permission, authorizer = ReadAuthorization(), Authorizer() + authorizer = Authorizer() actor.insert_resource("response_graph_authorizer", authorizer) - actor.insert_resource("blob_response_contributor", - SqliteBlobResponseContributor(catalog, read_permission, 3)) options = ViewOptions("ordinary", "read_only", datetime.now(timezone.utc) + timedelta(seconds=60)) view = await ViewPreparationService().prepare(actor, None, options) @@ -461,9 +488,6 @@ async def unbind(graph): .value("document", None)) await graph.transaction.mutate(graph.context, graph.request(command, graph.scope(EntityKey("Asset", 1)))) - assert await catalog.bind_in_graph(graph, graph.context, - BlobBindingChange(BlobBindingTarget("Asset", "1", "document"), - None, 1, 2, "Update"), binding_authority) == 2 await actor.execute_graph_save(unbind, comment="unbind S3 Blob in business graph") async with aiosqlite.connect(path) as db: From 1ed8ed4cfe744d055f977f0c47f5280c61ca1e95 Mon Sep 17 00:00:00 2001 From: Philip Z Date: Thu, 29 Oct 2026 06:59:47 +0800 Subject: [PATCH 63/75] Test multi-entity native Blob graph rollback --- tests/provider/sqlite/test_blob_module.py | 136 ++++++++++++++++++++++ 1 file changed, 136 insertions(+) diff --git a/tests/provider/sqlite/test_blob_module.py b/tests/provider/sqlite/test_blob_module.py index 2440495a..471957fa 100644 --- a/tests/provider/sqlite/test_blob_module.py +++ b/tests/provider/sqlite/test_blob_module.py @@ -277,3 +277,139 @@ async def work(graph): (ledger_ref.key,)) as cursor: assert await cursor.fetchone() == ("available",) store.close() + + +@pytest.mark.asyncio +async def test_native_blob_multi_entity_graph_rolls_back_together(tmp_path): + path = str(tmp_path / "multi-native.db") + descriptor = EntityDescriptor("Asset").table_name("asset") + descriptor.property(PropertyDescriptor("id", DataType.I64).is_id()) + descriptor.property(PropertyDescriptor("version", DataType.I64).is_version()) + for field in ("document", "legacy_document"): + descriptor.property(PropertyDescriptor(field, DataType.Text).blob( + BlobFieldSpec("Asset", field, ["pdf"], True, "default"))) + schema = SimpleSchemaProvider() + schema.register_entity(descriptor) + context = RuntimeModule.new().entity(descriptor).require_capability("blob.v1").into_context() + context.insert_resource("dataService", SqlDataServiceExecutor( + SqliteDialect(), SqliteTransport(path), schema)) + context.disable_sql_log() + store = FileObjectStorage(tmp_path / "multi-private", os.urandom(32)) + context.with_object_storage(store) + catalog = SqliteBlobCatalog(SqliteTransport(path)) + await catalog.ensure_schema() + refs = [BlobRef(f"blob_multi_{suffix}") for suffix in ("first", "second", "third")] + async with aiosqlite.connect(path) as db: + await db.execute("CREATE TABLE asset (id INTEGER PRIMARY KEY, version INTEGER, " + "document TEXT, legacy_document TEXT)") + for ref in refs: + await db.execute("INSERT INTO teaql_blob_catalog " + "(key,scope,object_key,content_version,file_name,content_type,size_bytes,status," + "completed_at_ms,policy_profile,upload_mode,unbound_retention_seconds,effective_unref_seconds) " + "VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?)", + (ref.key, "tenant-a", f"objects/{ref.key}", "v1", f"{ref.key}.pdf", + "application/pdf", 7, "uploaded_unbound", time.time_ns() // 1_000_000, + "default", "server_stream", 86400, 604800)) + await db.commit() + + class MultiBindingAuthority(_BindingAuthority): + def __init__(self): + self.denied_id = None + self.denied = False + self.visited = [] + + async def authorize(self, context, target, previous, next_ref, action): + self.visited.append((target.entity_id, target.field)) + if target.entity_id == self.denied_id and target.field == "legacy_document": + self.denied = True + return False + return target.entity == "Asset" + + authority = MultiBindingAuthority() + upload = SqliteBlobUploadService(catalog, _UploadAuthority(), _Verifier(), 10_000) + module = BlobReferenceModule(catalog, upload, _ReadAuthority(), authority) + module.install(context) + await module.ready(context) + allocated = [] + + async def save(*, catch_denial=False, outer_reject=False): + allocated.clear() + authority.visited.clear() + authority.denied_id = None + authority.denied = False + ledger = EntityRoot() + provisional = [EntityKey("Asset", -1), EntityKey("Asset", -2)] + for key in provisional: + ledger.mark_as_new(key) + ledger.set(provisional[0], "document", refs[0]) + ledger.set(provisional[1], "document", refs[1]) + ledger.set(provisional[1], "legacy_document", refs[2]) + + async def work(graph): + for _ in provisional: + allocated.append(await graph.transaction.next_id("Asset")) + if catch_denial: + authority.denied_id = str(allocated[1]) + for index, old in enumerate(provisional): + actual = EntityKey("Asset", allocated[index]) + ledger.rekey(old, actual) + assert ledger.get(actual, "document") == refs[index] + command = (InsertCommand.new("Asset").value("id", allocated[index]) + .value("document", ledger.get(actual, "document"))) + if index == 1: + command.value("legacy_document", ledger.get(actual, "legacy_document")) + try: + await graph.transaction.mutate(graph.context, + graph.request(command, graph.scope(actual), ledger, actual)) + except RuntimeError: + if catch_denial: + return + raise + if outer_reject: + raise RuntimeError("outer owner rejected") + + await context.execute_graph_save(work, comment="save multi-entity Blob graph") + return tuple(allocated) + + async def assert_rollback(ids): + async with aiosqlite.connect(path) as db: + async with db.execute("SELECT COUNT(*) FROM asset WHERE id IN (?,?)", ids) as cursor: + assert await cursor.fetchone() == (0,) + async with db.execute("SELECT COUNT(*) FROM teaql_blob_binding") as cursor: + assert await cursor.fetchone() == (0,) + async with db.execute("SELECT key,status FROM teaql_blob_catalog ORDER BY key") as cursor: + assert await cursor.fetchall() == [(ref.key, "uploaded_unbound") for ref in refs] + + try: + with pytest.raises(RuntimeError, match="GRAPH_TRANSACTION_ROLLBACK_ONLY"): + await save(catch_denial=True) + assert authority.denied + assert authority.visited == [ + (str(allocated[0]), "document"), (str(allocated[0]), "legacy_document"), + (str(allocated[1]), "document"), (str(allocated[1]), "legacy_document")] + await assert_rollback(tuple(allocated)) + + with pytest.raises(RuntimeError, match="outer owner rejected"): + await save(outer_reject=True) + await assert_rollback(tuple(allocated)) + + committed = await save() + assert authority.visited == [ + (str(committed[0]), "document"), (str(committed[0]), "legacy_document"), + (str(committed[1]), "document"), (str(committed[1]), "legacy_document")] + async with aiosqlite.connect(path) as db: + async with db.execute("SELECT id,document,legacy_document FROM asset ORDER BY id") as cursor: + assert await cursor.fetchall() == [ + (committed[0], refs[0].key, None), + (committed[1], refs[1].key, refs[2].key)] + async with db.execute("SELECT entity_id,field,key FROM teaql_blob_binding " + "ORDER BY entity_id,field") as cursor: + assert await cursor.fetchall() == [ + (str(committed[0]), "document", refs[0].key), + (str(committed[0]), "legacy_document", None), + (str(committed[1]), "document", refs[1].key), + (str(committed[1]), "legacy_document", refs[2].key)] + async with db.execute("SELECT key,status FROM teaql_blob_catalog ORDER BY key") as cursor: + assert await cursor.fetchall() == [(ref.key, "available") for ref in refs] + finally: + store.close() From 6d2a61378a69f3b8bd5ec1a01a8d2a28a615238a Mon Sep 17 00:00:00 2001 From: Philip Z Date: Thu, 29 Oct 2026 07:53:29 +0800 Subject: [PATCH 64/75] Test native shared Blob final-unbind File retention --- tests/provider/sqlite/test_blob_module.py | 155 ++++++++++++++++++++++ 1 file changed, 155 insertions(+) diff --git a/tests/provider/sqlite/test_blob_module.py b/tests/provider/sqlite/test_blob_module.py index 471957fa..1bc7a1d0 100644 --- a/tests/provider/sqlite/test_blob_module.py +++ b/tests/provider/sqlite/test_blob_module.py @@ -2,6 +2,8 @@ import os import time +from hashlib import sha256 +from io import BytesIO import aiosqlite import pytest @@ -10,6 +12,7 @@ from teaql.core.entity import EntityKey, EntityRoot from teaql.core.meta import EntityDescriptor, PropertyDescriptor from teaql.core.mutation import DeleteCommand, InsertCommand, UpdateCommand +from teaql.core.object_storage import ObjectKey from teaql.core.value import DataType from teaql.provider.sqlite import SimpleSchemaProvider from teaql.provider.sqlite.dialect import SqliteDialect @@ -18,6 +21,7 @@ from teaql.sql.executor import SqlDataServiceExecutor from teaql_blob_reference import ( BlobReferenceModule, BlobRetentionPolicy, FileObjectStorage, SqliteBlobCatalog, + BlobUploadInput, BlobUploadTarget, BlobVerifiedContent, SqliteBlobMaintenanceService, SqliteBlobUploadService, ) @@ -413,3 +417,154 @@ async def assert_rollback(ids): assert await cursor.fetchall() == [(ref.key, "available") for ref in refs] finally: store.close() + + +@pytest.mark.asyncio +async def test_native_shared_blob_final_unbind_controls_file_retention(tmp_path): + path = str(tmp_path / "shared-native.db") + descriptor = EntityDescriptor("Asset").table_name("asset") + descriptor.property(PropertyDescriptor("id", DataType.I64).is_id()) + descriptor.property(PropertyDescriptor("version", DataType.I64).is_version()) + descriptor.property(PropertyDescriptor("document", DataType.Text).blob( + BlobFieldSpec("Asset", "document", ["pdf"], True, "default"))) + schema = SimpleSchemaProvider() + schema.register_entity(descriptor) + context = RuntimeModule.new().entity(descriptor).require_capability("blob.v1").into_context() + context.insert_resource("dataService", SqlDataServiceExecutor( + SqliteDialect(), SqliteTransport(path), schema)) + context.disable_sql_log() + store = FileObjectStorage(tmp_path / "shared-private", os.urandom(32)) + context.with_object_storage(store).with_object_storage_observer(lambda event: None) + clock = [1_700_000_000_000] + catalog = SqliteBlobCatalog(SqliteTransport(path), now_ms=lambda: clock[0]) + policy = BlobRetentionPolicy("default", "server_stream", 10, 20, 0) + + class UploadAuthority: + def scope(self, _context): + return "tenant-a" + + def field(self, _context, _target): + return BlobFieldSpec("Asset", "document", ["pdf"], True, "default") + + async def authorize(self, _context, target, action, _input): + return target.entity == "Asset" and target.field == "document" and action in ( + "Begin", "Write", "Confirm") + + async def current_policy_on_lease(self, _context, lease, _target): + assert lease is not None + return policy + + class BindingAuthority(_BindingAuthority): + async def current_policy_on_lease(self, _context, lease, _target): + assert lease is not None + return policy + + class FixtureVerifier: + def verify(self, stream, extension, maximum_bytes): + data = stream.read(maximum_bytes + 1) + assert extension == "pdf" and len(data) <= maximum_bytes + assert data.startswith(b"%PDF-") and b"%%EOF" in data + assert stream.read(1) == b"" + return BlobVerifiedContent(len(data), sha256(data).hexdigest(), "application/pdf") + + upload = SqliteBlobUploadService(catalog, UploadAuthority(), FixtureVerifier(), 10_000, + now_ms=lambda: clock[0]) + maintenance = SqliteBlobMaintenanceService(catalog, _MaintenanceAuthority(), + now_ms=lambda: clock[0]) + module = BlobReferenceModule(catalog, upload, _ReadAuthority(), BindingAuthority(), + maintenance=maintenance) + async with aiosqlite.connect(path) as db: + await db.execute("CREATE TABLE asset (id INTEGER PRIMARY KEY, version INTEGER, document TEXT)") + await db.commit() + await catalog.ensure_schema() + module.install(context) + await module.ready(context) + + async def row(sql, params=()): + async with aiosqlite.connect(path) as db: + async with db.execute(sql, params) as cursor: + return await cursor.fetchone() + + try: + data = b"%PDF-1.7\nshared native Blob\n%%EOF" + session = await upload.begin(context, BlobUploadTarget("Asset", "1", "document"), + BlobUploadInput("shared.pdf", len(data), "application/pdf"), + "shared-native-blob") + assert await upload.write(context, session.upload_id, BytesIO(data)) == len(data) + assert await upload.complete(context, session.upload_id) == session.reference + object_key = ObjectKey((await row("SELECT object_key FROM teaql_blob_catalog WHERE key=?", + (session.reference.key,)))[0]) + object_store = context.object_storage("shared native Blob graph") + assert object_store.head(object_key) is not None + + ledger = EntityRoot() + provisional = [EntityKey("Asset", -1), EntityKey("Asset", -2)] + for key in provisional: + ledger.mark_as_new(key) + ledger.set(key, "document", session.reference) + + async def bind(graph): + ids = [] + for key in provisional: + allocated = await graph.transaction.next_id("Asset") + actual = EntityKey("Asset", allocated) + ledger.rekey(key, actual) + command = (InsertCommand.new("Asset").value("id", allocated) + .value("document", ledger.get(actual, "document"))) + await graph.transaction.mutate(graph.context, + graph.request(command, graph.scope(actual), ledger, actual)) + ids.append(allocated) + return ids + + ids = await context.execute_graph_save(bind, comment="bind shared Blob in two native rows") + assert len(ids) == 2 + assert await row("SELECT COUNT(*) FROM teaql_blob_binding WHERE key=?", + (session.reference.key,)) == (2,) + assert await row("SELECT status,last_unbound_at_ms FROM teaql_blob_catalog WHERE key=?", + (session.reference.key,)) == ("available", None) + + clock[0] += 40_000 + + async def unbind(id, outer_reject=False): + async def work(graph): + command = UpdateCommand.new("Asset", id).value("document", None) + await graph.transaction.mutate(graph.context, + graph.request(command, graph.scope(EntityKey("Asset", id)))) + if outer_reject: + raise RuntimeError("outer owner rejected final unbind") + await context.execute_graph_save(work, comment="unbind shared Blob row") + + await unbind(ids[0]) + assert await row("SELECT COUNT(*) FROM teaql_blob_binding WHERE key=?", + (session.reference.key,)) == (1,) + assert await row("SELECT last_unbound_at_ms FROM teaql_blob_catalog WHERE key=?", + (session.reference.key,)) == (None,) + assert (await maintenance.run_batch(context)).outcomes == () + assert object_store.head(object_key) is not None + + with pytest.raises(RuntimeError, match="outer owner rejected final unbind"): + await unbind(ids[1], outer_reject=True) + assert await row("SELECT document FROM asset WHERE id=?", (ids[1],)) == (session.reference.key,) + assert await row("SELECT COUNT(*) FROM teaql_blob_binding WHERE key=?", + (session.reference.key,)) == (1,) + assert await row("SELECT last_unbound_at_ms FROM teaql_blob_catalog WHERE key=?", + (session.reference.key,)) == (None,) + assert (await maintenance.run_batch(context)).outcomes == () + + await unbind(ids[1]) + assert await row("SELECT COUNT(*) FROM teaql_blob_binding WHERE key=?", + (session.reference.key,)) == (0,) + assert await row("SELECT last_unbound_at_ms FROM teaql_blob_catalog WHERE key=?", + (session.reference.key,)) == (clock[0],) + clock[0] += 19_999 + assert (await maintenance.run_batch(context)).outcomes == () + assert object_store.head(object_key) is not None + clock[0] += 1 + batch = await maintenance.run_batch(context) + assert [(item.reference.key, item.status) for item in batch.outcomes] == [ + (session.reference.key, "deleted")] + assert object_store.head(object_key) is None + assert await row("SELECT status FROM teaql_blob_catalog WHERE key=?", + (session.reference.key,)) == ("deleted",) + finally: + store.close() From fcb0fff0d47a3c50f0a2cb0b072dc5953ecaf730 Mon Sep 17 00:00:00 2001 From: Philip Z Date: Thu, 29 Oct 2026 08:39:06 +0800 Subject: [PATCH 65/75] test: cover shared Blob final unbind on real S3 --- tests/provider/sqlite/test_blob_module.py | 68 +++++++++++++++++++++-- 1 file changed, 64 insertions(+), 4 deletions(-) diff --git a/tests/provider/sqlite/test_blob_module.py b/tests/provider/sqlite/test_blob_module.py index 1bc7a1d0..1e3d8959 100644 --- a/tests/provider/sqlite/test_blob_module.py +++ b/tests/provider/sqlite/test_blob_module.py @@ -2,8 +2,10 @@ import os import time +from datetime import datetime, timedelta, timezone from hashlib import sha256 from io import BytesIO +from urllib.request import Request, urlopen import aiosqlite import pytest @@ -18,6 +20,10 @@ from teaql.provider.sqlite.dialect import SqliteDialect from teaql.provider.sqlite.transport import SqliteTransport from teaql.runtime import RuntimeModule, validate_capabilities +from teaql.runtime.view_preparation import ( + AuthorizedBlobSlot, AuthorizedResponseGraph, ViewOptions, + ViewPreparationError, ViewPreparationService, +) from teaql.sql.executor import SqlDataServiceExecutor from teaql_blob_reference import ( BlobReferenceModule, BlobRetentionPolicy, FileObjectStorage, SqliteBlobCatalog, @@ -27,8 +33,10 @@ class _ReadAuthority: + allowed = True + async def authorize(self, context, slot): - return True + return self.allowed class _UploadAuthority: @@ -421,6 +429,26 @@ async def assert_rollback(ids): @pytest.mark.asyncio async def test_native_shared_blob_final_unbind_controls_file_retention(tmp_path): + store = FileObjectStorage(tmp_path / "shared-private", os.urandom(32)) + await _run_native_shared_blob_final_unbind(tmp_path, store, real_s3=False) + + +@pytest.mark.asyncio +async def test_native_shared_blob_final_unbind_controls_real_s3_retention(tmp_path): + if os.environ.get("TEAQL_S3_TEST_SHARED_BLOB") != "1": + pytest.skip("explicit shared-reference S3 gate required") + endpoint = os.environ.get("TEAQL_S3_TEST_ENDPOINT") + bucket = os.environ.get("TEAQL_S3_TEST_BUCKET") + access = os.environ.get("TEAQL_S3_TEST_ACCESS_KEY") + secret = os.environ.get("TEAQL_S3_TEST_SECRET_KEY") + assert all((endpoint, bucket, access, secret)), "explicit S3 test configuration required" + from teaql.provider.s3 import S3ObjectStorage + store = S3ObjectStorage.connect(bucket, endpoint_url=endpoint, + aws_access_key_id=access, aws_secret_access_key=secret) + await _run_native_shared_blob_final_unbind(tmp_path, store, real_s3=True) + + +async def _run_native_shared_blob_final_unbind(tmp_path, store, real_s3): path = str(tmp_path / "shared-native.db") descriptor = EntityDescriptor("Asset").table_name("asset") descriptor.property(PropertyDescriptor("id", DataType.I64).is_id()) @@ -433,7 +461,6 @@ async def test_native_shared_blob_final_unbind_controls_file_retention(tmp_path) context.insert_resource("dataService", SqlDataServiceExecutor( SqliteDialect(), SqliteTransport(path), schema)) context.disable_sql_log() - store = FileObjectStorage(tmp_path / "shared-private", os.urandom(32)) context.with_object_storage(store).with_object_storage_observer(lambda event: None) clock = [1_700_000_000_000] catalog = SqliteBlobCatalog(SqliteTransport(path), now_ms=lambda: clock[0]) @@ -471,7 +498,8 @@ def verify(self, stream, extension, maximum_bytes): now_ms=lambda: clock[0]) maintenance = SqliteBlobMaintenanceService(catalog, _MaintenanceAuthority(), now_ms=lambda: clock[0]) - module = BlobReferenceModule(catalog, upload, _ReadAuthority(), BindingAuthority(), + read_permission = _ReadAuthority() + module = BlobReferenceModule(catalog, upload, read_permission, BindingAuthority(), maintenance=maintenance) async with aiosqlite.connect(path) as db: await db.execute("CREATE TABLE asset (id INTEGER PRIMARY KEY, version INTEGER, document TEXT)") @@ -485,6 +513,7 @@ async def row(sql, params=()): async with db.execute(sql, params) as cursor: return await cursor.fetchone() + object_key = None try: data = b"%PDF-1.7\nshared native Blob\n%%EOF" session = await upload.begin(context, BlobUploadTarget("Asset", "1", "document"), @@ -542,6 +571,27 @@ async def work(graph): assert (await maintenance.run_batch(context)).outcomes == () assert object_store.head(object_key) is not None + if real_s3: + class Authorizer: + async def authorize(self, _context, _result, _options): + slot = AuthorizedBlobSlot(("document",), "Asset", str(ids[1]), "document", + "tenant-a", 1, session.reference) + return AuthorizedResponseGraph({"document": session.reference}, (slot,)) + + context.insert_resource("response_graph_authorizer", Authorizer()) + options = ViewOptions("ordinary", "read_only", + datetime.now(timezone.utc) + timedelta(seconds=60)) + view = await ViewPreparationService().prepare(context, None, options) + link = view.presentation["document"] + assert link.startswith("http") and view.cache_control == "private, no-store" + with urlopen(Request(link, headers={"Range": "bytes=0-7"}), timeout=5) as response: + assert response.status == 206 and response.read() == data[:8] + read_permission.allowed = False + with pytest.raises(ViewPreparationError) as denied: + await ViewPreparationService().prepare(context, None, options) + assert denied.value.code == "BLOB_READ_DENIED" + read_permission.allowed = True + with pytest.raises(RuntimeError, match="outer owner rejected final unbind"): await unbind(ids[1], outer_reject=True) assert await row("SELECT document FROM asset WHERE id=?", (ids[1],)) == (session.reference.key,) @@ -556,6 +606,10 @@ async def work(graph): (session.reference.key,)) == (0,) assert await row("SELECT last_unbound_at_ms FROM teaql_blob_catalog WHERE key=?", (session.reference.key,)) == (clock[0],) + if real_s3: + with pytest.raises(ViewPreparationError) as stale_view: + await ViewPreparationService().prepare(context, None, options) + assert stale_view.value.code == "BLOB_READ_DENIED" clock[0] += 19_999 assert (await maintenance.run_batch(context)).outcomes == () assert object_store.head(object_key) is not None @@ -566,5 +620,11 @@ async def work(graph): assert object_store.head(object_key) is None assert await row("SELECT status FROM teaql_blob_catalog WHERE key=?", (session.reference.key,)) == ("deleted",) + if real_s3: + print("PASS Python shared S3 native graph/final unbind/authorized Range/retention GC") finally: - store.close() + try: + if real_s3 and object_key is not None: + context.object_storage("clean shared S3 Blob").delete(object_key) + finally: + store.close() From f7d9d34180f8fda0f8dde8361ec2f7a9b6afc239 Mon Sep 17 00:00:00 2001 From: Philip Z Date: Thu, 29 Oct 2026 17:58:21 +0800 Subject: [PATCH 66/75] fix(blob): verify Python business database and catalog assembly before readiness Refs #66 and teaql/teaql-conformance#241. Compare actual native SQLite main filenames via resource identity SPI, support same-file URI connections, reject wrong owner/dialect/database and catalog drift before verifier activation. Five red regressions fixed; full suite passes 958 with 23 conditional skips using isolated declared parsers. --- packages/teaql-blob-reference/README.md | 12 ++++ .../src/teaql_blob_reference/module.py | 19 +++++- src/teaql/provider/sqlite/transport.py | 21 +++++++ src/teaql/sql/executor.py | 14 +++++ tests/provider/sqlite/test_blob_module.py | 59 +++++++++++++++++++ 5 files changed, 122 insertions(+), 3 deletions(-) diff --git a/packages/teaql-blob-reference/README.md b/packages/teaql-blob-reference/README.md index 50323da2..e75c5f01 100644 --- a/packages/teaql-blob-reference/README.md +++ b/packages/teaql-blob-reference/README.md @@ -2,6 +2,18 @@ Optional Blob catalog and current-read services for TeaQL Python. Install this wheel alongside the matching `teaql` version. Catalog migration and runtime service installation are explicit host actions; this package does not install schema or expose routes on import. +`BlobReferenceModule.ready(context)` checks that the current native SQLite +business executor and catalog refer to the same database before activating the +content verifier. The executor's `uses_sql_database` SPI asks the transport for +identity; the native SQLite transport compares SQLite's actual `main` filename +from `PRAGMA database_list`, so ordinary paths and native file URIs can match. +An empty filename cannot establish a durable database for its independent +per-operation connections and is rejected. Missing/non-native owners, a different +dialect or file, and drift between the installed services' catalogs fail with +`BLOB_CAPABILITY_MISSING`. Separate native transports to the same file remain +supported. Installation stays passive; explicit readiness performs metadata +reads, schema validation and verifier readiness without migrating or uploading. + `SqliteBlobUploadService` offers a host-invoked, scope-bound upload journal. The host must provide current model/action authorization and a trusted parser that consumes every stored byte and returns actual format, size, and SHA256. The service writes through the context's object-storage provider, rechecks the stored generation before confirmation, and promotes the row to `uploaded_unbound` only after full verification. The included PDF verifier in tests is a fixture, not a production format parser. Other SQL backends and production ten-format parsing remain open. For permissions backed by mutable business rows, the host may implement `current_on_lease(actor, lease, target, action, input)` and return `BlobUploadAuthorizationState(scope, field, allowed)`. Begin, Write, and Confirm then recheck that state on the same physical SQLite transaction as the upload journal; a missing row, revoked permission, malformed state, or scope/field mismatch denies the operation. The ordinary `scope`, `field`, and `authorize` callbacks remain the preflight path and the compatibility path for hosts without this optional hook. diff --git a/packages/teaql-blob-reference/src/teaql_blob_reference/module.py b/packages/teaql-blob-reference/src/teaql_blob_reference/module.py index 94ca8e62..b89b83a1 100644 --- a/packages/teaql-blob-reference/src/teaql_blob_reference/module.py +++ b/packages/teaql-blob-reference/src/teaql_blob_reference/module.py @@ -1,6 +1,8 @@ """Passive assembly of the optional SQLite Blob reference services.""" from teaql.runtime.capability import install_runtime_capability +from teaql.sql.executor import SqlDataServiceExecutor +from teaql.sql.types import DatabaseKind from .read_service import SqliteBlobResponseContributor from .mutation_participant import SqliteBlobMutationParticipant @@ -16,15 +18,26 @@ def __init__(self, module: "BlobReferenceModule") -> None: self.module = module async def validate_ready(self, context) -> None: - if context.get_resource("blob_response_contributor") is not self.module.read or \ + if not isinstance(self.module.upload, SqliteBlobUploadService) or \ + self.module.upload.catalog is not self.module.catalog or \ + self.module.read.catalog is not self.module.catalog or \ + self.module.mutation.catalog is not self.module.catalog or \ + context.get_resource("blob_response_contributor") is not self.module.read or \ context.get_resource("blob_mutation_participant") is not self.module.mutation or \ context.get_resource("object_storage_provider") is None: raise RuntimeError("BLOB_CAPABILITY_MISSING") if self.module.maintenance is not None and \ - context.get_resource("blob_maintenance_service") is not self.module.maintenance: + (self.module.maintenance.catalog is not self.module.catalog or \ + context.get_resource("blob_maintenance_service") is not self.module.maintenance): raise RuntimeError("BLOB_CAPABILITY_MISSING") owner = context.get_resource("dataService") - if owner is None or not callable(getattr(owner, "begin", None)): + try: + compatible = (isinstance(owner, SqlDataServiceExecutor) and + owner.dialect.kind() == DatabaseKind.Sqlite and + await owner.uses_sql_database(self.module.catalog.transport)) + except Exception: + compatible = False + if not compatible: raise RuntimeError("BLOB_CAPABILITY_MISSING") await self.module.catalog.validate_schema() verifier_ready = getattr(self.module.upload.verifier, "validate_ready", None) diff --git a/src/teaql/provider/sqlite/transport.py b/src/teaql/provider/sqlite/transport.py index e35ff175..af8bce45 100644 --- a/src/teaql/provider/sqlite/transport.py +++ b/src/teaql/provider/sqlite/transport.py @@ -1,5 +1,6 @@ import json import aiosqlite +from pathlib import Path from decimal import Decimal from datetime import date, time from typing import List, Dict, Any, Optional, AsyncIterator @@ -36,6 +37,26 @@ def __init__(self, db_path: str): async def enable_soundex(self) -> None: self._soundex_enabled = True + async def shares_database(self, transport) -> bool: + """Compare SQLite's actual main file, including native file URI paths. + + Empty filenames (private or shared memory) cannot prove this provider's + independent per-operation connections share a durable database. + """ + if not isinstance(transport, SqliteTransport): + return False + + async def main_file(source): + rows = await source.fetch_all_sql(CompiledQuery("PRAGMA database_list", [])) + scalar = lambda value: getattr(value, "val", value) + files = [scalar(row.get("file")) for row in rows if scalar(row.get("name")) == "main"] + if len(files) != 1 or not isinstance(files[0], str) or not files[0]: + return None + return Path(files[0]).resolve() + + own_file = await main_file(self) + return own_file is not None and own_file == await main_file(transport) + async def _prepare(self, db: aiosqlite.Connection) -> None: if self._soundex_enabled: await db.create_function("soundex", 1, _soundex, deterministic=True) diff --git a/src/teaql/sql/executor.py b/src/teaql/sql/executor.py index 6ecd5518..10233da6 100644 --- a/src/teaql/sql/executor.py +++ b/src/teaql/sql/executor.py @@ -5,6 +5,7 @@ import asyncio import hashlib import logging +from inspect import isawaitable import time import threading from array import array @@ -160,6 +161,19 @@ def __init__(self, dialect: SqlDialect, transport: SqlTransport, schema_provider # executor still needs a stable, non-recycled cache isolation identity. self._id_set_source_identity = uuid4().hex + async def uses_sql_database(self, transport) -> bool: + """Ask the native transport for resource identity; no Blob dependency.""" + shares = getattr(self.transport, "shares_database", None) + if not callable(shares): + return False + try: + result = shares(transport) + if isawaitable(result): + result = await result + return result is True + except Exception: + return False + def _backend_name(self) -> str: kind = self.dialect.kind() name = getattr(kind, 'name', str(kind)).lower() diff --git a/tests/provider/sqlite/test_blob_module.py b/tests/provider/sqlite/test_blob_module.py index 1e3d8959..e371cd4f 100644 --- a/tests/provider/sqlite/test_blob_module.py +++ b/tests/provider/sqlite/test_blob_module.py @@ -5,6 +5,7 @@ from datetime import datetime, timedelta, timezone from hashlib import sha256 from io import BytesIO +from types import SimpleNamespace from urllib.request import Request, urlopen import aiosqlite @@ -133,6 +134,64 @@ def test_blob_module_rejects_an_upload_from_another_catalog(tmp_path): BlobReferenceModule(first, upload, _ReadAuthority(), _BindingAuthority()) +@pytest.mark.asyncio +@pytest.mark.parametrize("scenario", ["other-file", "separate-memory", "non-native", "wrong-dialect", "upload-catalog-drift"]) +async def test_blob_module_rejects_incompatible_business_database(tmp_path, scenario): + path = str(tmp_path / "catalog.db") + catalog = SqliteBlobCatalog(SqliteTransport(path)) + await catalog.ensure_schema() + verifier = _Verifier() + upload = SqliteBlobUploadService(catalog, _UploadAuthority(), verifier, 10_000) + module = BlobReferenceModule(catalog, upload, _ReadAuthority(), _BindingAuthority()) + owner = SqlDataServiceExecutor(SqliteDialect(), SqliteTransport(path), SimpleSchemaProvider()) + if scenario == "other-file": + owner.transport = SqliteTransport(str(tmp_path / "other.db")) + elif scenario == "separate-memory": + owner.transport = SqliteTransport(":memory:") + elif scenario == "non-native": + async def begin(_context): + raise AssertionError("readiness must not open this non-native owner") + owner = SimpleNamespace(begin=begin) + elif scenario == "wrong-dialect": + from teaql.provider.mysql.dialect import MysqlDialect + owner.dialect = MysqlDialect() + context = RuntimeModule.new().require_capability("blob.v1").into_context() + context.insert_resource("dataService", owner) + provider = FileObjectStorage(tmp_path / "objects", os.urandom(32)) + context.with_object_storage(provider) + module.install(context) + assert verifier.ready_calls == 0 + if scenario == "upload-catalog-drift": + upload.catalog = SqliteBlobCatalog(SqliteTransport(str(tmp_path / "other.db"))) + try: + with pytest.raises(RuntimeError, match="^BLOB_CAPABILITY_MISSING$"): + await module.ready(context) + assert verifier.ready_calls == 0 + finally: + provider.close() + + +@pytest.mark.asyncio +async def test_blob_module_accepts_same_database_through_native_file_uri(tmp_path): + path = tmp_path / "module.db" + catalog = SqliteBlobCatalog(SqliteTransport(str(path))) + await catalog.ensure_schema() + verifier = _Verifier() + upload = SqliteBlobUploadService(catalog, _UploadAuthority(), verifier, 10_000) + module = BlobReferenceModule(catalog, upload, _ReadAuthority(), _BindingAuthority()) + owner = SqlDataServiceExecutor(SqliteDialect(), SqliteTransport(path.as_uri() + "?mode=rwc"), SimpleSchemaProvider()) + context = RuntimeModule.new().require_capability("blob.v1").into_context() + context.insert_resource("dataService", owner) + provider = FileObjectStorage(tmp_path / "objects", os.urandom(32)) + context.with_object_storage(provider) + try: + module.install(context) + await module.ready(context) + assert verifier.ready_calls == 1 + finally: + provider.close() + + @pytest.mark.asyncio async def test_native_mutation_binds_blob_in_same_sqlite_graph(tmp_path): path = str(tmp_path / "native.db") From 4c2157dae296ce385150735af206273c56a3bdb5 Mon Sep 17 00:00:00 2001 From: Philip Z Date: Thu, 29 Oct 2026 19:06:17 +0800 Subject: [PATCH 67/75] test(blob): preserve unloaded fields in native ledger updates Refs #66 --- tests/provider/sqlite/test_blob_module.py | 35 ++++++++++++++++++++++- 1 file changed, 34 insertions(+), 1 deletion(-) diff --git a/tests/provider/sqlite/test_blob_module.py b/tests/provider/sqlite/test_blob_module.py index e371cd4f..4f38cd8d 100644 --- a/tests/provider/sqlite/test_blob_module.py +++ b/tests/provider/sqlite/test_blob_module.py @@ -198,6 +198,7 @@ async def test_native_mutation_binds_blob_in_same_sqlite_graph(tmp_path): descriptor = EntityDescriptor("Asset").table_name("asset") descriptor.property(PropertyDescriptor("id", DataType.I64).is_id()) descriptor.property(PropertyDescriptor("version", DataType.I64).is_version()) + descriptor.property(PropertyDescriptor("caption", DataType.Text)) descriptor.property(PropertyDescriptor("document", DataType.Text).blob( BlobFieldSpec("Asset", "document", ["pdf"], True, "default"))) schema = SimpleSchemaProvider() @@ -212,7 +213,7 @@ async def test_native_mutation_binds_blob_in_same_sqlite_graph(tmp_path): catalog = SqliteBlobCatalog(SqliteTransport(path)) await catalog.ensure_schema() async with aiosqlite.connect(path) as db: - await db.execute("CREATE TABLE asset (id INTEGER PRIMARY KEY, version INTEGER, document TEXT)") + await db.execute("CREATE TABLE asset (id INTEGER PRIMARY KEY, version INTEGER, document TEXT, caption TEXT)") for key in ("blob_native_one", "blob_native_two"): await db.execute("INSERT INTO teaql_blob_catalog " "(key,scope,object_key,content_version,file_name,content_type,size_bytes,status," @@ -347,6 +348,38 @@ async def work(graph): async with db.execute("SELECT status FROM teaql_blob_catalog WHERE key=?", (ledger_ref.key,)) as cursor: assert await cursor.fetchone() == ("available",) + async def snapshot(query, parameters): + async with aiosqlite.connect(path) as db: + async with db.execute(query, parameters) as cursor: + return await cursor.fetchall() + + binding_query = "SELECT * FROM teaql_blob_binding WHERE entity_id=?" + catalog_query = "SELECT * FROM teaql_blob_catalog WHERE key=?" + binding_before = await snapshot(binding_query, (str(allocated),)) + catalog_before = await snapshot(catalog_query, (ledger_ref.key,)) + ledger = EntityRoot() + actual = EntityKey("Asset", allocated) + ledger.set(actual, "caption", "after") + assigned = dict(ledger.current_change_set().changes())[actual] + assert "document" not in assigned + authority.allowed = False + + async def update_ordinary(graph): + command = (UpdateCommand.new("Asset", allocated).expected_version(1) + .value("caption", assigned["caption"])) + return await graph.transaction.mutate(graph.context, + graph.request(command, graph.scope(actual), ledger, actual)) + + await context.execute_graph_save(update_ordinary, comment="update caption with unloaded Blob") + assert await snapshot("SELECT document,caption,version FROM asset WHERE id=?", (allocated,)) == [ + (ledger_ref.key, "after", 2)] + assert await snapshot(binding_query, (str(allocated),)) == binding_before + assert await snapshot(catalog_query, (ledger_ref.key,)) == catalog_before + authority.allowed = True + await save(UpdateCommand.new("Asset", allocated).expected_version(2) + .value("document", None), "explicitly clear Blob", asset_id=allocated) + assert await snapshot("SELECT key,version FROM teaql_blob_binding WHERE entity_id=?", + (str(allocated),)) == [(None, 3)] store.close() From b29bdce5b543451e44ab539d64f2c04bebf9418b Mon Sep 17 00:00:00 2001 From: Philip Z Date: Thu, 29 Oct 2026 20:10:08 +0800 Subject: [PATCH 68/75] Refs #66: verify durable document recovery under current authority --- .../tests/test_sqlite_document_service.py | 63 ++++++++++++++++++- 1 file changed, 61 insertions(+), 2 deletions(-) diff --git a/packages/teaql-secure-document-reference/tests/test_sqlite_document_service.py b/packages/teaql-secure-document-reference/tests/test_sqlite_document_service.py index 7d5105b9..4e96a325 100644 --- a/packages/teaql-secure-document-reference/tests/test_sqlite_document_service.py +++ b/packages/teaql-secure-document-reference/tests/test_sqlite_document_service.py @@ -85,7 +85,7 @@ async def present(self, _context, document: ContextualDocument): raise RuntimeError("presentation failed") -async def fixture(tmp_path): +async def fixture(tmp_path, transport_factory=SqliteTransport): path = str(tmp_path / "secure-document.sqlite") async with aiosqlite.connect(path) as db: await db.execute("""CREATE TABLE document_order (id INTEGER PRIMARY KEY, @@ -94,7 +94,7 @@ async def fixture(tmp_path): await db.execute("INSERT INTO document_order VALUES (10,'ORD-10',1,1)") await db.execute("INSERT INTO document_permission VALUES ('alice',1)") await db.commit() - transport = SqliteTransport(path) + transport = transport_factory(path) context = RuntimeModule.new().into_context() context.insert_resource("dataService", SqlDataServiceExecutor( SqliteDialect(), transport, SimpleSchemaProvider())) @@ -123,6 +123,65 @@ async def receipts(path): return (await cursor.fetchone())[0] +@pytest.mark.asyncio +async def test_document_commit_unknown_replays_durable_receipt_under_current_authority(tmp_path): + class LostReplyTransport(SqliteTransport): + lose_next = False + + async def begin_sql(self): + transaction = await super().begin_sql() + if not self.lose_next: + return transaction + self.lose_next = False + + class CommitThenFail: + def __getattr__(self, name): + return getattr(transaction, name) + + async def commit_sql(self): + await transaction.commit_sql() + raise OSError("document commit acknowledgement unavailable") + + return CommitThenFail() + + transports = [] + def make_transport(path): + transport = LostReplyTransport(path) + transports.append(transport) + return transport + + path, context, host, service = await fixture(tmp_path, make_transport) + opened = await service.open(context, "Order", "ORD-10", "edit-order", 60) + command = SubmittedDocumentCommand("ORD-10", opened.document_token, + "command-commit-unknown", {"quantity": 3}) + transports[0].lose_next = True + with pytest.raises(OSError, match="commit acknowledgement unavailable"): + await service.accept(context, "Order", "edit-order", command) + assert await row(path) == (2, 3) and await receipts(path) == 1 + assert host.mutations == 1 + + async def durable_receipt(): + async with aiosqlite.connect(path) as db: + async with db.execute("SELECT * FROM teaql_document_command_receipt") as cursor: + return await cursor.fetchall() + + original = await durable_receipt() + async with aiosqlite.connect(path) as db: + await db.execute("UPDATE document_permission SET can_edit=0 WHERE subject='alice'") + await db.commit() + with pytest.raises(RuntimeError, match="DOCUMENT_AUTHORIZATION_REQUIRED"): + await service.accept(context, "Order", "edit-order", command) + async with aiosqlite.connect(path) as db: + await db.execute("UPDATE document_permission SET can_edit=1 WHERE subject='alice'") + await db.commit() + with pytest.raises(RuntimeError, match="DOCUMENT_REPLAY_CONFLICT"): + await service.accept(context, "Order", "edit-order", replace(command, body={"quantity": 4})) + recovered = await service.accept(context, "Order", "edit-order", command) + assert recovered.replayed and recovered.new_revision == 2 + assert host.mutations == 1 and await row(path) == (2, 3) + assert await durable_receipt() == original + + @pytest.mark.asyncio async def test_sqlite_open_accept_replay_authority_and_rollback(tmp_path): path, context, host, service = await fixture(tmp_path) From 319ea4a458a568c0eb3e41030bf56c2439cbf94c Mon Sep 17 00:00:00 2001 From: Philip Z Date: Thu, 29 Oct 2026 20:26:30 +0800 Subject: [PATCH 69/75] Refs #66: verify projection integrity and hidden document field preservation --- .../tests/test_document_round_trip.py | 86 +++++++++++++++++++ .../tests/test_sqlite_document_service.py | 6 ++ 2 files changed, 92 insertions(+) diff --git a/packages/teaql-secure-document-reference/tests/test_document_round_trip.py b/packages/teaql-secure-document-reference/tests/test_document_round_trip.py index 6789ef7c..d339a1cf 100644 --- a/packages/teaql-secure-document-reference/tests/test_document_round_trip.py +++ b/packages/teaql-secure-document-reference/tests/test_document_round_trip.py @@ -157,3 +157,89 @@ async def snapshot(self, _context, _graph): await ViewPreparationService().prepare(wrong, None, ViewOptions("governed", "committed")) assert (error.value.code, error.value.business_state, error.value.retry) == \ ("DOCUMENT_PROJECTION_VIOLATION", "committed", "read") + + +@pytest.mark.parametrize("scenario", ["writable-unloaded", "duplicate-row", "orphan-relation", + "duplicate-relation-row", "readonly-relation-mutation", "binding-unloaded", + "binding-readonly", "binding-foreign-entity", "duplicate-binding"]) +def test_rejects_inconsistent_issued_projection(scenario): + row, relation, binding = SNAPSHOT.entities[0], SNAPSHOT.relations[0], DOCUMENT.bindings[0] + if scenario == "writable-unloaded": row = replace(row, writable_fields=frozenset({"hidden"})) + if scenario == "orphan-relation": relation = replace(relation, row_keys=("absent",)) + if scenario == "duplicate-relation-row": relation = replace(relation, row_keys=("item:0", "item:0")) + if scenario == "readonly-relation-mutation": relation = replace(relation, writable=False) + if scenario == "binding-unloaded": + row = replace(row, loaded_fields=frozenset({"quantity"}), writable_fields=frozenset({"quantity"})) + binding = replace(binding, writable=False) + if scenario == "binding-readonly": row = replace(row, writable_fields=frozenset({"quantity"})) + if scenario == "binding-foreign-entity": binding = replace(binding, entity_type="OtherItem") + snapshot = replace(SNAPSHOT, entities=(row, row) if scenario == "duplicate-row" else (row,), + relations=(relation,)) + document = replace(DOCUMENT, snapshot=snapshot, + bindings=(binding, binding) if scenario == "duplicate-binding" else (binding,)) + assert code(lambda: runtime().issue_bound_document_snapshot(ALICE, document, 60)) == \ + "DOCUMENT_PROJECTION_VIOLATION" + + +@pytest.mark.parametrize("scenario", ["readonly-existing", "hidden-existing", "partial-child", + "readonly-child", "no-create-child"]) +def test_selection_cannot_expand_the_issued_surface(scenario): + document, target = DOCUMENT, TARGET + if scenario == "readonly-existing": + row = replace(SNAPSHOT.entities[0], writable_fields=frozenset({"quantity"})) + document = replace(DOCUMENT, snapshot=replace(SNAPSHOT, entities=(row,)), + bindings=(replace(DOCUMENT.bindings[0], writable=False),)) + elif scenario == "hidden-existing": target = replace(TARGET, canonical_field="hidden_attachment") + else: + target = replace(TARGET, kind="NEW_CLIENT_KEY", target_key="client-hidden") + relation = SNAPSHOT.relations[0] + if scenario == "partial-child": relation = replace(relation, completeness="PARTIAL") + if scenario == "readonly-child": relation = replace(relation, writable=False, allowed_mutations=frozenset()) + if scenario == "no-create-child": relation = replace(relation, allowed_mutations=frozenset({"REMOVE_CHILD"})) + document = replace(DOCUMENT, snapshot=replace(SNAPSHOT, relations=(relation,))) + module = runtime() + token = module.issue_bound_document_snapshot(ALICE, document, 60) + assert module.consume_bound_document_snapshot(ALICE, token, "Order", "edit-order").document == document + assert code(lambda: module.issue_blob_selection(ALICE, token, replace(CLAIM, target=target), 30)) == \ + "DOCUMENT_PROJECTION_VIOLATION" + + +@pytest.mark.asyncio +async def test_governed_view_hides_unlisted_blob_and_bounds_url_by_document_deadline(): + now = datetime.now(timezone.utc).replace(microsecond=0) + deadline = now + timedelta(seconds=60) + visible, hidden = BlobRef("blob_A"), BlobRef("blob_hidden") + source = {"attachment": visible, "hidden_attachment": hidden} + graph = AuthorizedResponseGraph({"attachment": visible}, ( + AuthorizedBlobSlot(("attachment",), "OrderItem", "20", "attachment", "tenant-a", 4, visible),)) + projected = [] + late = False + + class Authorizer: + async def authorize(self, *_): return graph + class Assembly: + def principal(self, _): return ALICE + async def snapshot(self, *_): + return PreparedBoundDocument(replace(DOCUMENT, + bindings=(replace(DOCUMENT.bindings[0], binding_version=4),)), 60) + class Projector: + async def project_batch(self, _context, authorized, ceiling): + assert ceiling == deadline + projected.extend(slot.reference.key for slot in authorized.blob_slots) + return (ProjectedBlobURL("https://files.example.invalid/asset", + deadline + timedelta(milliseconds=1) if late else deadline - timedelta(seconds=5)),) + + context = UserContext().insert_resource("response_graph_authorizer", Authorizer()) + context.insert_resource("blob_response_contributor", Projector()) + runtime(now=now).install(context, Assembly()) + options = ViewOptions("governed", "committed", now + timedelta(seconds=120)) + view = await ViewPreparationService().prepare(context, source, options) + assert view.presentation == {"attachment": "https://files.example.invalid/asset"} + assert view.expires_at == deadline - timedelta(seconds=5) + assert projected == ["blob_A"] and source["hidden_attachment"] is hidden + late = True + with pytest.raises(ViewPreparationError) as failure: + await ViewPreparationService().prepare(context, source, options) + assert failure.value.code == "BLOB_SIGN_FAILED" + assert failure.value.business_state == "committed" and failure.value.retry == "read" + assert projected == ["blob_A", "blob_A"] diff --git a/packages/teaql-secure-document-reference/tests/test_sqlite_document_service.py b/packages/teaql-secure-document-reference/tests/test_sqlite_document_service.py index 4e96a325..0b843446 100644 --- a/packages/teaql-secure-document-reference/tests/test_sqlite_document_service.py +++ b/packages/teaql-secure-document-reference/tests/test_sqlite_document_service.py @@ -151,6 +151,9 @@ def make_transport(path): return transport path, context, host, service = await fixture(tmp_path, make_transport) + async with aiosqlite.connect(path) as db: + await db.execute("ALTER TABLE document_order ADD COLUMN hidden_note TEXT NOT NULL DEFAULT 'server-only'") + await db.commit() opened = await service.open(context, "Order", "ORD-10", "edit-order", 60) command = SubmittedDocumentCommand("ORD-10", opened.document_token, "command-commit-unknown", {"quantity": 3}) @@ -180,6 +183,9 @@ async def durable_receipt(): assert recovered.replayed and recovered.new_revision == 2 assert host.mutations == 1 and await row(path) == (2, 3) assert await durable_receipt() == original + async with aiosqlite.connect(path) as db: + async with db.execute("SELECT hidden_note FROM document_order WHERE id=10") as cursor: + assert await cursor.fetchone() == ("server-only",) @pytest.mark.asyncio From 1e4f7f059e031f54dfd266ed62c29b94e9d16057 Mon Sep 17 00:00:00 2001 From: Philip Z Date: Thu, 29 Oct 2026 21:07:03 +0800 Subject: [PATCH 70/75] Refs #66: install declared parser extras in reference release CI --- .github/workflows/optional-blob-wheel.yml | 2 +- .github/workflows/release.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/optional-blob-wheel.yml b/.github/workflows/optional-blob-wheel.yml index 0670838a..e0baf391 100644 --- a/.github/workflows/optional-blob-wheel.yml +++ b/.github/workflows/optional-blob-wheel.yml @@ -15,7 +15,7 @@ jobs: cache: pip - run: python -m pip install --upgrade build - run: python -m pip install -e ".[test,object-storage-s3]" - - run: python -m pip install --no-deps -e ./packages/teaql-blob-reference + - run: python -m pip install -e "./packages/teaql-blob-reference[image,pdf,office]" - run: python -m pip install --no-deps -e ./packages/teaql-secure-document-reference - run: python -m pytest -q tests/provider/sqlite/test_blob_catalog_graph.py tests/provider/sqlite/test_file_blob_storage.py tests/provider/sqlite/test_blob_upload.py tests/provider/sqlite/test_blob_module.py - run: python -m pytest -q packages/teaql-secure-document-reference/tests diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index e994ca0d..ec1fecea 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -15,7 +15,7 @@ jobs: python-version: "3.12" - run: python -m pip install --upgrade build - run: python -m pip install --editable ".[test,telemetry]" - - run: python -m pip install --no-deps --editable ./packages/teaql-blob-reference + - run: python -m pip install --editable "./packages/teaql-blob-reference[image,pdf,office]" - run: python -m pip install --no-deps --editable ./packages/teaql-secure-document-reference - name: Verify tag matches package version run: test "${GITHUB_REF_NAME#python-v}" = "$(python -c 'import importlib.metadata; print(importlib.metadata.version("teaql"))')" From 778391f541c0e2aec9298e9127a4f542c08cdd6f Mon Sep 17 00:00:00 2001 From: Philip Z Date: Fri, 30 Oct 2026 09:30:42 +0800 Subject: [PATCH 71/75] fix(release): verify signed Python release tag (#66) Gate PyPI build and publication on GitHub-verified annotated tag identity. Retain native negative controls. Refs teaql-conformance#241. --- .github/workflows/release.yml | 8 +++ scripts/test_verify_signed_release_tag.py | 83 +++++++++++++++++++++++ scripts/verify_signed_release_tag.py | 73 ++++++++++++++++++++ 3 files changed, 164 insertions(+) create mode 100644 scripts/test_verify_signed_release_tag.py create mode 100644 scripts/verify_signed_release_tag.py diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index ec1fecea..b13c2386 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -8,6 +8,8 @@ on: jobs: build: runs-on: ubuntu-latest + permissions: + contents: read steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 @@ -19,6 +21,12 @@ jobs: - run: python -m pip install --no-deps --editable ./packages/teaql-secure-document-reference - name: Verify tag matches package version run: test "${GITHUB_REF_NAME#python-v}" = "$(python -c 'import importlib.metadata; print(importlib.metadata.version("teaql"))')" + - name: Require a verified signed release tag + env: + GITHUB_TOKEN: ${{ github.token }} + run: python scripts/verify_signed_release_tag.py + - name: Test signed release tag verifier + run: python scripts/test_verify_signed_release_tag.py - run: python -m pytest - run: python -m pytest packages/teaql-secure-document-reference/tests - run: python -m build --outdir dist . diff --git a/scripts/test_verify_signed_release_tag.py b/scripts/test_verify_signed_release_tag.py new file mode 100644 index 00000000..ba1ba95d --- /dev/null +++ b/scripts/test_verify_signed_release_tag.py @@ -0,0 +1,83 @@ +#!/usr/bin/env python3 +"""Exercise the actual signed-tag gate with valid and invalid Git objects.""" + +import importlib.util +from pathlib import Path +import unittest + + +path = Path(__file__).with_name("verify_signed_release_tag.py") +spec = importlib.util.spec_from_file_location("verify_signed_release_tag", path) +subject = importlib.util.module_from_spec(spec) +spec.loader.exec_module(subject) + +COMMIT = "a" * 40 +TAG_SHA = "b" * 40 +TAG = "python-v0.3.2" +BASE = "https://api.github.com/repos/teaql/teaql-python" + + +def valid_responses(): + return { + BASE + "/git/ref/tags/" + TAG: { + "ref": "refs/tags/" + TAG, "object": {"type": "tag", "sha": TAG_SHA}}, + BASE + "/git/tags/" + TAG_SHA: { + "sha": TAG_SHA, "tag": TAG, + "object": {"type": "commit", "sha": COMMIT}, + "verification": {"verified": True, "reason": "valid", "signature": "test-signature"}}, + } + + +class SignedReleaseTagTests(unittest.TestCase): + def run_case(self, responses): + requested = [] + def get(url): + requested.append(url) + return responses[url] + return subject.verify("teaql/teaql-python", TAG, COMMIT, get), requested + + def test_verified_annotated_tag_points_to_exact_release_commit(self): + tag_sha, requested = self.run_case(valid_responses()) + self.assertEqual(tag_sha, TAG_SHA) + self.assertEqual(requested, list(valid_responses())) + + def test_lightweight_tag_cannot_publish(self): + responses = valid_responses() + responses[BASE + "/git/ref/tags/" + TAG]["object"] = {"type": "commit", "sha": COMMIT} + with self.assertRaisesRegex(ValueError, "annotated tag"): + self.run_case(responses) + + def test_unsigned_annotated_tag_cannot_publish(self): + responses = valid_responses() + responses[BASE + "/git/tags/" + TAG_SHA]["verification"] = { + "verified": False, "reason": "unsigned", "signature": None} + with self.assertRaisesRegex(ValueError, "verified a signature"): + self.run_case(responses) + + def test_tag_for_another_commit_cannot_publish(self): + responses = valid_responses() + responses[BASE + "/git/tags/" + TAG_SHA]["object"]["sha"] = "c" * 40 + with self.assertRaisesRegex(ValueError, "release commit"): + self.run_case(responses) + + def test_changed_tag_reference_cannot_publish(self): + responses = valid_responses() + responses[BASE + "/git/ref/tags/" + TAG]["ref"] = "refs/tags/vother" + with self.assertRaisesRegex(ValueError, "different tag reference"): + self.run_case(responses) + + def test_unverified_signature_cannot_publish(self): + responses = valid_responses() + responses[BASE + "/git/tags/" + TAG_SHA]["verification"]["verified"] = False + with self.assertRaisesRegex(ValueError, "verified a signature"): + self.run_case(responses) + + def test_verified_flag_without_signature_cannot_publish(self): + responses = valid_responses() + responses[BASE + "/git/tags/" + TAG_SHA]["verification"]["signature"] = None + with self.assertRaisesRegex(ValueError, "verified a signature"): + self.run_case(responses) + + +if __name__ == "__main__": + unittest.main(verbosity=2) diff --git a/scripts/verify_signed_release_tag.py b/scripts/verify_signed_release_tag.py new file mode 100644 index 00000000..fc209958 --- /dev/null +++ b/scripts/verify_signed_release_tag.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python3 +"""Require the selected release tag to be a signed tag verified by GitHub.""" + +import json +import os +import re +import sys +from urllib.parse import quote +from urllib.request import Request, urlopen + + +def _sha(value): + return isinstance(value, str) and re.fullmatch(r"[0-9a-f]{40}|[0-9a-f]{64}", value) is not None + + +def fetch_json(url, token): + request = Request(url, headers={ + "Accept": "application/vnd.github+json", + "Authorization": "Bearer " + token, + "X-GitHub-Api-Version": "2022-11-28", + }) + with urlopen(request, timeout=20) as response: + if response.status != 200: + raise ValueError("GitHub tag API did not return HTTP 200") + return json.load(response) + + +def verify(repository, tag, commit, get_json): + if re.fullmatch(r"[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+", repository or "") is None: + raise ValueError("invalid GitHub repository") + if not tag or not tag.startswith("python-v") or any(ord(c) < 32 for c in tag): + raise ValueError("invalid release tag") + if not _sha(commit): + raise ValueError("invalid release commit SHA") + + base = "https://api.github.com/repos/" + repository + reference = get_json(base + "/git/ref/tags/" + quote(tag, safe="")) + if reference.get("ref") != "refs/tags/" + tag: + raise ValueError("GitHub returned a different tag reference") + target = reference.get("object") or {} + if target.get("type") != "tag" or not _sha(target.get("sha")): + raise ValueError("release tag is not an annotated tag object") + + tag_object = get_json(base + "/git/tags/" + target["sha"]) + if tag_object.get("sha") != target["sha"] or tag_object.get("tag") != tag: + raise ValueError("annotated tag object does not match its reference") + tagged_commit = tag_object.get("object") or {} + if tagged_commit.get("type") != "commit" or tagged_commit.get("sha") != commit: + raise ValueError("signed tag does not point to the release commit") + verification = tag_object.get("verification") or {} + if verification.get("verified") is not True or not verification.get("signature"): + raise ValueError("GitHub has not verified a signature on the release tag") + return target["sha"] + + +def main(): + try: + repository = os.environ["GITHUB_REPOSITORY"] + tag = os.environ["GITHUB_REF_NAME"] + commit = os.environ["GITHUB_SHA"] + token = os.environ["GITHUB_TOKEN"] + if not token: + raise ValueError("missing GitHub token") + tag_sha = verify(repository, tag, commit, lambda url: fetch_json(url, token)) + except (KeyError, ValueError, OSError) as error: + print("Release tag verification failed: " + str(error), file=sys.stderr) + return 1 + print("PASS verified signed tag " + tag + " at " + commit + " (" + tag_sha + ")") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) From 69723d1762bdf7cce69c76c2d9b9ad6781d56c43 Mon Sep 17 00:00:00 2001 From: Philip Z Date: Fri, 30 Oct 2026 09:46:24 +0800 Subject: [PATCH 72/75] fix(release): preflight all Python project versions on PyPI (#66) --- .github/workflows/release.yml | 11 +++ scripts/test_verify_public_release_absence.py | 92 +++++++++++++++++++ scripts/verify_public_release_absence.py | 85 +++++++++++++++++ 3 files changed, 188 insertions(+) create mode 100644 scripts/test_verify_public_release_absence.py create mode 100644 scripts/verify_public_release_absence.py diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index b13c2386..45862a11 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -27,6 +27,8 @@ jobs: run: python scripts/verify_signed_release_tag.py - name: Test signed release tag verifier run: python scripts/test_verify_signed_release_tag.py + - name: Test PyPI version-absence verifier + run: python scripts/test_verify_public_release_absence.py - run: python -m pytest - run: python -m pytest packages/teaql-secure-document-reference/tests - run: python -m build --outdir dist . @@ -50,10 +52,19 @@ jobs: name: pypi url: https://pypi.org/project/teaql/ permissions: + contents: read id-token: write steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" - uses: actions/download-artifact@v4 with: name: python-distributions path: dist/ + - name: Reverify exact downloaded distributions + run: python scripts/verify-reference-release-artifacts.py dist + - name: Refuse any occupied PyPI version + run: python scripts/verify_public_release_absence.py dist - uses: pypa/gh-action-pypi-publish@release/v1 diff --git a/scripts/test_verify_public_release_absence.py b/scripts/test_verify_public_release_absence.py new file mode 100644 index 00000000..a4137609 --- /dev/null +++ b/scripts/test_verify_public_release_absence.py @@ -0,0 +1,92 @@ +#!/usr/bin/env python3 +"""Exercise PyPI release-absence decisions without external network access.""" + +import importlib.util +from io import BytesIO +from pathlib import Path +import tempfile +import unittest +from urllib.error import HTTPError, URLError + + +SCRIPT = Path(__file__).with_name("verify_public_release_absence.py") +SPEC = importlib.util.spec_from_file_location("teaql_pypi_absence", SCRIPT) +MODULE = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(MODULE) +IDENTITIES = (("teaql", "9.8.7"), ("teaql-blob-reference", "9.8.7"), + ("teaql-secure-document-reference", "9.8.7")) + + +class Response(BytesIO): + def __init__(self, status): + super().__init__(b"") + self.status = status + + +class PyPIAbsenceTests(unittest.TestCase): + def setUp(self): + self.temporary = tempfile.TemporaryDirectory(prefix="teaql-pypi-absence-") + self.dist = Path(self.temporary.name) + for name, version in IDENTITIES: + (self.dist / f"{name.replace('-', '_')}-{version}-py3-none-any.whl").write_bytes(b"wheel") + (self.dist / f"{name}-{version}.tar.gz").write_bytes(b"sdist") + + def tearDown(self): + self.temporary.cleanup() + + def mock_opener(self, statuses, observed): + def open_request(request, timeout): + self.assertEqual(timeout, 15) + observed.append(request.full_url) + status = statuses.get(request.full_url, 404) + if isinstance(status, Exception): + raise status + if status >= 400: + raise HTTPError(request.full_url, status, "mock", None, None) + return Response(status) + return open_request + + def test_all_three_absent(self): + MODULE.require_exact_distribution_names(self.dist, IDENTITIES) + observed = [] + MODULE.require_absent(IDENTITIES, self.mock_opener({}, observed)) + self.assertEqual(observed, [f"https://pypi.org/pypi/{name}/{version}/json" + for name, version in IDENTITIES]) + + def test_occupied_non_base_project_blocks_publication(self): + observed = [] + url = "https://pypi.org/pypi/teaql-blob-reference/9.8.7/json" + with self.assertRaisesRegex(RuntimeError, "already exists"): + MODULE.require_absent(IDENTITIES, self.mock_opener({url: 200}, observed)) + self.assertEqual(observed[-1], url) + self.assertEqual(len(observed), 2) + + def test_registry_failure_fails_closed(self): + url = "https://pypi.org/pypi/teaql/9.8.7/json" + with self.assertRaisesRegex(RuntimeError, "Could not prove"): + MODULE.require_absent(IDENTITIES, self.mock_opener({url: 503}, [])) + + def test_network_failure_fails_closed(self): + url = "https://pypi.org/pypi/teaql/9.8.7/json" + with self.assertRaisesRegex(RuntimeError, "Could not prove"): + MODULE.require_absent(IDENTITIES, self.mock_opener({url: URLError("offline")}, [])) + + def test_unexpected_success_status_fails_closed(self): + url = "https://pypi.org/pypi/teaql/9.8.7/json" + with self.assertRaisesRegex(RuntimeError, "HTTP 204"): + MODULE.require_absent(IDENTITIES, self.mock_opener({url: 204}, [])) + + def test_exact_six_distributions_required(self): + (self.dist / "teaql-blob-reference-9.8.7.tar.gz").unlink() + with self.assertRaisesRegex(ValueError, "exactly three wheels"): + MODULE.require_exact_distribution_names(self.dist, IDENTITIES) + + def test_current_source_identifies_three_matching_projects(self): + identities = MODULE.release_identities() + self.assertEqual([name for name, _ in identities], + [name for name, _ in IDENTITIES]) + self.assertEqual(len({version for _, version in identities}), 1) + + +if __name__ == "__main__": + unittest.main(verbosity=2) diff --git a/scripts/verify_public_release_absence.py b/scripts/verify_public_release_absence.py new file mode 100644 index 00000000..ca221668 --- /dev/null +++ b/scripts/verify_public_release_absence.py @@ -0,0 +1,85 @@ +#!/usr/bin/env python3 +"""Fail closed if any exact TeaQL Python release version is already on PyPI.""" + +from pathlib import Path +import re +import sys +from urllib.error import HTTPError, URLError +from urllib.request import Request, urlopen + +try: + import tomllib +except ImportError: # Local Python 3.10; release CI uses Python 3.12. + import tomli as tomllib + + +ROOT = Path(__file__).resolve().parents[1] +PROJECTS = ( + ROOT / "pyproject.toml", + ROOT / "packages/teaql-blob-reference/pyproject.toml", + ROOT / "packages/teaql-secure-document-reference/pyproject.toml", +) + + +def release_identities(project_files=PROJECTS): + identities = [] + for project_file in project_files: + project = tomllib.loads(project_file.read_text())["project"] + name, version = project["name"], project["version"] + if (not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9._-]*", name) + or not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9._+-]*", version)): + raise ValueError(f"Invalid release identity in {project_file}") + identities.append((name, version)) + if len({name for name, _ in identities}) != 3 or len({version for _, version in identities}) != 1: + raise ValueError("Expected three distinct TeaQL projects with one release version") + return identities + + +def require_exact_distribution_names(dist, identities): + if not dist.is_dir(): + raise ValueError(f"Missing distribution directory: {dist}") + names = [path.name for path in dist.iterdir() + if path.is_file() and not path.is_symlink()] + if len(names) != 6 or len(list(dist.iterdir())) != 6: + raise ValueError("Expected exactly three wheels and three source distributions") + for name, version in identities: + wheel = name.replace("-", "_") + f"-{version}-py3-none-any.whl" + sdist = {f"{name}-{version}.tar.gz", f"{name.replace('-', '_')}-{version}.tar.gz"} + if names.count(wheel) != 1 or sum(names.count(item) for item in sdist) != 1: + raise ValueError(f"Missing exact distribution filenames for {name} {version}") + + +def require_absent(identities, opener=urlopen): + for name, version in identities: + url = f"https://pypi.org/pypi/{name}/{version}/json" + request = Request(url, headers={"Accept": "application/json", + "User-Agent": "teaql-release-preflight/1"}) + try: + with opener(request, timeout=15) as response: + status = response.status + except HTTPError as exc: + if exc.code == 404: + exc.close() + continue + raise RuntimeError(f"Could not prove {name} {version} absent on PyPI: HTTP {exc.code}") from exc + except (URLError, TimeoutError) as exc: + raise RuntimeError(f"Could not prove {name} {version} absent on PyPI") from exc + if status == 200: + raise RuntimeError(f"PyPI version already exists: {name} {version}") + raise RuntimeError(f"Could not prove {name} {version} absent on PyPI: HTTP {status}") + + +def main(argv): + if len(argv) != 2: + raise SystemExit("Usage: verify_public_release_absence.py ") + identities = release_identities() + require_exact_distribution_names(Path(argv[1]), identities) + require_absent(identities) + print("PASS all three TeaQL Python release versions are absent on PyPI") + + +if __name__ == "__main__": + try: + main(sys.argv) + except (OSError, KeyError, TypeError, ValueError, RuntimeError) as exc: + raise SystemExit(str(exc)) from exc From 0aaca244b54ef84565c4384092c96e1409e2de92 Mon Sep 17 00:00:00 2001 From: Philip Z Date: Fri, 30 Oct 2026 09:49:08 +0800 Subject: [PATCH 73/75] fix(release): verify exact PyPI distribution hashes after upload (#66) --- .github/workflows/release.yml | 2 + scripts/test_verify_public_release_absence.py | 90 ++++++++++++++++++- scripts/verify_public_release_absence.py | 73 +++++++++++++-- 3 files changed, 158 insertions(+), 7 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 45862a11..14828884 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -68,3 +68,5 @@ jobs: - name: Refuse any occupied PyPI version run: python scripts/verify_public_release_absence.py dist - uses: pypa/gh-action-pypi-publish@release/v1 + - name: Verify exact public wheel and source distribution bytes + run: python scripts/verify_public_release_absence.py --published dist diff --git a/scripts/test_verify_public_release_absence.py b/scripts/test_verify_public_release_absence.py index a4137609..2057991d 100644 --- a/scripts/test_verify_public_release_absence.py +++ b/scripts/test_verify_public_release_absence.py @@ -2,7 +2,9 @@ """Exercise PyPI release-absence decisions without external network access.""" import importlib.util +import hashlib from io import BytesIO +import json from pathlib import Path import tempfile import unittest @@ -18,8 +20,8 @@ class Response(BytesIO): - def __init__(self, status): - super().__init__(b"") + def __init__(self, status, body=b""): + super().__init__(body) self.status = status @@ -46,6 +48,33 @@ def open_request(request, timeout): return Response(status) return open_request + def release_payload(self, name, version): + files = [path for path in self.dist.iterdir() + if path.name.startswith(name.replace("-", "_") + f"-{version}-") + or path.name in (f"{name}-{version}.tar.gz", + f"{name.replace('-', '_')}-{version}.tar.gz")] + return {"info": {"name": name, "version": version}, + "urls": [{"filename": path.name, + "digests": {"sha256": hashlib.sha256(path.read_bytes()).hexdigest()}} + for path in files]} + + def published_opener(self, observed, replacements=None): + replacements = replacements or {} + def open_request(request, timeout): + self.assertEqual(timeout, 15) + observed.append(request.full_url) + parts = request.full_url.removeprefix("https://pypi.org/pypi/").split("/") + name, version = parts[0], parts[1] + value = replacements.get(request.full_url) + if isinstance(value, list): + value = value[min(observed.count(request.full_url) - 1, len(value) - 1)] + if isinstance(value, Exception): + raise value + if value is None: + value = self.release_payload(name, version) + return Response(200, json.dumps(value).encode()) + return open_request + def test_all_three_absent(self): MODULE.require_exact_distribution_names(self.dist, IDENTITIES) observed = [] @@ -87,6 +116,63 @@ def test_current_source_identifies_three_matching_projects(self): [name for name, _ in IDENTITIES]) self.assertEqual(len({version for _, version in identities}), 1) + def test_exact_three_published_release_file_sets(self): + observed = [] + MODULE.require_published_exact(self.dist, IDENTITIES, + self.published_opener(observed), lambda _: None) + self.assertEqual(len(observed), 3) + + def test_eventually_visible_release_files(self): + observed = [] + name, version = IDENTITIES[0] + url = f"https://pypi.org/pypi/{name}/{version}/json" + partial = self.release_payload(name, version) + partial["urls"] = partial["urls"][:1] + MODULE.require_published_exact(self.dist, IDENTITIES, + self.published_opener(observed, {url: [partial, self.release_payload(name, version)]}), + lambda _: None) + self.assertEqual(observed.count(url), 2) + + def test_published_digest_mismatch_rejected(self): + observed = [] + name, version = IDENTITIES[0] + url = f"https://pypi.org/pypi/{name}/{version}/json" + wrong = self.release_payload(name, version) + wrong["urls"][0]["digests"]["sha256"] = "0" * 64 + with self.assertRaisesRegex(RuntimeError, "digest mismatch"): + MODULE.require_published_exact(self.dist, IDENTITIES, + self.published_opener(observed, {url: wrong}), lambda _: None) + + def test_unexpected_published_file_rejected(self): + observed = [] + name, version = IDENTITIES[0] + url = f"https://pypi.org/pypi/{name}/{version}/json" + extra = self.release_payload(name, version) + extra["urls"].append({"filename": "unrecognized.whl", "digests": {"sha256": "0" * 64}}) + with self.assertRaisesRegex(RuntimeError, "Unexpected extra"): + MODULE.require_published_exact(self.dist, IDENTITIES, + self.published_opener(observed, {url: extra}), lambda _: None) + + def test_duplicate_published_filename_rejected(self): + observed = [] + name, version = IDENTITIES[0] + url = f"https://pypi.org/pypi/{name}/{version}/json" + duplicate = self.release_payload(name, version) + duplicate["urls"].append(dict(duplicate["urls"][0])) + with self.assertRaisesRegex(RuntimeError, "Duplicate"): + MODULE.require_published_exact(self.dist, IDENTITIES, + self.published_opener(observed, {url: duplicate}), lambda _: None) + + def test_published_release_never_appears_fails_closed(self): + observed = [] + name, version = IDENTITIES[0] + url = f"https://pypi.org/pypi/{name}/{version}/json" + absent = HTTPError(url, 404, "mock", None, None) + with self.assertRaisesRegex(RuntimeError, "incomplete after retries"): + MODULE.require_published_exact(self.dist, IDENTITIES, + self.published_opener(observed, {url: absent}), lambda _: None) + self.assertEqual(observed.count(url), 5) + if __name__ == "__main__": unittest.main(verbosity=2) diff --git a/scripts/verify_public_release_absence.py b/scripts/verify_public_release_absence.py index ca221668..8efc3062 100644 --- a/scripts/verify_public_release_absence.py +++ b/scripts/verify_public_release_absence.py @@ -1,9 +1,12 @@ #!/usr/bin/env python3 """Fail closed if any exact TeaQL Python release version is already on PyPI.""" +import hashlib +import json from pathlib import Path import re import sys +import time from urllib.error import HTTPError, URLError from urllib.request import Request, urlopen @@ -69,13 +72,73 @@ def require_absent(identities, opener=urlopen): raise RuntimeError(f"Could not prove {name} {version} absent on PyPI: HTTP {status}") +def require_published_exact(dist, identities, opener=urlopen, sleeper=time.sleep): + for name, version in identities: + expected = { + path.name: hashlib.sha256(path.read_bytes()).hexdigest() + for path in dist.iterdir() + if path.name.startswith(name.replace("-", "_") + f"-{version}-") + or path.name in (f"{name}-{version}.tar.gz", + f"{name.replace('-', '_')}-{version}.tar.gz") + } + if len(expected) != 2: + raise ValueError(f"Expected exact wheel and source distribution for {name} {version}") + url = f"https://pypi.org/pypi/{name}/{version}/json" + request = Request(url, headers={"Accept": "application/json", + "User-Agent": "teaql-release-integrity/1"}) + for attempt in range(5): + try: + with opener(request, timeout=15) as response: + if response.status != 200: + raise RuntimeError(f"Unexpected PyPI HTTP {response.status}: {name} {version}") + raw = response.read(1_000_001) + if len(raw) > 1_000_000: + raise RuntimeError(f"Oversized PyPI release metadata: {name} {version}") + payload = json.loads(raw) + actual = {item["filename"]: item["digests"]["sha256"] + for item in payload["urls"]} + if len(actual) != len(payload["urls"]): + raise RuntimeError(f"Duplicate PyPI release filename: {name} {version}") + if (payload["info"]["name"], payload["info"]["version"]) != (name, version): + raise RuntimeError(f"Unexpected PyPI release identity: {name} {version}") + if actual == expected: + break + if any(filename in actual and actual[filename] != digest + for filename, digest in expected.items()): + raise RuntimeError(f"PyPI release byte digest mismatch: {name} {version}") + if set(actual) - set(expected): + raise RuntimeError(f"Unexpected extra PyPI release file: {name} {version}") + except HTTPError as exc: + if exc.code != 404: + exc.close() + raise RuntimeError(f"Could not verify PyPI release: {name} {version}, HTTP {exc.code}") from exc + exc.close() + except (URLError, TimeoutError) as exc: + raise RuntimeError(f"Could not verify PyPI release: {name} {version}") from exc + except (KeyError, TypeError, ValueError) as exc: + raise RuntimeError(f"Malformed PyPI release metadata: {name} {version}") from exc + if attempt == 4: + raise RuntimeError(f"PyPI release incomplete after retries: {name} {version}") + sleeper(3) + print(f"PASS exact published PyPI bytes: {name} {version}") + + def main(argv): - if len(argv) != 2: - raise SystemExit("Usage: verify_public_release_absence.py ") + if len(argv) == 2: + published = False + dist = Path(argv[1]) + elif len(argv) == 3 and argv[1] == "--published": + published = True + dist = Path(argv[2]) + else: + raise SystemExit("Usage: verify_public_release_absence.py [--published] ") identities = release_identities() - require_exact_distribution_names(Path(argv[1]), identities) - require_absent(identities) - print("PASS all three TeaQL Python release versions are absent on PyPI") + require_exact_distribution_names(dist, identities) + if published: + require_published_exact(dist, identities) + else: + require_absent(identities) + print("PASS all three TeaQL Python release versions are absent on PyPI") if __name__ == "__main__": From e0cb6da7f788a08182875200dc40130ad0435a6e Mon Sep 17 00:00:00 2001 From: Philip Z Date: Fri, 30 Oct 2026 17:04:42 +0800 Subject: [PATCH 74/75] Fence Python legacy Blob mutations with before-row facts --- .../mutation_participant.py | 20 +++- src/teaql/sql/executor.py | 39 +++++- tests/provider/sqlite/test_blob_module.py | 113 ++++++++++++++++++ 3 files changed, 166 insertions(+), 6 deletions(-) diff --git a/packages/teaql-blob-reference/src/teaql_blob_reference/mutation_participant.py b/packages/teaql-blob-reference/src/teaql_blob_reference/mutation_participant.py index 58721921..0ff51690 100644 --- a/packages/teaql-blob-reference/src/teaql_blob_reference/mutation_participant.py +++ b/packages/teaql-blob-reference/src/teaql_blob_reference/mutation_participant.py @@ -18,6 +18,9 @@ def __init__(self, catalog: SqliteBlobCatalog, authorization): self.authorization = authorization async def apply(self, context, descriptor, mutation, persisted) -> None: + await self.apply_with_before(context, descriptor, mutation, persisted, None) + + async def apply_with_before(self, context, descriptor, mutation, persisted, before) -> None: graph = getattr(context, "_graph_session", None) if graph is None or context is not graph.context: raise RuntimeError("BLOB_TRANSACTION_REQUIRED") @@ -45,16 +48,16 @@ async def apply(self, context, descriptor, mutation, persisted) -> None: if not isinstance(scope, str) or not scope or scope != scope.strip(): raise RuntimeError("BLOB_BIND_DENIED") - async def prior_version(target): + async def prior_binding(target): async def query(sql): rows = await sql.fetch_all_sql(CompiledQuery( - "SELECT version FROM teaql_blob_binding WHERE scope = ? AND entity = ? " + "SELECT key,version FROM teaql_blob_binding WHERE scope = ? AND entity = ? " "AND entity_id = ? AND field = ?", [Value.Text(scope), Value.Text(target.entity), Value.Text(target.entity_id), Value.Text(target.field)])) if len(rows) > 1: raise RuntimeError("BLOB_CATALOG_FAILED") - return rows[0]["version"] if rows else None + return rows[0] if rows else None return await graph.with_sql_transport(query) for prop in descriptor.properties: @@ -68,14 +71,21 @@ async def query(sql): if action != "Create": continue target = BlobBindingTarget(descriptor._name, entity_id, prop.name) - prior = await prior_version(target) + binding = await prior_binding(target) + prior = binding["version"] if binding is not None else None if prior is not None and (type(prior) is not int or prior < 0): raise RuntimeError("BLOB_CATALOG_FAILED") if action == "Create" and prior is not None: raise RuntimeError("BLOB_BIND_DENIED") + if action != "Create": + if not isinstance(before, dict) or prop.name not in before or \ + before[prop.name] != (binding["key"] if binding is not None else None): + raise RuntimeError("BLOB_BIND_DENIED") if action == "Delete": if prior is None: - raise RuntimeError("BLOB_BIND_DENIED") + # A pre-binding nullable legacy slot has no catalog fact + # to clear. Its authoritative old value must be NULL. + continue next_ref = None else: value = mutation.values.get(prop.name, Value.Null()) diff --git a/src/teaql/sql/executor.py b/src/teaql/sql/executor.py index 10233da6..f33291b0 100644 --- a/src/teaql/sql/executor.py +++ b/src/teaql/sql/executor.py @@ -1096,6 +1096,39 @@ async def _mutate(self, context: 'UserContext', request: MutationRequest, privac except SqlCompileError as e: raise CompileError(e) + # A legacy row can predate the Blob binding table. Read its modeled Blob + # columns inside this mutation's SQL owner before an explicit update or + # delete, so a NULL slot can be handled without accepting a non-NULL + # orphan Key after the business row has changed or disappeared. + blob_fields = [prop for prop in entity_desc.properties + if getattr(prop, 'blob_spec', None) is not None] + before_blob_record = None + blob_participant = context.get_resource('blob_mutation_participant') if context is not None else None + if callable(getattr(blob_participant, 'apply_with_before', None)) and blob_fields and id_prop is not None and ( + isinstance(req_data, DeleteCommand) or + isinstance(req_data, UpdateCommand) and any( + prop.name in req_data.values for prop in blob_fields)): + columns = ", ".join( + self.dialect.quote_ident(prop.column_name_val) + if prop.column_name_val == prop.name else + f"{self.dialect.quote_ident(prop.column_name_val)} AS " + f"{self.dialect.quote_ident(prop.name)}" + for prop in blob_fields) + before_query = CompiledQuery( + f"SELECT {columns} FROM {self.dialect.quote_ident(entity_desc.table_name_val)} " + f"WHERE {self.dialect.quote_ident(id_prop.column_name_val)} = " + f"{self.dialect.placeholder(1)}", + [req_data.id], + parameter_log_policies=[self.dialect.field_log_policy(entity_desc, id_prop.name)], + sql_origin='generated') + try: + before_rows = await self.transport.fetch_all_sql(before_query) + except Exception: + raise RuntimeError('BLOB_CATALOG_FAILED') from None + if len(before_rows) > 1: + raise RuntimeError('BLOB_CATALOG_FAILED') + before_blob_record = before_rows[0] if before_rows else None + start = datetime.now() last_insert_id = None operation = {'insert': DataServiceOperation.Insert, 'update': DataServiceOperation.Update, @@ -1198,7 +1231,11 @@ async def _mutate(self, context: 'UserContext', request: MutationRequest, privac participant = context.get_resource('blob_mutation_participant') if not callable(getattr(participant, 'apply', None)): raise RuntimeError('BLOB_CAPABILITY_MISSING') - await participant.apply(context, entity_desc, req_data, persisted_record) + if callable(getattr(participant, 'apply_with_before', None)): + await participant.apply_with_before(context, entity_desc, req_data, + persisted_record, before_blob_record) + else: + await participant.apply(context, entity_desc, req_data, persisted_record) if affected_rows > 0 and context is not None: from teaql.runtime.audit import AuditFieldChange, MutationAuditKind, RawAuditEvent diff --git a/tests/provider/sqlite/test_blob_module.py b/tests/provider/sqlite/test_blob_module.py index 4f38cd8d..7713dea9 100644 --- a/tests/provider/sqlite/test_blob_module.py +++ b/tests/provider/sqlite/test_blob_module.py @@ -383,6 +383,119 @@ async def update_ordinary(graph): store.close() +@pytest.mark.asyncio +async def test_native_legacy_nullable_blob_slot_checks_authoritative_old_value(tmp_path): + path = str(tmp_path / "legacy-blob-slot.db") + descriptor = EntityDescriptor("Asset").table_name("asset") + descriptor.property(PropertyDescriptor("id", DataType.I64).is_id()) + descriptor.property(PropertyDescriptor("version", DataType.I64).is_version()) + descriptor.property(PropertyDescriptor("document", DataType.Text).blob( + BlobFieldSpec("Asset", "document", ["pdf"], True, "default"))) + schema = SimpleSchemaProvider() + schema.register_entity(descriptor) + context = RuntimeModule.new().entity(descriptor).require_capability("blob.v1").into_context() + context.insert_resource("dataService", SqlDataServiceExecutor( + SqliteDialect(), SqliteTransport(path), schema)) + store = FileObjectStorage(tmp_path / "legacy-private", os.urandom(32)) + context.with_object_storage(store) + catalog = SqliteBlobCatalog(SqliteTransport(path)) + await catalog.ensure_schema() + async with aiosqlite.connect(path) as db: + await db.execute("CREATE TABLE asset (id INTEGER PRIMARY KEY, version INTEGER, document TEXT)") + await db.executemany("INSERT INTO asset(id,version,document) VALUES (?,?,?)", + [(10, 1, None), (11, 1, "blob_old_orphan"), + (12, 1, None), (13, 1, "blob_business_old")]) + await db.execute("INSERT INTO teaql_blob_catalog " + "(key,scope,object_key,content_version,file_name,content_type,size_bytes,status) " + "VALUES (?,?,?,?,?,?,?,?)", + ("blob_catalog_other", "tenant-a", "objects/catalog-other", "v1", + "other.pdf", "application/pdf", 7, "available")) + await db.execute("INSERT INTO teaql_blob_binding " + "(scope,entity,entity_id,field,key,version) VALUES (?,?,?,?,?,?)", + ("tenant-a", "Asset", "13", "document", "blob_catalog_other", 1)) + await db.commit() + module = BlobReferenceModule(catalog, + SqliteBlobUploadService(catalog, _UploadAuthority(), _Verifier(), 10_000), + _ReadAuthority(), _BindingAuthority()) + module.install(context) + await module.ready(context) + + async def save(command, asset_id): + async def work(graph): + return await graph.transaction.mutate(graph.context, + graph.request(command, graph.scope(EntityKey("Asset", asset_id)))) + return await context.execute_graph_save(work, comment="legacy nullable Blob mutation") + + with pytest.raises(RuntimeError, match="BLOB_BIND_DENIED"): + await save(UpdateCommand.new("Asset", 11).expected_version(1) + .value("document", None), 11) + with pytest.raises(RuntimeError, match="BLOB_BIND_DENIED"): + await save(DeleteCommand.new("Asset", 11).expected_version(1).hard_delete(), 11) + with pytest.raises(RuntimeError, match="BLOB_BIND_DENIED"): + await save(UpdateCommand.new("Asset", 13).expected_version(1) + .value("document", None), 13) + async with aiosqlite.connect(path) as db: + async with db.execute("SELECT id,version,document FROM asset WHERE id=11") as cursor: + assert await cursor.fetchone() == (11, 1, "blob_old_orphan") + async with db.execute("SELECT count(*) FROM teaql_blob_binding") as cursor: + assert await cursor.fetchone() == (1,) + async with db.execute("SELECT key,version FROM teaql_blob_binding WHERE entity_id='13'") as cursor: + assert await cursor.fetchone() == ("blob_catalog_other", 1) + + await save(UpdateCommand.new("Asset", 12).expected_version(1) + .value("document", None), 12) + async with aiosqlite.connect(path) as db: + async with db.execute("SELECT key,version FROM teaql_blob_binding WHERE entity_id='12'") as cursor: + assert await cursor.fetchone() == (None, 2) + await save(DeleteCommand.new("Asset", 10).expected_version(1).hard_delete(), 10) + async with aiosqlite.connect(path) as db: + async with db.execute("SELECT id FROM asset WHERE id=10") as cursor: + assert await cursor.fetchone() is None + async with db.execute("SELECT count(*) FROM teaql_blob_binding WHERE entity_id='10'") as cursor: + assert await cursor.fetchone() == (0,) + store.close() + + +@pytest.mark.asyncio +async def test_native_executor_keeps_four_argument_blob_participant_contract(tmp_path): + path = str(tmp_path / "custom-participant.db") + descriptor = EntityDescriptor("Asset").table_name("asset") + descriptor.property(PropertyDescriptor("id", DataType.I64).is_id()) + descriptor.property(PropertyDescriptor("version", DataType.I64).is_version()) + descriptor.property(PropertyDescriptor("document", DataType.Text).blob( + BlobFieldSpec("Asset", "document", ["pdf"], True, "default"))) + schema = SimpleSchemaProvider() + schema.register_entity(descriptor) + context = RuntimeModule.new().entity(descriptor).into_context() + context.insert_resource("dataService", SqlDataServiceExecutor( + SqliteDialect(), SqliteTransport(path), schema)) + async with aiosqlite.connect(path) as db: + await db.execute("CREATE TABLE asset (id INTEGER PRIMARY KEY, version INTEGER, document TEXT)") + await db.commit() + + class ExistingParticipant: + def __init__(self): + self.calls = 0 + + async def apply(self, _context, _descriptor, _mutation, persisted): + assert persisted["document"] is None + self.calls += 1 + + participant = ExistingParticipant() + context.insert_resource("blob_mutation_participant", participant) + + async def save(graph): + command = InsertCommand.new("Asset").value("id", 1).value("document", None) + return await graph.transaction.mutate(graph.context, + graph.request(command, graph.scope(EntityKey("Asset", 1)))) + + await context.execute_graph_save(save, comment="four argument Blob participant") + assert participant.calls == 1 + async with aiosqlite.connect(path) as db: + async with db.execute("SELECT id,document FROM asset") as cursor: + assert await cursor.fetchone() == (1, None) + + @pytest.mark.asyncio async def test_native_blob_multi_entity_graph_rolls_back_together(tmp_path): path = str(tmp_path / "multi-native.db") From 5b60c27310cd304c921e8e99940968c4dc0eab8b Mon Sep 17 00:00:00 2001 From: Philip Z Date: Fri, 30 Oct 2026 19:25:25 +0800 Subject: [PATCH 75/75] Reject unsafe raw whitespace in projected Blob URLs --- src/teaql/runtime/view_preparation.py | 5 ++++ tests/test_view_preparation_blob.py | 34 +++++++++++++++++++++++++++ 2 files changed, 39 insertions(+) diff --git a/src/teaql/runtime/view_preparation.py b/src/teaql/runtime/view_preparation.py index f982f3b0..fd61203f 100644 --- a/src/teaql/runtime/view_preparation.py +++ b/src/teaql/runtime/view_preparation.py @@ -4,6 +4,7 @@ from dataclasses import dataclass from datetime import datetime, timezone +import re from typing import Any, Protocol from urllib.parse import urlsplit @@ -222,6 +223,10 @@ async def prepare(self, context: UserContext, result: Any, if len(urls) != len(graph.blob_slots): _fail("BLOB_SIGN_FAILED") for slot, signed in zip(graph.blob_slots, urls): + # urlsplit strips or normalizes raw whitespace; reject the original value. + if (not isinstance(signed.url, str) + or re.search(r"[\s\x00-\x1f\x7f-\x9f]", signed.url)): + _fail("BLOB_SIGN_FAILED") parsed = urlsplit(signed.url) if (parsed.scheme not in ("http", "https") or not parsed.hostname or not _future(signed.expires_at) diff --git a/tests/test_view_preparation_blob.py b/tests/test_view_preparation_blob.py index da89eeb9..3dad41f7 100644 --- a/tests/test_view_preparation_blob.py +++ b/tests/test_view_preparation_blob.py @@ -124,6 +124,40 @@ async def project_batch(self, _context, _graph, _expiry): assert "secret.invalid" not in str(second.value) +@pytest.mark.asyncio +async def test_provider_url_with_raw_whitespace_or_control_fails_after_commit(): + reference = BlobRef("blob_asset_1") + + class Authorizer: + async def authorize(self, _context, _result, _options): + return AuthorizedResponseGraph({"document": reference}, ( + AuthorizedBlobSlot(("document",), "Asset", "1", "document", + "tenant-a", 4, reference),)) + + class Blob: + url = "" + + async def project_batch(self, _context, _graph, _expiry): + return (ProjectedBlobURL(self.url, + datetime.now(timezone.utc) + timedelta(seconds=30)),) + + blob = Blob() + context = UserContext().insert_resource("response_graph_authorizer", Authorizer()) + context.insert_resource("blob_response_contributor", blob) + for invalid in (" https://files.example.invalid/read", + "https://files.example.invalid/read file", + "https://files.example.invalid/read\n", + "https://files.example.invalid/\tread", + "https://files.example.invalid/read\u2028next", + "https://files.example.invalid/read\u2029next"): + blob.url = invalid + with pytest.raises(ViewPreparationError) as failure: + await ViewPreparationService().prepare(context, None, + ViewOptions(mode="ordinary", business_outcome="committed")) + assert failure.value.code == "BLOB_SIGN_FAILED" + assert failure.value.business_state == "committed" + + @pytest.mark.asyncio async def test_invalid_or_expired_view_options_are_rejected_before_authorization(): context = UserContext()