diff --git a/.gitattributes b/.gitattributes index d4a292f1..a5a4a7b3 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1 +1,3 @@ test-vectors/*.tsv whitespace=-blank-at-eol +tests/fixtures/reference-modules/*.doc binary +tests/fixtures/reference-modules/*.xls binary diff --git a/.github/workflows/example-gate.yml b/.github/workflows/example-gate.yml index c44661b1..7c5cd67a 100644 --- a/.github/workflows/example-gate.yml +++ b/.github/workflows/example-gate.yml @@ -21,6 +21,7 @@ jobs: python-version: '3.10' cache: pip - run: pip install -e ".[test,telemetry,object-storage-s3]" + - run: pip install --no-deps -e ./packages/teaql-blob-reference - name: Start dedicated S3 example fixture run: | docker run -d --name teaql-python-example-s3 \ diff --git a/.github/workflows/optional-blob-wheel.yml b/.github/workflows/optional-blob-wheel.yml new file mode 100644 index 00000000..e0baf391 --- /dev/null +++ b/.github/workflows/optional-blob-wheel.yml @@ -0,0 +1,58 @@ +name: Optional reference wheels +on: + pull_request: + push: + branches: [main] + workflow_dispatch: +jobs: + optional-wheel: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.10' + cache: pip + - run: python -m pip install --upgrade build + - run: python -m pip install -e ".[test,object-storage-s3]" + - run: python -m pip install -e "./packages/teaql-blob-reference[image,pdf,office]" + - run: python -m pip install --no-deps -e ./packages/teaql-secure-document-reference + - run: python -m pytest -q tests/provider/sqlite/test_blob_catalog_graph.py tests/provider/sqlite/test_file_blob_storage.py tests/provider/sqlite/test_blob_upload.py tests/provider/sqlite/test_blob_module.py + - run: python -m pytest -q packages/teaql-secure-document-reference/tests + - run: python -m build --wheel --no-isolation --outdir .artifacts/base . + - run: python -m build --wheel --no-isolation --outdir .artifacts/blob ./packages/teaql-blob-reference + - run: python -m build --wheel --no-isolation --outdir .artifacts/secure ./packages/teaql-secure-document-reference + - name: Check wheel isolation + run: | + python - <<'PY' + from pathlib import Path + from zipfile import ZipFile + + base = list(Path('.artifacts/base').glob('*.whl')) + blob = list(Path('.artifacts/blob').glob('*.whl')) + secure = list(Path('.artifacts/secure').glob('*.whl')) + assert len(base) == len(blob) == len(secure) == 1, (base, blob, secure) + with ZipFile(base[0]) as wheel: + entries = wheel.namelist() + assert any(name.startswith('teaql/') for name in entries) + assert not any(name.startswith('teaql_blob_reference/') for name in entries) + assert not any(name.startswith('teaql_secure_document_reference/') for name in entries) + with ZipFile(blob[0]) as wheel: + entries = wheel.namelist() + assert any(name.startswith('teaql_blob_reference/') for name in entries) + assert not any(name.startswith('teaql/') for name in entries) + assert not any(name.startswith('teaql_secure_document_reference/') for name in entries) + for module in ('__init__.py', 'module.py', 'mutation_participant.py', + 'file_storage.py', 'sqlite_catalog.py', 'upload_service.py'): + name = f'teaql_blob_reference/{module}' + assert name in entries + assert wheel.read(name) == (Path('packages/teaql-blob-reference/src') / name).read_bytes() + with ZipFile(secure[0]) as wheel: + entries = wheel.namelist() + assert any(name.startswith('teaql_secure_document_reference/') for name in entries) + assert not any(name.startswith('teaql/') or name.startswith('teaql_blob_reference/') for name in entries) + for module in ('__init__.py', 'round_trip_reference.py', 'document_round_trip.py'): + name = f'teaql_secure_document_reference/{module}' + assert name in entries + assert wheel.read(name) == (Path('packages/teaql-secure-document-reference/src') / name).read_bytes() + PY diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index d90b5522..14828884 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -8,6 +8,8 @@ on: jobs: build: runs-on: ubuntu-latest + permissions: + contents: read steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 @@ -15,10 +17,29 @@ jobs: python-version: "3.12" - run: python -m pip install --upgrade build - run: python -m pip install --editable ".[test,telemetry]" + - run: python -m pip install --editable "./packages/teaql-blob-reference[image,pdf,office]" + - run: python -m pip install --no-deps --editable ./packages/teaql-secure-document-reference - name: Verify tag matches package version run: test "${GITHUB_REF_NAME#python-v}" = "$(python -c 'import importlib.metadata; print(importlib.metadata.version("teaql"))')" + - name: Require a verified signed release tag + env: + GITHUB_TOKEN: ${{ github.token }} + run: python scripts/verify_signed_release_tag.py + - name: Test signed release tag verifier + run: python scripts/test_verify_signed_release_tag.py + - name: Test PyPI version-absence verifier + run: python scripts/test_verify_public_release_absence.py - run: python -m pytest - - run: python -m build + - run: python -m pytest packages/teaql-secure-document-reference/tests + - run: python -m build --outdir dist . + - run: python -m build --outdir dist ./packages/teaql-blob-reference + - run: python -m build --outdir dist ./packages/teaql-secure-document-reference + - name: Verify exact release wheels and source distributions + run: python scripts/verify-reference-release-artifacts.py dist + - name: Run four isolated consumers from exact wheels + run: python scripts/verify-reference-release-consumers.py dist + - name: Resolve and install exact wheels with declared dependencies in clean environments + run: python scripts/verify-reference-release-dependencies.py dist - uses: actions/upload-artifact@v4 with: name: python-distributions @@ -31,10 +52,21 @@ jobs: name: pypi url: https://pypi.org/project/teaql/ permissions: + contents: read id-token: write steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" - uses: actions/download-artifact@v4 with: name: python-distributions path: dist/ + - name: Reverify exact downloaded distributions + run: python scripts/verify-reference-release-artifacts.py dist + - name: Refuse any occupied PyPI version + run: python scripts/verify_public_release_absence.py dist - uses: pypa/gh-action-pypi-publish@release/v1 + - name: Verify exact public wheel and source distribution bytes + run: python scripts/verify_public_release_absence.py --published dist diff --git a/README.md b/README.md index c7547a9c..44766e34 100644 --- a/README.md +++ b/README.md @@ -424,7 +424,7 @@ distinguish implemented source, executed examples, artifact verification and rel ```bash python -m pip install -e '.[test]' -PYTHONDONTWRITEBYTECODE=1 PYTHONPATH=src python -m pytest -q +PYTHONDONTWRITEBYTECODE=1 PYTHONPATH=src:packages/teaql-blob-reference/src python -m pytest -q PYTHONDONTWRITEBYTECODE=1 bash scripts/verify-examples.sh ``` diff --git a/packages/teaql-blob-reference/README.md b/packages/teaql-blob-reference/README.md new file mode 100644 index 00000000..e75c5f01 --- /dev/null +++ b/packages/teaql-blob-reference/README.md @@ -0,0 +1,99 @@ +# teaql-blob-reference + +Optional Blob catalog and current-read services for TeaQL Python. Install this wheel alongside the matching `teaql` version. Catalog migration and runtime service installation are explicit host actions; this package does not install schema or expose routes on import. + +`BlobReferenceModule.ready(context)` checks that the current native SQLite +business executor and catalog refer to the same database before activating the +content verifier. The executor's `uses_sql_database` SPI asks the transport for +identity; the native SQLite transport compares SQLite's actual `main` filename +from `PRAGMA database_list`, so ordinary paths and native file URIs can match. +An empty filename cannot establish a durable database for its independent +per-operation connections and is rejected. Missing/non-native owners, a different +dialect or file, and drift between the installed services' catalogs fail with +`BLOB_CAPABILITY_MISSING`. Separate native transports to the same file remain +supported. Installation stays passive; explicit readiness performs metadata +reads, schema validation and verifier readiness without migrating or uploading. + +`SqliteBlobUploadService` offers a host-invoked, scope-bound upload journal. The host must provide current model/action authorization and a trusted parser that consumes every stored byte and returns actual format, size, and SHA256. The service writes through the context's object-storage provider, rechecks the stored generation before confirmation, and promotes the row to `uploaded_unbound` only after full verification. The included PDF verifier in tests is a fixture, not a production format parser. Other SQL backends and production ten-format parsing remain open. + +For permissions backed by mutable business rows, the host may implement `current_on_lease(actor, lease, target, action, input)` and return `BlobUploadAuthorizationState(scope, field, allowed)`. Begin, Write, and Confirm then recheck that state on the same physical SQLite transaction as the upload journal; a missing row, revoked permission, malformed state, or scope/field mismatch denies the operation. The ordinary `scope`, `field`, and `authorize` callbacks remain the preflight path and the compatibility path for hosts without this optional hook. + +To use several optional parsers in one upload service, pass a +`CompositeBlobContentVerifier` with explicit extension-to-parser routes. It +copies and validates the route table at construction, calls each distinct +parser's readiness check once, and rejects unregistered formats before reading +stored bytes. It does not import optional decoder packages or add formats by +itself; install only the image/PDF/Office/media extras the host selects. + +For retention-aware uploads, the host may implement `current_policy_on_lease(actor, lease, target)` on its upload authorization and return an immutable `BlobRetentionPolicy`. The service obtains this trusted policy on the physical SQLite begin lease and snapshots its profile, upload mode and retention durations into the catalog. It rejects direct mode on this server-stream path. Confirmation records `completed_at_ms`; replay never changes the accepted policy or completion time. Older rows remain NULL-policy after migration and are excluded from retention collection. A binding authorization must provide its own current policy on the graph lease; new bindings of unknown-policy rows fail closed. Successful binding preserves the longest accepted unreferenced duration and minimum retention deadline. The final unbind records `last_unbound_at_ms` while the row remains `available`, so its lifetime is measured from the actual final unbind. The shared policy vectors are pinned by SHA-256 in the tests. + +`SqliteBlobMaintenanceService` is an optional, explicitly host-scheduled service for expired incomplete uploads and policy-backed completed, unbound server-stream objects. It includes formerly bound objects only after the final unbind deadline and retention floor. Active bindings, unknown-policy legacy rows and direct uploads are excluded. The host supplies a current scope and async `can_maintain` policy, then calls `run_batch(context, cursor)` and may install the service through `BlobReferenceModule(..., maintenance=service)`. Run `catalog.ensure_schema()` as a separate migration before readiness; it adds a unique object-key index and nullable policy/time columns. Cleanup claims rows under SQLite `BEGIN IMMEDIATE`, pins the observed ETag, uses only version-conditional storage deletion, and can resume after a failed final catalog acknowledgment. `FileObjectStorage` guards manifest publication and conditional deletion with a persistent per-key OS file lock. Its ETag includes an immutable generation token, so identical-byte replacement cannot match an older version. A matching delete removes the current data generation and manifest. The host may separately call `FileObjectStorage.sweep_old_generations(key)` for a known object key to remove inaccessible older immutable data generations; it waits for same-key writers and preserves the current manifest generation. Use this only after all processes using that private root run a writer-lock-capable version, and schedule/retry it separately from the catalog receipt. Keep each Blob object key unique and immutable, including across manual S3 writes. Direct-upload quiescence and cross-machine S3 retention remain outside this service. + +`SqliteBlobFileGenerationSweepService` supplies a separate, host-scheduled +catalog-driven File sweep. After every process sharing the private File root +has upgraded to the writer-lock protocol, the host constructs it with +`all_writers_upgraded=True` and calls `run_batch(actor, provider, cursor)` +outside graph SQL. It pages only the authorized scope's `server_stream` rows, +including `deleted` rows that may retain inaccessible old data generations. +The provider must be the exact `FileObjectStorage` installed in the current +context. Each outcome reports removed generations or a sanitized failure; +restart the scan from the beginning to retry a failed row. Keep terminal +catalog rows until the File sweep has completed. The host assertion cannot +detect an older external writer on its own; no worker or timer starts at import. + +For actual JPEG/PNG parsing, install `teaql-blob-reference[image]` and inject +`teaql_blob_reference.content_pillow.PillowBlobContentVerifier`. The image extra +pins Pillow 12.3.0, the newest release supporting Python 3.10 at this checkpoint. +The adapter validates complete JPEG/PNG containers and PNG CRCs, checks the +decoder's real format and loads all pixels with explicit input, pixel and +concurrent-work bounds. It has no hard CPU/process isolation. Construction is +passive; `BlobReferenceModule.ready` invokes its optional decoder readiness check. +It accepts `jpg`, `jpeg` and `png` only. The other formats need separate +production adapters; the earlier PDF fixture test does not establish them. + +For actual PDF parsing, install `teaql-blob-reference[pdf]` and inject +`teaql_blob_reference.content_pypdf.PypdfBlobContentVerifier`. The PDF extra +pins pypdf 6.21.3. This adapter checks a complete PDF container with strict +cross-reference parsing, a real page tree and content operators, and bounded +input, pages, decoded page content and concurrent work. It rejects encrypted +PDFs. This is parser validation, not malware scanning or hard CPU/process +isolation. The PDF fixture verifier used in older S3 tests remains test-only. + +For actual DOCX/XLSX parsing, install `teaql-blob-reference[office]` and inject +`teaql_blob_reference.content_office.OfficeBlobContentVerifier`. The office extra +pins python-docx 1.2.0 and openpyxl 3.1.5. The adapter checks every ZIP member +and its CRC under input, decoded-byte and part-count limits, rejects XML entity +declarations, checks the expected +Office main content type and package relationship, then parses the Word body or +every Excel worksheet under an element/cell count limit. It hashes original +bytes and returns the precise Office MIME type. Construction is passive, and +`validate_ready` requires both optional parsers. The in-process parsers do not +provide hard CPU or process-memory isolation. + +For binary DOC/XLS on Linux, inject +`teaql_blob_reference.content_legacy_office.LegacyOfficeBlobContentVerifier` +with absolute paths to patched LibreOffice (`soffice`), `bwrap`, `prlimit` and +an owner-only (`0700`) canonical scratch directory. Install the `office` extra +for its converted OOXML verifier. The adapter checks original CFB header and +sector layout, converts each file in a network/PID-isolated bubblewrap +namespace with read-only system mounts and only one private writable scratch, +then fully validates the DOCX/XLSX output. CPU, address-space, file-size, +wall-time, input and concurrency limits apply. It returns original DOC/XLS +SHA, size and MIME, so storage keeps the original bytes. This is bounded +decodability verification, not malware or macro detection. + +For actual MP3/MP4 validation on POSIX hosts, inject +`teaql_blob_reference.content_ffmpeg.FfmpegBlobContentVerifier` with absolute +paths to trusted `ffprobe` and `ffmpeg` executables. The host installs and +patches those binaries separately; this wheel does not bundle or fetch them. +`validate_ready` checks availability when the module is activated. The adapter +walks complete MP3 frames or top-level MP4 boxes before forcing the matching +FFmpeg demuxer, probing decoded frame counts and stream properties, and fully +decoding media from a pipe. Client filenames and URLs never reach FFmpeg. It +allows only the `pipe` protocol and bounds input bytes, frame count, video +pixels, concurrent checks, child CPU time and address space, and wall time. +The accepted MP3 profile is MPEG Layer III with fixed, non-free bitrate and a +constant sample rate; MP4 must contain a supported `ftyp` brand, `moov`, +`mdat`, and at least one decodable non-cover-art video stream. The adapter returns the SHA-256 +of the original bytes and `audio/mpeg` or `video/mp4`. It is content validation, +not malware scanning; ffprobe/ffmpeg themselves must be kept current. diff --git a/packages/teaql-blob-reference/pyproject.toml b/packages/teaql-blob-reference/pyproject.toml new file mode 100644 index 00000000..f8b4acd0 --- /dev/null +++ b/packages/teaql-blob-reference/pyproject.toml @@ -0,0 +1,22 @@ +[build-system] +requires = ["setuptools>=61.0"] +build-backend = "setuptools.build_meta" + +[project] +name = "teaql-blob-reference" +version = "0.3.1" +description = "Optional TeaQL Blob reference module" +readme = "README.md" +requires-python = ">=3.10" +license = "Apache-2.0" +authors = [{ name = "TeaQL" }] +dependencies = ["teaql==0.3.1"] + +[project.optional-dependencies] +image = ["Pillow==12.3.0"] +pdf = ["pypdf==6.21.3"] +office = ["python-docx==1.2.0", "openpyxl==3.1.5"] + +[tool.setuptools.packages.find] +where = ["src"] +include = ["teaql_blob_reference*"] diff --git a/packages/teaql-blob-reference/src/teaql_blob_reference/__init__.py b/packages/teaql-blob-reference/src/teaql_blob_reference/__init__.py new file mode 100644 index 00000000..02dcda1f --- /dev/null +++ b/packages/teaql-blob-reference/src/teaql_blob_reference/__init__.py @@ -0,0 +1,28 @@ +"""Optional Blob reference services; the base teaql runtime does not import them.""" + +from .sqlite_catalog import BlobBindingChange, BlobBindingTarget, BlobCatalogRead, SqliteBlobCatalog +from .read_service import SqliteBlobResponseContributor +from .module import BlobReferenceModule +from .maintenance_service import (BlobMaintenanceAuthorization, BlobMaintenanceBatch, + BlobMaintenanceCursor, BlobMaintenanceOutcome, + SqliteBlobMaintenanceService) +from .mutation_participant import SqliteBlobMutationParticipant +from .file_storage import FileObjectStorage +from .file_generation_maintenance import (BlobFileGenerationSweepBatch, + BlobFileGenerationSweepOutcome, + SqliteBlobFileGenerationSweepService) +from .retention import BlobRetentionPolicy +from .content_composite import CompositeBlobContentVerifier +from .upload_service import (BlobUploadAuthorizationState, BlobUploadInput, + BlobUploadSession, BlobUploadTarget, + BlobVerifiedContent, SqliteBlobUploadService) + +__all__ = ["BlobReferenceModule", "SqliteBlobMutationParticipant", "BlobBindingChange", "BlobBindingTarget", "BlobCatalogRead", + "BlobMaintenanceAuthorization", "BlobMaintenanceBatch", "BlobMaintenanceCursor", + "BlobMaintenanceOutcome", "SqliteBlobMaintenanceService", + "BlobFileGenerationSweepBatch", "BlobFileGenerationSweepOutcome", + "SqliteBlobFileGenerationSweepService", + "SqliteBlobCatalog", "SqliteBlobResponseContributor", "FileObjectStorage", + "BlobUploadAuthorizationState", "BlobUploadInput", "BlobUploadSession", "BlobUploadTarget", + "BlobVerifiedContent", "SqliteBlobUploadService", "BlobRetentionPolicy", + "CompositeBlobContentVerifier"] diff --git a/packages/teaql-blob-reference/src/teaql_blob_reference/content_composite.py b/packages/teaql-blob-reference/src/teaql_blob_reference/content_composite.py new file mode 100644 index 00000000..85c291be --- /dev/null +++ b/packages/teaql-blob-reference/src/teaql_blob_reference/content_composite.py @@ -0,0 +1,42 @@ +"""Passive host-selected routes for independently optional Blob parsers.""" + +from __future__ import annotations + +from collections.abc import Mapping + +from .upload_service import BlobContentVerifier, BlobVerifiedContent + + +class CompositeBlobContentVerifier: + """Connect multiple parsers to the single trusted upload verifier SPI.""" + + def __init__(self, routes: Mapping[str, BlobContentVerifier]): + if not isinstance(routes, Mapping): + raise RuntimeError("BLOB_POLICY_INVALID") + selected = dict(routes) + if (not 1 <= len(selected) <= 32 + or any(not isinstance(extension, str) or not 1 <= len(extension) <= 16 + or not extension.isascii() or not extension.isalnum() + or extension != extension.lower() or verifier is None + or not callable(getattr(verifier, "verify", None)) + for extension, verifier in selected.items())): + raise RuntimeError("BLOB_POLICY_INVALID") + self._routes = selected + self.supported_extensions = frozenset(self._routes) + + def validate_ready(self) -> None: + seen: set[int] = set() + for verifier in self._routes.values(): + identity = id(verifier) + if identity in seen: + continue + seen.add(identity) + ready = getattr(verifier, "validate_ready", None) + if callable(ready): + ready() + + def verify(self, stream, extension: str, maximum_bytes: int) -> BlobVerifiedContent: + verifier = self._routes.get(extension) + if verifier is None: + raise RuntimeError("BLOB_FORMAT_NOT_ALLOWED") + return verifier.verify(stream, extension, maximum_bytes) diff --git a/packages/teaql-blob-reference/src/teaql_blob_reference/content_ffmpeg.py b/packages/teaql-blob-reference/src/teaql_blob_reference/content_ffmpeg.py new file mode 100644 index 00000000..0997d5da --- /dev/null +++ b/packages/teaql-blob-reference/src/teaql_blob_reference/content_ffmpeg.py @@ -0,0 +1,235 @@ +"""Optional bounded MP3/MP4 verifier backed by host-supplied FFmpeg binaries. + +Input is passed over a pipe with only the pipe protocol enabled. The subprocess +never receives a client filename, URL, or shell command. +""" + +from __future__ import annotations + +from hashlib import sha256 +from json import loads +from os import access, name as os_name, path, X_OK +from subprocess import DEVNULL, PIPE, run +from threading import BoundedSemaphore + +from .upload_service import BlobVerifiedContent + +if os_name == "posix": + from resource import RLIMIT_AS, RLIMIT_CPU, setrlimit + + +_MIME = {"mp3": "audio/mpeg", "mp4": "video/mp4"} +_MP4_BRANDS = {b"isom", b"iso2", b"iso5", b"iso6", b"mp41", b"mp42", + b"avc1", b"M4V ", b"M4V2", b"dash"} +_MPEG1_LAYER3_BITRATES = (0, 32, 40, 48, 56, 64, 80, 96, 112, 128, + 160, 192, 224, 256, 320, 0) +_MPEG2_LAYER3_BITRATES = (0, 8, 16, 24, 32, 40, 48, 56, 64, 80, + 96, 112, 128, 144, 160, 0) + + +def _complete_mp3(data: bytes, maximum_frames: int) -> bool: + offset = 0 + if data.startswith(b"ID3"): + if len(data) < 10 or data[3] not in (2, 3, 4) or data[4] == 0xff: + return False + if any(part & 0x80 for part in data[6:10]): + return False + tag_size = (data[6] << 21) | (data[7] << 14) | (data[8] << 7) | data[9] + footer = 10 if data[3] == 4 and data[5] & 0x10 else 0 + offset = 10 + tag_size + footer + if offset > len(data): + return False + end = len(data) - (128 if len(data) >= 128 and data[-128:-125] == b"TAG" else 0) + frames = 0 + first_rate = None + while offset < end: + if end - offset < 4 or frames >= maximum_frames: + return False + header = int.from_bytes(data[offset:offset + 4], "big") + version = (header >> 19) & 3 + layer = (header >> 17) & 3 + bitrate_index = (header >> 12) & 15 + sample_index = (header >> 10) & 3 + padding = (header >> 9) & 1 + if (header >> 21 != 0x7ff or version == 1 or layer != 1 + or bitrate_index in (0, 15) or sample_index == 3): + return False + rate = (44100, 48000, 32000)[sample_index] + rate //= 1 if version == 3 else 2 if version == 2 else 4 + if first_rate is None: + first_rate = rate + elif first_rate != rate: + return False + bitrates = _MPEG1_LAYER3_BITRATES if version == 3 else _MPEG2_LAYER3_BITRATES + length = (144000 if version == 3 else 72000) * bitrates[bitrate_index] // rate + padding + if length < 4 or length > end - offset: + return False + offset += length + frames += 1 + return frames > 0 and offset == end + + +def _complete_mp4(data: bytes, maximum_boxes: int) -> bool: + offset = 0 + count = 0 + kinds = set() + while offset < len(data): + if len(data) - offset < 8 or count >= maximum_boxes: + return False + size = int.from_bytes(data[offset:offset + 4], "big") + kind = data[offset + 4:offset + 8] + header = 8 + if size == 1: + if len(data) - offset < 16: + return False + size = int.from_bytes(data[offset + 8:offset + 16], "big") + header = 16 + elif size == 0: + if kind != b"mdat": + return False + size = len(data) - offset + if size < header or size > len(data) - offset: + return False + if count == 0: + if kind != b"ftyp" or size < header + 8 or data[offset + header:offset + header + 4] not in _MP4_BRANDS: + return False + if kind in (b"ftyp", b"moov") and kind in kinds: + return False + kinds.add(kind) + offset += size + count += 1 + return {b"ftyp", b"moov", b"mdat"} <= kinds and offset == len(data) + + +class FfmpegBlobContentVerifier: + """Passive Linux/POSIX adapter; full decode occurs only during verification.""" + + supported_extensions = frozenset(_MIME) + + def __init__(self, ffprobe_path: str, ffmpeg_path: str, + maximum_input_bytes: int = 32 * 1024 * 1024, + maximum_frames: int = 100_000, maximum_video_pixels: int = 16_000_000, + concurrent_verifications: int = 2): + if (not isinstance(ffprobe_path, str) or not path.isabs(ffprobe_path) + or not isinstance(ffmpeg_path, str) or not path.isabs(ffmpeg_path) + or type(maximum_input_bytes) is not int or not 1 <= maximum_input_bytes <= 100 * 1024 * 1024 + or type(maximum_frames) is not int or not 1 <= maximum_frames <= 1_000_000 + or type(maximum_video_pixels) is not int or not 1 <= maximum_video_pixels <= 64_000_000 + or type(concurrent_verifications) is not int + or not 1 <= concurrent_verifications <= 32): + raise RuntimeError("BLOB_POLICY_INVALID") + self.ffprobe_path = ffprobe_path + self.ffmpeg_path = ffmpeg_path + self.maximum_input_bytes = maximum_input_bytes + self.maximum_frames = maximum_frames + self.maximum_video_pixels = maximum_video_pixels + self._permits = BoundedSemaphore(concurrent_verifications) + + @staticmethod + def _limits() -> None: + setrlimit(RLIMIT_CPU, (10, 10)) + setrlimit(RLIMIT_AS, (512 * 1024 * 1024, 512 * 1024 * 1024)) + + def validate_ready(self) -> None: + if os_name != "posix": + raise RuntimeError("BLOB_CAPABILITY_MISSING") + for binary, prefix in ((self.ffprobe_path, b"ffprobe version "), + (self.ffmpeg_path, b"ffmpeg version ")): + if not path.isfile(binary) or not access(binary, X_OK): + raise RuntimeError("BLOB_CAPABILITY_MISSING") + try: + completed = run((binary, "-version"), stdout=PIPE, stderr=DEVNULL, + timeout=3, check=False, env={"LC_ALL": "C"}) + except Exception: + raise RuntimeError("BLOB_CAPABILITY_MISSING") from None + if completed.returncode != 0 or not completed.stdout.startswith(prefix): + raise RuntimeError("BLOB_CAPABILITY_MISSING") + + def _run(self, binary: str, args: tuple[str, ...], data: bytes, output: bool): + try: + completed = run((binary, *args), input=data, stdout=PIPE if output else DEVNULL, + stderr=DEVNULL, timeout=15, check=False, + preexec_fn=self._limits, env={"LC_ALL": "C"}) + except Exception: + raise RuntimeError("BLOB_FORMAT_NOT_ALLOWED") from None + if completed.returncode != 0: + raise RuntimeError("BLOB_FORMAT_NOT_ALLOWED") + return completed.stdout if output else b"" + + def verify(self, stream, extension: str, maximum_bytes: int) -> BlobVerifiedContent: + if extension not in _MIME: + raise RuntimeError("BLOB_FORMAT_NOT_ALLOWED") + if type(maximum_bytes) is not int or maximum_bytes < 1: + raise RuntimeError("BLOB_POLICY_INVALID") + if not self._permits.acquire(blocking=False): + raise RuntimeError("BLOB_VERIFICATION_BUSY") + try: + maximum = min(maximum_bytes, self.maximum_input_bytes) + chunks = [] + digest = sha256() + size = 0 + while True: + try: + block = stream.read(65536) + except Exception: + raise RuntimeError("BLOB_STORE_FAILED") from None + if not isinstance(block, bytes): + raise RuntimeError("BLOB_STORE_FAILED") + if not block: + break + if len(block) > maximum - size: + raise RuntimeError("BLOB_SIZE_EXCEEDED") + chunks.append(block) + digest.update(block) + size += len(block) + data = b"".join(chunks) + if extension == "mp3": + valid = _complete_mp3(data, self.maximum_frames) + demuxer = "mp3" + else: + valid = _complete_mp4(data, 1000) + demuxer = "mov" + if not valid: + raise RuntimeError("BLOB_FORMAT_NOT_ALLOWED") + self.validate_ready() + probe = self._run(self.ffprobe_path, ( + "-v", "error", "-protocol_whitelist", "pipe", "-f", demuxer, + "-count_frames", "-show_entries", + "stream=codec_type,codec_name,nb_read_frames,width,height,sample_rate:stream_disposition=attached_pic:format=format_name", + "-of", "json", "-i", "pipe:0"), data, True) + if len(probe) > 65536: + raise RuntimeError("BLOB_FORMAT_NOT_ALLOWED") + try: + result = loads(probe) + streams = result["streams"] + if not isinstance(streams, list) or not 1 <= len(streams) <= 8: + raise ValueError("invalid streams") + if extension == "mp3": + if (result["format"]["format_name"] != "mp3" or len(streams) != 1 + or streams[0]["codec_type"] != "audio" + or streams[0]["codec_name"] != "mp3" + or not 8000 <= int(streams[0]["sample_rate"]) <= 192000 + or not 1 <= int(streams[0]["nb_read_frames"]) <= self.maximum_frames): + raise ValueError("invalid MP3") + maps = ("-map", "0:a:0") + else: + videos = [item for item in streams if item["codec_type"] == "video"] + if ("mp4" not in result["format"]["format_name"] + or not 1 <= len(videos) <= 4 + or any(item["codec_type"] not in ("video", "audio") for item in streams) + or any(item.get("disposition", {}).get("attached_pic", 0) != 0 + for item in videos) + or any(not 1 <= int(item["width"]) * int(item["height"]) <= self.maximum_video_pixels + or not 1 <= int(item["nb_read_frames"]) <= self.maximum_frames + for item in videos)): + raise ValueError("invalid MP4") + maps = ("-map", "0:v", "-map", "0:a?") + except Exception: + raise RuntimeError("BLOB_FORMAT_NOT_ALLOWED") from None + self._run(self.ffmpeg_path, ( + "-nostdin", "-v", "error", "-xerror", "-err_detect", "explode", + "-threads", "1", "-protocol_whitelist", "pipe", "-f", demuxer, + "-i", "pipe:0", *maps, "-f", "null", "-"), data, False) + return BlobVerifiedContent(size, digest.hexdigest(), _MIME[extension]) + finally: + self._permits.release() diff --git a/packages/teaql-blob-reference/src/teaql_blob_reference/content_legacy_office.py b/packages/teaql-blob-reference/src/teaql_blob_reference/content_legacy_office.py new file mode 100644 index 00000000..1be708a7 --- /dev/null +++ b/packages/teaql-blob-reference/src/teaql_blob_reference/content_legacy_office.py @@ -0,0 +1,192 @@ +"""Optional Linux sandboxed binary DOC/XLS verifier for Blob uploads.""" + +from __future__ import annotations + +from hashlib import sha256 +from io import BytesIO +from pathlib import Path +from tempfile import TemporaryDirectory +from threading import BoundedSemaphore +import os +import signal +import stat +import subprocess +import sys + +from .content_office import OfficeBlobContentVerifier +from .upload_service import BlobVerifiedContent + + +_MIME = {"doc": "application/msword", "xls": "application/vnd.ms-excel"} +_MAGIC = bytes.fromhex("d0cf11e0a1b11ae1") + + +def _u16(data: bytes, offset: int) -> int: + return int.from_bytes(data[offset:offset + 2], "little") + + +def _u32(data: bytes, offset: int) -> int: + return int.from_bytes(data[offset:offset + 4], "little") + + +def _valid_container(data: bytes) -> bool: + if (len(data) < 1024 or not data.startswith(_MAGIC) + or _u16(data, 28) != 0xfffe or _u16(data, 32) != 6): + return False + major, shift = _u16(data, 26), _u16(data, 30) + if not ((major, shift) in ((3, 9), (4, 12))): + return False + sector_size = 1 << shift + if len(data) % sector_size or len(data) < 2 * sector_size: + return False + sectors = len(data) // sector_size - 1 + fat_count, directory = _u32(data, 44), _u32(data, 48) + mini_fat, mini_count = _u32(data, 60), _u32(data, 64) + difat, difat_count = _u32(data, 68), _u32(data, 72) + return (0 < fat_count <= sectors and directory < sectors + and _u32(data, 56) == 4096 and mini_count <= sectors + and (not mini_count or mini_fat < sectors) + and difat_count <= sectors and (not difat_count or difat < sectors)) + + +class LegacyOfficeBlobContentVerifier: + """Host-selected, passive DOC/XLS adapter; the host provisions sandbox tools.""" + + supported_extensions = frozenset(_MIME) + + def __init__(self, soffice_path: str, bwrap_path: str, prlimit_path: str, + scratch_root: str | Path, maximum_input_bytes: int = 32 * 1024 * 1024, + maximum_converted_bytes: int = 10 * 1024 * 1024, + concurrent_verifications: int = 2): + paths = (soffice_path, bwrap_path, prlimit_path, str(scratch_root)) + if (any(not isinstance(path, str) or not os.path.isabs(path) for path in paths) + or os.path.normpath(paths[3]) != paths[3] + or type(maximum_input_bytes) is not int + or not 1 <= maximum_input_bytes <= 100 * 1024 * 1024 + or type(maximum_converted_bytes) is not int + or not 1 <= maximum_converted_bytes <= 10 * 1024 * 1024 + or type(concurrent_verifications) is not int + or not 1 <= concurrent_verifications <= 32): + raise RuntimeError("BLOB_POLICY_INVALID") + self.soffice_path, self.bwrap_path, self.prlimit_path = paths[:3] + self.scratch_root = Path(paths[3]) + self.maximum_input_bytes = maximum_input_bytes + self.maximum_converted_bytes = maximum_converted_bytes + self._permits = BoundedSemaphore(concurrent_verifications) + self._package = OfficeBlobContentVerifier( + maximum_input_bytes=maximum_converted_bytes, + maximum_decoded_bytes=100 * 1024 * 1024, + maximum_parts=10000, maximum_elements=2_000_000, + concurrent_verifications=concurrent_verifications) + + def validate_ready(self) -> None: + if sys.platform != "linux": + raise RuntimeError("BLOB_CAPABILITY_MISSING") + try: + for executable in (self.soffice_path, self.bwrap_path, self.prlimit_path): + if not Path(executable).is_file() or not os.access(executable, os.X_OK): + raise ValueError("missing converter") + root = self.scratch_root.lstat() + if (not stat.S_ISDIR(root.st_mode) or stat.S_IMODE(root.st_mode) != 0o700 + or root.st_uid != os.geteuid() + or self.scratch_root.resolve() != self.scratch_root): + raise ValueError("unsafe scratch root") + self._package.validate_ready() + except Exception: + raise RuntimeError("BLOB_CAPABILITY_MISSING") from None + + def _convert(self, data: bytes, extension: str) -> bytes: + try: + with TemporaryDirectory(prefix="legacy-office-", dir=self.scratch_root) as name: + job = Path(name) + output, profile = job / "output", job / "profile" + output.mkdir(mode=0o700) + profile.mkdir(mode=0o700) + source = job / ("source." + extension) + descriptor = os.open(source, os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o600) + with os.fdopen(descriptor, "wb") as writer: + writer.write(data) + args = [self.prlimit_path, "--cpu=20:20", + "--as=1073741824:1073741824", "--fsize=10485760:10485760", + "--", self.bwrap_path, + "--unshare-net", "--unshare-pid", "--die-with-parent", + "--ro-bind", "/usr", "/usr", "--ro-bind", "/lib", "/lib", + "--ro-bind", "/lib64", "/lib64", "--symlink", "usr/bin", "/bin", + "--ro-bind", "/etc", "/etc", "--dev", "/dev", "--proc", "/proc", + "--dir", "/tmp", "--bind", name, name, "--clearenv", + "--setenv", "LC_ALL", "C", "--setenv", "PATH", "/usr/bin:/bin", + "--setenv", "HOME", name, "--setenv", "TMPDIR", name, + "--", self.soffice_path, "-env:UserInstallation=" + profile.as_uri(), + "--headless", "--nodefault", "--norestore", "--nolockcheck", + "--convert-to", extension + "x", "--outdir", str(output), str(source)] + try: + process = subprocess.Popen(args, stdin=subprocess.DEVNULL, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, cwd="/", + env={"LC_ALL": "C", "PATH": "/usr/bin:/bin"}, + start_new_session=True) + except OSError: + raise RuntimeError("BLOB_CAPABILITY_MISSING") from None + try: + code = process.wait(timeout=30) + except subprocess.TimeoutExpired: + try: + os.killpg(process.pid, signal.SIGKILL) + except ProcessLookupError: + pass + process.wait() + raise RuntimeError("BLOB_FORMAT_NOT_ALLOWED") from None + if code: + raise RuntimeError("BLOB_FORMAT_NOT_ALLOWED") + entries = list(output.iterdir()) + expected = output / ("source." + extension + "x") + if len(entries) != 1 or entries[0] != expected: + raise RuntimeError("BLOB_FORMAT_NOT_ALLOWED") + info = expected.lstat() + if (not stat.S_ISREG(info.st_mode) or not 1 <= info.st_size <= + self.maximum_converted_bytes): + raise RuntimeError("BLOB_FORMAT_NOT_ALLOWED") + converted = expected.read_bytes() + if len(converted) != info.st_size: + raise RuntimeError("BLOB_FORMAT_NOT_ALLOWED") + return converted + except RuntimeError: + raise + except Exception: + raise RuntimeError("BLOB_STORE_FAILED") from None + + def verify(self, stream, extension: str, maximum_bytes: int) -> BlobVerifiedContent: + if extension not in _MIME: + raise RuntimeError("BLOB_FORMAT_NOT_ALLOWED") + if type(maximum_bytes) is not int or maximum_bytes < 1: + raise RuntimeError("BLOB_POLICY_INVALID") + if not self._permits.acquire(blocking=False): + raise RuntimeError("BLOB_VERIFICATION_BUSY") + try: + self.validate_ready() + maximum = min(maximum_bytes, self.maximum_input_bytes) + chunks, size = [], 0 + while True: + try: + block = stream.read(65536) + except Exception: + raise RuntimeError("BLOB_STORE_FAILED") from None + if not isinstance(block, bytes): + raise RuntimeError("BLOB_STORE_FAILED") + if not block: + break + if len(block) > maximum - size: + raise RuntimeError("BLOB_SIZE_EXCEEDED") + chunks.append(block) + size += len(block) + data = b"".join(chunks) + if not _valid_container(data): + raise RuntimeError("BLOB_FORMAT_NOT_ALLOWED") + converted = self._convert(data, extension) + try: + self._package.verify(BytesIO(converted), extension + "x", len(converted)) + except Exception: + raise RuntimeError("BLOB_FORMAT_NOT_ALLOWED") from None + return BlobVerifiedContent(size, sha256(data).hexdigest(), _MIME[extension]) + finally: + self._permits.release() diff --git a/packages/teaql-blob-reference/src/teaql_blob_reference/content_office.py b/packages/teaql-blob-reference/src/teaql_blob_reference/content_office.py new file mode 100644 index 00000000..ce12ee6b --- /dev/null +++ b/packages/teaql-blob-reference/src/teaql_blob_reference/content_office.py @@ -0,0 +1,180 @@ +"""Optional bounded DOCX/XLSX parser for Blob upload verification.""" + +from __future__ import annotations + +from hashlib import sha256 +from io import BytesIO +from threading import BoundedSemaphore +from xml.etree import ElementTree +from zipfile import BadZipFile, ZIP_DEFLATED, ZIP_STORED, ZipFile + +from .upload_service import BlobVerifiedContent + + +_MIME = {"docx": "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + "xlsx": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"} +_REQUIRED = {"docx": "word/document.xml", "xlsx": "xl/workbook.xml"} +_MAIN_TYPE = { + "docx": "application/vnd.openxmlformats-officedocument.wordprocessingml.document.main+xml", + "xlsx": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet.main+xml", +} + + +class OfficeBlobContentVerifier: + """Host-selected Office parser; construction does no parser import or storage I/O.""" + + supported_extensions = frozenset(_MIME) + + def __init__(self, maximum_input_bytes: int = 16 * 1024 * 1024, + maximum_decoded_bytes: int = 32 * 1024 * 1024, + maximum_parts: int = 1000, maximum_elements: int = 1_000_000, + concurrent_verifications: int = 2): + if (type(maximum_input_bytes) is not int or not 1 <= maximum_input_bytes <= 100 * 1024 * 1024 + or type(maximum_decoded_bytes) is not int + or not 1 <= maximum_decoded_bytes <= 100 * 1024 * 1024 + or type(maximum_parts) is not int or not 1 <= maximum_parts <= 10000 + or type(maximum_elements) is not int or not 1 <= maximum_elements <= 2_000_000 + or type(concurrent_verifications) is not int + or not 1 <= concurrent_verifications <= 32): + raise RuntimeError("BLOB_POLICY_INVALID") + self.maximum_input_bytes = maximum_input_bytes + self.maximum_decoded_bytes = maximum_decoded_bytes + self.maximum_parts = maximum_parts + self.maximum_elements = maximum_elements + self._permits = BoundedSemaphore(concurrent_verifications) + + def validate_ready(self) -> None: + try: + from docx import Document # noqa: F401 + from openpyxl import load_workbook # noqa: F401 + except ImportError: + raise RuntimeError("BLOB_CAPABILITY_MISSING") from None + + def _check_package(self, data: bytes, extension: str) -> None: + try: + with ZipFile(BytesIO(data)) as archive: + parts = archive.infolist() + if not 1 <= len(parts) <= self.maximum_parts: + raise ValueError("Office part count") + names = set() + for part in parts: + name = part.filename + if (name in names or name.startswith("/") or "\\" in name + or any(piece in ("", ".", "..") for piece in name.split("/")) + or part.flag_bits & 1 or part.compress_type not in (ZIP_STORED, ZIP_DEFLATED)): + raise ValueError("invalid Office member") + names.add(name) + if not {"[Content_Types].xml", "_rels/.rels", _REQUIRED[extension]} <= names: + raise ValueError("missing Office document parts") + decoded = 0 + for part in parts: + if part.is_dir(): + continue + if part.file_size > self.maximum_decoded_bytes - decoded: + raise ValueError("Office decoded size") + xml_part = part.filename.endswith((".xml", ".rels")) + overlap = b"" + with archive.open(part) as member: + while True: + block = member.read(65536) + if not block: + break + decoded += len(block) + if decoded > self.maximum_decoded_bytes: + raise ValueError("Office decoded size") + if xml_part: + scan = (overlap + block).upper() + if b" None: + from docx import Document + + document = Document(BytesIO(data)) + if document.element.body is None: + raise ValueError("missing Word body") + count = 0 + for _ in document.element.body.iter(): + count += 1 + if count > self.maximum_elements: + raise ValueError("Word element limit") + + def _parse_xlsx(self, data: bytes) -> None: + from openpyxl import load_workbook + + workbook = load_workbook(BytesIO(data), read_only=True, data_only=False, + keep_links=False) + try: + if not workbook.worksheets: + raise ValueError("missing Excel worksheet") + count = 0 + for sheet in workbook.worksheets: + sheet.reset_dimensions() # Do not trust producer-supplied used range. + for row in sheet.iter_rows(): + count += len(row) + if count > self.maximum_elements: + raise ValueError("Excel cell limit") + for cell in row: + _ = cell.value + finally: + workbook.close() + + def verify(self, stream, extension: str, maximum_bytes: int) -> BlobVerifiedContent: + if extension not in _MIME: + raise RuntimeError("BLOB_FORMAT_NOT_ALLOWED") + if type(maximum_bytes) is not int or maximum_bytes < 1: + raise RuntimeError("BLOB_POLICY_INVALID") + if not self._permits.acquire(blocking=False): + raise RuntimeError("BLOB_VERIFICATION_BUSY") + try: + maximum = min(maximum_bytes, self.maximum_input_bytes) + chunks = [] + size = 0 + digest = sha256() + while True: + try: + block = stream.read(65536) + except Exception: + raise RuntimeError("BLOB_STORE_FAILED") from None + if not isinstance(block, bytes): + raise RuntimeError("BLOB_STORE_FAILED") + if not block: + break + if len(block) > maximum - size: + raise RuntimeError("BLOB_SIZE_EXCEEDED") + chunks.append(block) + digest.update(block) + size += len(block) + data = b"".join(chunks) + try: + self.validate_ready() + self._check_package(data, extension) + if extension == "docx": + self._parse_docx(data) + else: + self._parse_xlsx(data) + except RuntimeError as error: + if str(error) == "BLOB_CAPABILITY_MISSING": + raise + raise RuntimeError("BLOB_FORMAT_NOT_ALLOWED") from None + except Exception: + raise RuntimeError("BLOB_FORMAT_NOT_ALLOWED") from None + return BlobVerifiedContent(size, digest.hexdigest(), _MIME[extension]) + finally: + self._permits.release() diff --git a/packages/teaql-blob-reference/src/teaql_blob_reference/content_pillow.py b/packages/teaql-blob-reference/src/teaql_blob_reference/content_pillow.py new file mode 100644 index 00000000..9a0e37c8 --- /dev/null +++ b/packages/teaql-blob-reference/src/teaql_blob_reference/content_pillow.py @@ -0,0 +1,133 @@ +"""Optional bounded JPEG/PNG parser for the trusted Blob upload verifier SPI.""" + +from __future__ import annotations + +from hashlib import sha256 +from io import BytesIO +from threading import BoundedSemaphore +from warnings import catch_warnings, simplefilter +import struct +import zlib + +from .upload_service import BlobVerifiedContent + + +_MIME = {"jpg": "image/jpeg", "jpeg": "image/jpeg", "png": "image/png"} +_PNG_SIGNATURE = b"\x89PNG\r\n\x1a\n" + + +def _complete_png(data: bytes) -> bool: + if len(data) < 45 or not data.startswith(_PNG_SIGNATURE): + return False + offset = 8 + count = 0 + image_data = False + while offset <= len(data) - 12: + length = struct.unpack_from(">I", data, offset)[0] + if length > len(data) - offset - 12: + return False + kind = data[offset + 4:offset + 8] + if len(kind) != 4 or not all(65 <= c <= 90 or 97 <= c <= 122 for c in kind): + return False + if count == 0 and (kind != b"IHDR" or length != 13): + return False + expected = struct.unpack_from(">I", data, offset + 8 + length)[0] + if zlib.crc32(data[offset + 4:offset + 8 + length]) != expected: + return False + offset += length + 12 + count += 1 + if kind == b"IDAT": + image_data = True + if kind == b"IEND": + return length == 0 and image_data and offset == len(data) + return False + + +class PillowBlobContentVerifier: + """Passive optional adapter; no Pillow import or parser work at construction.""" + + supported_extensions = frozenset(_MIME) + + def __init__(self, maximum_input_bytes: int = 32 * 1024 * 1024, + maximum_decoded_bytes: int = 100 * 1024 * 1024, + concurrent_verifications: int = 2): + if (type(maximum_input_bytes) is not int or not 1 <= maximum_input_bytes <= 100 * 1024 * 1024 + or type(maximum_decoded_bytes) is not int + or not 4 <= maximum_decoded_bytes <= 512 * 1024 * 1024 + or type(concurrent_verifications) is not int + or not 1 <= concurrent_verifications <= 32): + raise RuntimeError("BLOB_POLICY_INVALID") + self.maximum_input_bytes = maximum_input_bytes + self.maximum_decoded_bytes = maximum_decoded_bytes + self._permits = BoundedSemaphore(concurrent_verifications) + + def validate_ready(self) -> None: + try: + from PIL import ImageFile, features + if ImageFile.LOAD_TRUNCATED_IMAGES: + raise RuntimeError("BLOB_CAPABILITY_MISSING") + if not features.check("jpg") or not features.check("zlib"): + raise RuntimeError("BLOB_CAPABILITY_MISSING") + except Exception: + raise RuntimeError("BLOB_CAPABILITY_MISSING") from None + + def verify(self, stream, extension: str, maximum_bytes: int) -> BlobVerifiedContent: + if extension not in _MIME: + raise RuntimeError("BLOB_FORMAT_NOT_ALLOWED") + if type(maximum_bytes) is not int or maximum_bytes < 1: + raise RuntimeError("BLOB_POLICY_INVALID") + if not self._permits.acquire(blocking=False): + raise RuntimeError("BLOB_VERIFICATION_BUSY") + try: + maximum = min(maximum_bytes, self.maximum_input_bytes) + chunks = [] + size = 0 + digest = sha256() + while True: + try: + block = stream.read(65536) + except Exception: + raise RuntimeError("BLOB_STORE_FAILED") from None + if not isinstance(block, bytes): + raise RuntimeError("BLOB_STORE_FAILED") + if not block: + break + if len(block) > maximum - size: + raise RuntimeError("BLOB_SIZE_EXCEEDED") + chunks.append(block) + digest.update(block) + size += len(block) + if not size: + raise RuntimeError("BLOB_FORMAT_NOT_ALLOWED") + data = b"".join(chunks) + if extension == "png": + complete = _complete_png(data) + else: + complete = len(data) >= 4 and data.startswith(b"\xff\xd8") and data.endswith(b"\xff\xd9") + if not complete: + raise RuntimeError("BLOB_FORMAT_NOT_ALLOWED") + try: + from PIL import Image, ImageFile + if ImageFile.LOAD_TRUNCATED_IMAGES: + raise RuntimeError("BLOB_CAPABILITY_MISSING") + expected = "PNG" if extension == "png" else "JPEG" + with catch_warnings(): + simplefilter("error", Image.DecompressionBombWarning) + with Image.open(BytesIO(data), formats=[expected]) as image: + width, height = image.size + if (image.format != expected or width < 1 or height < 1 + or width * height > self.maximum_decoded_bytes // 4 + or getattr(image, "n_frames", 1) != 1): + raise RuntimeError("BLOB_FORMAT_NOT_ALLOWED") + image.load() # Header detection alone is insufficient. + except RuntimeError as error: + if str(error) == "BLOB_CAPABILITY_MISSING": + raise + raise RuntimeError("BLOB_FORMAT_NOT_ALLOWED") from None + except ImportError: + raise RuntimeError("BLOB_CAPABILITY_MISSING") from None + except Exception: + raise RuntimeError("BLOB_FORMAT_NOT_ALLOWED") from None + return BlobVerifiedContent(size, digest.hexdigest(), _MIME[extension]) + finally: + self._permits.release() diff --git a/packages/teaql-blob-reference/src/teaql_blob_reference/content_pypdf.py b/packages/teaql-blob-reference/src/teaql_blob_reference/content_pypdf.py new file mode 100644 index 00000000..dc32d460 --- /dev/null +++ b/packages/teaql-blob-reference/src/teaql_blob_reference/content_pypdf.py @@ -0,0 +1,94 @@ +"""Optional bounded PDF structure/content parser for the Blob upload verifier SPI.""" + +from __future__ import annotations + +from hashlib import sha256 +from io import BytesIO +from math import isfinite +from threading import BoundedSemaphore + +from .upload_service import BlobVerifiedContent + + +class PypdfBlobContentVerifier: + """Host-selected PDF parser; construction does no parser I/O or imports.""" + + supported_extensions = frozenset(("pdf",)) + + def __init__(self, maximum_input_bytes: int = 32 * 1024 * 1024, + maximum_decoded_bytes: int = 64 * 1024 * 1024, + maximum_pages: int = 1000, concurrent_verifications: int = 2): + if (type(maximum_input_bytes) is not int or not 1 <= maximum_input_bytes <= 100 * 1024 * 1024 + or type(maximum_decoded_bytes) is not int + or not 1 <= maximum_decoded_bytes <= 64 * 1024 * 1024 + or type(maximum_pages) is not int or not 1 <= maximum_pages <= 1000 + or type(concurrent_verifications) is not int + or not 1 <= concurrent_verifications <= 32): + raise RuntimeError("BLOB_POLICY_INVALID") + self.maximum_input_bytes = maximum_input_bytes + self.maximum_decoded_bytes = maximum_decoded_bytes + self.maximum_pages = maximum_pages + self._permits = BoundedSemaphore(concurrent_verifications) + + def validate_ready(self) -> None: + try: + from pypdf import PdfReader # noqa: F401 + except ImportError: + raise RuntimeError("BLOB_CAPABILITY_MISSING") from None + + def verify(self, stream, extension: str, maximum_bytes: int) -> BlobVerifiedContent: + if extension != "pdf": + raise RuntimeError("BLOB_FORMAT_NOT_ALLOWED") + if type(maximum_bytes) is not int or maximum_bytes < 1: + raise RuntimeError("BLOB_POLICY_INVALID") + if not self._permits.acquire(blocking=False): + raise RuntimeError("BLOB_VERIFICATION_BUSY") + try: + maximum = min(maximum_bytes, self.maximum_input_bytes) + chunks = [] + size = 0 + digest = sha256() + while True: + try: + block = stream.read(65536) + except Exception: + raise RuntimeError("BLOB_STORE_FAILED") from None + if not isinstance(block, bytes): + raise RuntimeError("BLOB_STORE_FAILED") + if not block: + break + if len(block) > maximum - size: + raise RuntimeError("BLOB_SIZE_EXCEEDED") + chunks.append(block) + digest.update(block) + size += len(block) + data = b"".join(chunks) + if not data.startswith(b"%PDF-") or not data.rstrip(b" \t\r\n\f").endswith(b"%%EOF"): + raise RuntimeError("BLOB_FORMAT_NOT_ALLOWED") + try: + from pypdf import PdfReader + except ImportError: + raise RuntimeError("BLOB_CAPABILITY_MISSING") from None + try: + reader = PdfReader(BytesIO(data), strict=True, root_object_recovery_limit=256) + if reader.is_encrypted or not 1 <= len(reader.pages) <= self.maximum_pages: + raise ValueError("invalid PDF page set") + decoded = 0 + for page in reader.pages: + width = float(page.mediabox.width) + height = float(page.mediabox.height) + if (not isfinite(width) or not isfinite(height) + or not 0 < width <= 14400 or not 0 < height <= 14400): + raise ValueError("invalid PDF media box") + contents = page.get_contents() + if contents is not None: + payload = contents.get_data() + decoded += len(payload) + if decoded > self.maximum_decoded_bytes: + raise ValueError("PDF decoded content limit") + _ = contents.operations # Parse content operators, not only the header. + except Exception: + raise RuntimeError("BLOB_FORMAT_NOT_ALLOWED") from None + return BlobVerifiedContent(size, digest.hexdigest(), "application/pdf") + finally: + self._permits.release() diff --git a/packages/teaql-blob-reference/src/teaql_blob_reference/file_generation_maintenance.py b/packages/teaql-blob-reference/src/teaql_blob_reference/file_generation_maintenance.py new file mode 100644 index 00000000..b0bd5693 --- /dev/null +++ b/packages/teaql-blob-reference/src/teaql_blob_reference/file_generation_maintenance.py @@ -0,0 +1,104 @@ +"""Explicit catalog-driven cleanup of inaccessible fenced File generations.""" + +from __future__ import annotations + +import asyncio +from dataclasses import dataclass + +from teaql.core.blob_ref import BlobRef +from teaql.core.object_storage import ObjectKey, StorageError +from teaql.core.value import Value +from teaql.sql.executor import SqlTransactionTransport +from teaql.sql.types import CompiledQuery + +from .file_storage import FileObjectStorage +from .maintenance_service import BlobMaintenanceAuthorization, BlobMaintenanceCursor +from .sqlite_catalog import SqliteBlobCatalog, _scope + + +@dataclass(frozen=True) +class BlobFileGenerationSweepOutcome: + reference: BlobRef + status: str + removed_generations: int | None = None + error: str | None = None + + +@dataclass(frozen=True) +class BlobFileGenerationSweepBatch: + outcomes: tuple[BlobFileGenerationSweepOutcome, ...] + next: BlobMaintenanceCursor | None + + +class SqliteBlobFileGenerationSweepService: + """Host-scheduled sweep; the host must first upgrade every root writer.""" + + def __init__(self, catalog: SqliteBlobCatalog, + authorization: BlobMaintenanceAuthorization, + all_writers_upgraded: bool, batch_size: int = 100): + if (not isinstance(catalog, SqliteBlobCatalog) + or not isinstance(catalog.transport, SqlTransactionTransport) + or authorization is None + or not callable(getattr(authorization, "scope", None)) + or not callable(getattr(authorization, "can_maintain", None)) + or all_writers_upgraded is not True + or type(batch_size) is not int or not 1 <= batch_size <= 1000): + raise RuntimeError("BLOB_POLICY_INVALID") + self.catalog = catalog + self.authorization = authorization + self.batch_size = batch_size + + async def _authorize(self, actor, expected: str | None = None) -> str: + if actor is None: + raise RuntimeError("BLOB_MAINTENANCE_DENIED") + try: + scope = self.authorization.scope(actor) + allowed = await self.authorization.can_maintain(actor, scope) + except Exception: + raise RuntimeError("BLOB_MAINTENANCE_DENIED") from None + if not _scope(scope) or expected is not None and scope != expected or not allowed: + raise RuntimeError("BLOB_MAINTENANCE_DENIED") + return scope + + async def run_batch(self, actor, provider: FileObjectStorage, + cursor: BlobMaintenanceCursor | None = None) -> BlobFileGenerationSweepBatch: + scope = await self._authorize(actor) + if cursor is not None and (not isinstance(cursor, BlobMaintenanceCursor) + or cursor.scope != scope): + raise RuntimeError("BLOB_MAINTENANCE_DENIED") + actor.require_no_active_graph_sql() + if (not isinstance(provider, FileObjectStorage) + or actor.get_resource("object_storage_provider") is not provider): + raise RuntimeError("BLOB_CAPABILITY_MISSING") + await self.catalog.validate_schema() + try: + rows = await self.catalog.transport.fetch_all_sql(CompiledQuery( + "SELECT key,object_key FROM teaql_blob_catalog " + "WHERE scope=? AND key>? AND upload_mode='server_stream' " + "ORDER BY key LIMIT ?", + [Value.Text(scope), Value.Text(cursor.after if cursor else ""), + Value.I64(self.batch_size)])) + except Exception: + raise RuntimeError("BLOB_CATALOG_FAILED") from None + candidates = [] + for row in rows: + try: + reference = BlobRef.from_sql(row.get("key")) + object_key = ObjectKey(row.get("object_key")) + except (ValueError, TypeError, StorageError): + raise RuntimeError("BLOB_CATALOG_FAILED") from None + candidates.append((reference, object_key)) + outcomes = [] + for reference, object_key in candidates: + await self._authorize(actor, scope) + try: + removed = await asyncio.to_thread(provider.sweep_old_generations, object_key) + if type(removed) is not int or removed < 0: + raise RuntimeError("BLOB_STORE_FAILED") + outcomes.append(BlobFileGenerationSweepOutcome(reference, "swept", removed)) + except Exception: + outcomes.append(BlobFileGenerationSweepOutcome(reference, "failed", None, + "BLOB_STORE_FAILED")) + next_cursor = (BlobMaintenanceCursor(scope, candidates[-1][0].key) + if len(candidates) == self.batch_size else None) + return BlobFileGenerationSweepBatch(tuple(outcomes), next_cursor) diff --git a/packages/teaql-blob-reference/src/teaql_blob_reference/file_storage.py b/packages/teaql-blob-reference/src/teaql_blob_reference/file_storage.py new file mode 100644 index 00000000..2117922b --- /dev/null +++ b/packages/teaql-blob-reference/src/teaql_blob_reference/file_storage.py @@ -0,0 +1,496 @@ +"""Optional private file store with an explicitly started authenticated local HTTP gateway.""" +from contextlib import contextmanager +from datetime import datetime, timedelta, timezone +from hashlib import sha256 +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path +from threading import RLock, Thread +from urllib.parse import urlsplit +import base64 +import json +import os +import re +import secrets +import stat +import time + +if os.name == "posix": + import fcntl +elif os.name == "nt": + import msvcrt + +from cryptography.hazmat.primitives.ciphers.aead import AESGCM + +from teaql.core.object_storage import ( + ObjectContent, ObjectMetadata, ObjectReadUrl, StorageError, StorageErrorCode, WriteMode, +) + + +def _error(code): + return StorageError(code) + + +def _b64(data): + return base64.urlsafe_b64encode(data).rstrip(b"=").decode("ascii") + + +def _unb64(value): + if not isinstance(value, str) or len(value) > 4096 or not re.fullmatch(r"[A-Za-z0-9_-]+", value): + return None + try: + decoded = base64.urlsafe_b64decode(value + "=" * (-len(value) % 4)) + except (ValueError, base64.binascii.Error): + return None + return decoded if _b64(decoded) == value else None + + +class FileObjectStorage: + """The host must protect root and its parent; old generations require explicit GC.""" + + def __init__(self, root, signing_key: bytes): + try: + path = Path(root) + except TypeError: + raise _error(StorageErrorCode.INVALID_INPUT) from None + if not path.is_absolute() or not isinstance(signing_key, bytes) or len(signing_key) != 32: + raise _error(StorageErrorCode.INVALID_INPUT) + self._root = path + self._cipher = AESGCM(signing_key) + self._lock = RLock() + self._server = None + self._thread = None + self._base_url = None + + def _private_root(self, create=False): + if create: + try: + self._root.mkdir(mode=0o700, parents=True, exist_ok=True) + except OSError: + raise _error(StorageErrorCode.IO) from None + try: + info = self._root.lstat() + except OSError: + raise _error(StorageErrorCode.UNAVAILABLE) from None + if not stat.S_ISDIR(info.st_mode) or info.st_mode & 0o077: + raise _error(StorageErrorCode.UNAVAILABLE) + + def _directory(self, key_hash): + return self._root / key_hash[:2] / key_hash + + @contextmanager + def _manifest_lock(self, key_hash, name="manifest.lock"): + """Persistent OS lock: never remove the lock file while this root is in use.""" + path = self._directory(key_hash) / name + flags = os.O_RDWR | os.O_CREAT | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NOFOLLOW", 0) + fd = None + locked = False + try: + fd = os.open(path, flags, 0o600) + info = os.fstat(fd) + if not stat.S_ISREG(info.st_mode) or (os.name == "posix" and info.st_mode & 0o077): + raise _error(StorageErrorCode.UNAVAILABLE) + if os.name == "nt": + if info.st_size == 0: + os.write(fd, b"\0") + elif os.name != "posix": + raise _error(StorageErrorCode.UNSUPPORTED) + deadline = time.monotonic() + 30 + while True: + try: + if os.name == "posix": + fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB) + else: + os.lseek(fd, 0, os.SEEK_SET) + msvcrt.locking(fd, msvcrt.LK_NBLCK, 1) + break + except OSError: + if time.monotonic() >= deadline: + raise _error(StorageErrorCode.TIMEOUT) from None + time.sleep(0.02) + locked = True + yield + except StorageError: + raise + except OSError: + raise _error(StorageErrorCode.IO) from None + finally: + if fd is not None: + try: + if locked and os.name == "posix": + fcntl.flock(fd, fcntl.LOCK_UN) + elif locked and os.name == "nt": + os.lseek(fd, 0, os.SEEK_SET) + msvcrt.locking(fd, msvcrt.LK_UNLCK, 1) + finally: + os.close(fd) + + @staticmethod + def _hash(key): + return sha256(key.value.encode("utf-8")).hexdigest() + + def _manifest(self, key_hash): + try: + raw = (self._directory(key_hash) / "manifest.json").read_bytes() + except FileNotFoundError: + return None + except OSError: + raise _error(StorageErrorCode.UNAVAILABLE) from None + if len(raw) > 16384: + raise _error(StorageErrorCode.UNAVAILABLE) + try: + value = json.loads(raw) + version = value["version"] + if not isinstance(version, str) or not re.fullmatch(r"[A-Za-z0-9_-]{32}", version) or \ + type(value["size"]) is not int or not 0 <= value["size"] <= 5 * 1024**3 or \ + not isinstance(value["content_type"], str) or not value["content_type"] or \ + not all(32 <= ord(c) <= 126 for c in value["content_type"]) or \ + not isinstance(value["sha256"], str) or not re.fullmatch(r"[0-9a-f]{64}", value["sha256"]) or \ + not isinstance(value["modified"], str) or not isinstance(value["metadata"], dict): + raise ValueError("invalid manifest") + datetime.fromisoformat(value["modified"]) + except (ValueError, TypeError, KeyError): + raise _error(StorageErrorCode.UNAVAILABLE) from None + return value + + def _metadata(self, key, manifest): + return ObjectMetadata(key, manifest["size"], manifest["content_type"], + manifest["sha256"] + "." + manifest["version"], + datetime.fromisoformat(manifest["modified"]), manifest["metadata"]) + + def _data_path(self, key_hash, manifest): + return self._directory(key_hash) / ("data-" + manifest["version"]) + + def _check_data(self, key_hash, manifest): + try: + info = self._data_path(key_hash, manifest).stat() + except OSError: + raise _error(StorageErrorCode.UNAVAILABLE) from None + if not stat.S_ISREG(info.st_mode) or info.st_size != manifest["size"]: + raise _error(StorageErrorCode.UNAVAILABLE) + + def put(self, invocation, key, content, options): + self._private_root(create=True) + key_hash = self._hash(key) + directory = self._directory(key_hash) + try: + directory.mkdir(mode=0o700, parents=True, exist_ok=True) + except OSError: + raise _error(StorageErrorCode.IO) from None + # The writer lock covers the data file before its manifest is published. + # A generation sweep takes this lock first and then the manifest lock. + with self._manifest_lock(key_hash, "writer.lock"): + return self._put_under_writer_lock(key, content, options, key_hash, directory) + + def _put_under_writer_lock(self, key, content, options, key_hash, directory): + version = _b64(secrets.token_bytes(24)) + data_path = directory / ("data-" + version) + temp_path = directory / ("manifest-" + version) + published = False + try: + digest = sha256() + size = 0 + with data_path.open("xb") as output: + while True: + try: + block = content.read(65536) + except Exception: + raise _error(StorageErrorCode.IO) from None + if not isinstance(block, bytes) or len(block) > options.size - size: + raise _error(StorageErrorCode.INVALID_INPUT) + if not block: + break + output.write(block) + digest.update(block) + size += len(block) + if size != options.size: + raise _error(StorageErrorCode.INVALID_INPUT) + output.flush() + os.fsync(output.fileno()) + manifest = dict(version=version, size=size, content_type=options.content_type, + sha256=digest.hexdigest(), modified=datetime.now(timezone.utc).isoformat(), + metadata=dict(options.metadata)) + with temp_path.open("xb") as output: + output.write(json.dumps(manifest, separators=(",", ":")).encode("utf-8")) + output.flush() + os.fsync(output.fileno()) + destination = directory / "manifest.json" + with self._manifest_lock(key_hash): + if options.mode == WriteMode.CREATE_ONLY: + try: + os.link(temp_path, destination) + except FileExistsError: + raise _error(StorageErrorCode.CONFLICT) from None + published = True + try: + temp_path.unlink(missing_ok=True) + except OSError: + pass + else: + os.replace(temp_path, destination) + published = True + return self._metadata(key, manifest) + except StorageError: + raise + except OSError: + raise _error(StorageErrorCode.IO) from None + finally: + if not published: + for path in (data_path, temp_path): + try: + path.unlink(missing_ok=True) + except OSError: + pass + + def get(self, invocation, key): + self._private_root() + key_hash = self._hash(key) + manifest = self._manifest(key_hash) + if manifest is None: + raise _error(StorageErrorCode.NOT_FOUND) + try: + stream = self._data_path(key_hash, manifest).open("rb") + info = os.fstat(stream.fileno()) + if not stat.S_ISREG(info.st_mode) or info.st_size != manifest["size"]: + stream.close() + raise _error(StorageErrorCode.UNAVAILABLE) + except OSError: + raise _error(StorageErrorCode.UNAVAILABLE) from None + return ObjectContent(self._metadata(key, manifest), stream) + + def head(self, invocation, key): + if not self._root.exists(): + return None + self._private_root() + key_hash = self._hash(key) + manifest = self._manifest(key_hash) + if manifest is None: + return None + self._check_data(key_hash, manifest) + return self._metadata(key, manifest) + + def delete(self, invocation, key): + if not self._root.exists(): + return + self._private_root() + key_hash = self._hash(key) + if not self._directory(key_hash).exists(): + return + try: + with self._manifest_lock(key_hash): + (self._directory(key_hash) / "manifest.json").unlink(missing_ok=True) + except OSError: + raise _error(StorageErrorCode.IO) from None + + def delete_if_version(self, invocation, key, content_version): + if not self._root.exists(): + return "already_missing" + self._private_root() + key_hash = self._hash(key) + if not self._directory(key_hash).exists(): + return "already_missing" + try: + with self._manifest_lock(key_hash): + manifest = self._manifest(key_hash) + if manifest is None: + return "already_missing" + if self._metadata(key, manifest).etag != content_version: + return "version_mismatch" + self._data_path(key_hash, manifest).unlink(missing_ok=True) + (self._directory(key_hash) / "manifest.json").unlink() + return "deleted" + except StorageError: + raise + except OSError: + raise _error(StorageErrorCode.IO) from None + + def sweep_old_generations(self, key): + """Host-scheduled cleanup of inaccessible File generations for one key. + + The caller owns the provider and must not mix pre-writer-lock process + versions on this root while sweeping. This never removes the current + manifest or its data file, even when the key was recreated. + """ + if not self._root.exists(): + return 0 + self._private_root() + key_hash = self._hash(key) + directory = self._directory(key_hash) + if not directory.exists(): + return 0 + removed = 0 + try: + with self._manifest_lock(key_hash, "writer.lock"): + with self._manifest_lock(key_hash): + manifest = self._manifest(key_hash) + current = "data-" + manifest["version"] if manifest else None + for path in directory.iterdir(): + if path.name == current or not re.fullmatch(r"data-[A-Za-z0-9_-]{32}", path.name): + continue + info = path.lstat() + if not stat.S_ISREG(info.st_mode): + raise _error(StorageErrorCode.UNAVAILABLE) + path.unlink() + removed += 1 + return removed + except StorageError: + raise + except OSError: + raise _error(StorageErrorCode.IO) from None + + def _seal(self, ticket): + nonce = secrets.token_bytes(12) + payload = json.dumps(ticket, separators=(",", ":")).encode("utf-8") + return _b64(nonce + self._cipher.encrypt(nonce, payload, None)) + + def _open_ticket(self, token): + raw = _unb64(token) + if raw is None or len(raw) < 29: + return None + try: + value = json.loads(self._cipher.decrypt(raw[:12], raw[12:], None)) + if not isinstance(value, dict) or not isinstance(value.get("hash"), str) or \ + not re.fullmatch(r"[0-9a-f]{64}", value["hash"]) or \ + not isinstance(value.get("version"), str) or \ + not re.fullmatch(r"[A-Za-z0-9_-]{32}", value["version"]) or \ + type(value.get("expires_ms")) is not int or \ + value["expires_ms"] <= time.time_ns() // 1_000_000 or \ + not (value.get("disposition") is None or isinstance(value["disposition"], str) and + all(32 <= ord(c) <= 126 for c in value["disposition"])): + return None + except Exception: + return None + return value + + def get_read_url(self, invocation, key, options): + with self._lock: + base = self._base_url + if base is None: + raise _error(StorageErrorCode.NOT_CONFIGURED) + self._private_root() + key_hash = self._hash(key) + manifest = self._manifest(key_hash) + if manifest is None: + raise _error(StorageErrorCode.NOT_FOUND) + self._check_data(key_hash, manifest) + expires_at = datetime.now(timezone.utc) + options.lifetime + ticket = self._seal(dict(hash=key_hash, version=manifest["version"], + expires_ms=int(expires_at.timestamp() * 1000), + disposition=options.content_disposition())) + return ObjectReadUrl(base + "/objects/" + ticket, expires_at) + + def start(self): + with self._lock: + if self._server is not None: + raise _error(StorageErrorCode.CONFLICT) + self._private_root(create=True) + owner = self + + class Handler(BaseHTTPRequestHandler): + def do_GET(self): + owner._serve(self, head=False) + + def do_HEAD(self): + owner._serve(self, head=True) + + def do_POST(self): + self.send_error(405) + + do_PUT = do_POST + do_DELETE = do_POST + + def log_message(self, *_): + pass # The path contains a private read ticket. + + try: + server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + except OSError: + raise _error(StorageErrorCode.UNAVAILABLE) from None + server.daemon_threads = True + thread = Thread(target=server.serve_forever, daemon=True) + self._server, self._thread = server, thread + self._base_url = f"http://127.0.0.1:{server.server_port}" + thread.start() + return self._base_url + + def close(self): + with self._lock: + server, thread = self._server, self._thread + self._server = self._thread = self._base_url = None + if server is not None: + server.shutdown() + server.server_close() + thread.join(timeout=5) + + def _serve(self, handler, head): + path = urlsplit(handler.path).path + token = path.removeprefix("/objects/") if path.startswith("/objects/") else "" + ticket = self._open_ticket(token) + if ticket is None: + handler.send_error(404) + return + try: + manifest = self._manifest(ticket["hash"]) + if manifest is None or manifest["version"] != ticket["version"]: + handler.send_error(404) + return + data_path = self._data_path(ticket["hash"], manifest) + with data_path.open("rb") as source: + info = os.fstat(source.fileno()) + if not stat.S_ISREG(info.st_mode) or info.st_size != manifest["size"]: + handler.send_error(404) + return + size, start, end = manifest["size"], 0, manifest["size"] - 1 + range_header = handler.headers.get("Range") + partial = range_header is not None + if partial: + match = re.fullmatch(r"bytes=(\d*)-(\d*)", range_header) + if not match or size == 0 or not any(match.groups()) or \ + any(len(part) > 20 for part in match.groups()): + handler.send_response(416) + handler.send_header("Content-Range", f"bytes */{size}") + handler.end_headers() + return + first, last = match.groups() + if first: + start = int(first) + if last: + end = int(last) + else: + suffix = int(last) + start = max(0, size - suffix) + if start >= size or end < start or not first and int(last) == 0: + handler.send_response(416) + handler.send_header("Content-Range", f"bytes */{size}") + handler.end_headers() + return + end = min(end, size - 1) + handler.send_response(206 if partial else 200) + handler.send_header("Content-Type", manifest["content_type"]) + handler.send_header("Content-Length", str(end - start + 1 if partial else size)) + handler.send_header("Accept-Ranges", "bytes") + handler.send_header("Cache-Control", "private, no-store") + handler.send_header("X-Content-Type-Options", "nosniff") + handler.send_header("ETag", '"' + manifest["sha256"] + "." + manifest["version"] + '"') + if ticket["disposition"]: + handler.send_header("Content-Disposition", ticket["disposition"]) + if partial: + handler.send_header("Content-Range", f"bytes {start}-{end}/{size}") + handler.end_headers() + if not head and size: + source.seek(start) + remaining = end - start + 1 + while remaining: + block = source.read(min(65536, remaining)) + if not block: + break + handler.wfile.write(block) + remaining -= len(block) + except (OSError, StorageError): + if not handler.wfile.closed: + try: + handler.send_error(404) + except OSError: + pass + + def __repr__(self): + return "FileObjectStorage[configuration=masked]" diff --git a/packages/teaql-blob-reference/src/teaql_blob_reference/maintenance_service.py b/packages/teaql-blob-reference/src/teaql_blob_reference/maintenance_service.py new file mode 100644 index 00000000..3d2a0789 --- /dev/null +++ b/packages/teaql-blob-reference/src/teaql_blob_reference/maintenance_service.py @@ -0,0 +1,357 @@ +"""Explicit SQLite cleanup of expired, uncompleted server-stream uploads.""" + +from __future__ import annotations + +import asyncio +from contextlib import suppress +from dataclasses import dataclass +import time +from typing import Callable, Protocol + +from teaql.core.blob_ref import BlobRef +from teaql.core.object_storage import ObjectKey, StorageError +from teaql.core.value import Value +from teaql.sql.executor import SqlTransactionTransport +from teaql.sql.types import CompiledQuery + +from .sqlite_catalog import SqliteBlobCatalog, _scope +from .retention import (retention_deadline, unreferenced_eligible_at, + valid_seconds, valid_time) + + +class BlobMaintenanceAuthorization(Protocol): + def scope(self, context) -> str: ... + async def can_maintain(self, context, scope: str) -> bool: ... + + +@dataclass(frozen=True, repr=False) +class BlobMaintenanceCursor: + scope: str + after: str + + def __repr__(self): + return "BlobMaintenanceCursor[masked]" + + +@dataclass(frozen=True) +class BlobMaintenanceOutcome: + reference: BlobRef + status: str + error: str | None = None + + +@dataclass(frozen=True) +class BlobMaintenanceBatch: + outcomes: tuple[BlobMaintenanceOutcome, ...] + next: BlobMaintenanceCursor | None + + +def _query(sql, *params): + return CompiledQuery(sql, list(params)) + + +def _version(value): + return (isinstance(value, str) and value and value != "*" + and len(value) <= 256 and value == value.strip() + and not any(ord(ch) < 32 or 127 <= ord(ch) <= 159 for ch in value)) + + +class SqliteBlobMaintenanceService: + """The host schedules this service; installing a Blob module never runs it.""" + + def __init__(self, catalog: SqliteBlobCatalog, authorization: BlobMaintenanceAuthorization, + batch_size: int = 100, now_ms: Callable[[], int] | None = None): + if (not isinstance(catalog, SqliteBlobCatalog) + or not isinstance(catalog.transport, SqlTransactionTransport) + or authorization is None + or not callable(getattr(authorization, "scope", None)) + or not callable(getattr(authorization, "can_maintain", None)) + or type(batch_size) is not int or not 1 <= batch_size <= 1000): + raise RuntimeError("BLOB_POLICY_INVALID") + self.catalog = catalog + self.authorization = authorization + self.batch_size = batch_size + self.now_ms = now_ms or (lambda: time.time_ns() // 1_000_000) + + async def _authorize(self, actor, expected: str | None = None) -> str: + if actor is None: + raise RuntimeError("BLOB_MAINTENANCE_DENIED") + try: + scope = self.authorization.scope(actor) + allowed = await self.authorization.can_maintain(actor, scope) + except Exception: + raise RuntimeError("BLOB_MAINTENANCE_DENIED") from None + if not _scope(scope) or (expected is not None and scope != expected) or not allowed: + raise RuntimeError("BLOB_MAINTENANCE_DENIED") + return scope + + def _now(self) -> int: + value = self.now_ms() + if not valid_time(value): + raise RuntimeError("BLOB_POLICY_INVALID") + return value + + async def _transaction(self, work): + try: + tx = await self.catalog.transport.begin_sql() + except Exception: + raise RuntimeError("BLOB_CATALOG_FAILED") from None + committed = False + try: + result = await work(tx) + try: + await tx.commit_sql() + except Exception: + raise RuntimeError("BLOB_CATALOG_FAILED") from None + committed = True + return result + finally: + if not committed: + with suppress(Exception): + await tx.rollback_sql() + + async def _row(self, sql, scope: str, key: str): + try: + rows = await sql.fetch_all_sql(_query( + "SELECT c.object_key,c.content_version,c.status,c.completed_at_ms," + "c.last_unbound_at_ms,c.policy_profile,c.upload_mode," + "c.unbound_retention_seconds,c.effective_unref_seconds," + "c.retention_floor_until_ms,u.state,u.expires_ms,u.written_etag " + "FROM teaql_blob_catalog c JOIN teaql_blob_upload u " + "ON u.key=c.key AND u.scope=c.scope WHERE c.scope=? AND c.key=?", + Value.Text(scope), Value.Text(key))) + except Exception: + raise RuntimeError("BLOB_CATALOG_FAILED") from None + if not rows: + return None + if len(rows) != 1: + raise RuntimeError("BLOB_CATALOG_FAILED") + row = rows[0] + object_key, version = row.get("object_key"), row.get("content_version") + raw_written = row.get("written_etag") + written = "" if raw_written is None else raw_written + if (not isinstance(object_key, str) or not object_key + or not isinstance(version, str) or version and not _version(version) + or not isinstance(row.get("status"), str) + or not isinstance(row.get("state"), str) + or type(row.get("expires_ms")) is not int + or not isinstance(written, str) or written and not _version(written) + or row["state"] == "written" and not written): + raise RuntimeError("BLOB_CATALOG_FAILED") + return {"key": key, "object_key": object_key, "version": version, + "status": row["status"], "state": row["state"], + "expires_ms": row["expires_ms"], "written_etag": written, + "completed_at_ms": row["completed_at_ms"], + "last_unbound_at_ms": row["last_unbound_at_ms"], + "policy_profile": row["policy_profile"], "upload_mode": row["upload_mode"], + "unbound_retention_seconds": row["unbound_retention_seconds"], + "effective_unref_seconds": row["effective_unref_seconds"], + "retention_floor_until_ms": row["retention_floor_until_ms"]} + + @staticmethod + def _eligible(row, now): + if row is None: + return False + if row["state"] in ("initiated", "written"): + # NULL is a pre-upload-mode server-stream session; explicit direct + # uploads need independent quiescence proof before object deletion. + return (row["upload_mode"] in (None, "server_stream") + and row["expires_ms"] <= now + and row["status"] in ("initiated", "deleting")) + if row["state"] != "completed" or row["status"] not in ( + "uploaded_unbound", "available", "deleting"): + return False + if (not _scope(row["policy_profile"]) or row["upload_mode"] != "server_stream" + or not valid_time(row["completed_at_ms"]) + or not valid_seconds(row["unbound_retention_seconds"]) + or not valid_seconds(row["effective_unref_seconds"]) + or not _version(row["version"])): + return False + last = row["last_unbound_at_ms"] + floor = row["retention_floor_until_ms"] + if last is None: + if row["status"] == "available": + return False + deadline = retention_deadline(row["completed_at_ms"], + row["unbound_retention_seconds"]) + else: + if not valid_time(last) or floor is not None and not valid_time(floor): + return False + deadline = unreferenced_eligible_at(last, row["effective_unref_seconds"], floor) + return now >= deadline + + async def _count(self, sql, statement, key): + try: + rows = await sql.fetch_all_sql(_query(statement, Value.Text(key))) + except Exception: + raise RuntimeError("BLOB_CATALOG_FAILED") from None + if len(rows) != 1 or type(rows[0].get("n")) is not int or rows[0]["n"] < 0: + raise RuntimeError("BLOB_CATALOG_FAILED") + return rows[0]["n"] + + async def _claim(self, scope, key, now): + async def work(sql): + row = await self._row(sql, scope, key) + if not self._eligible(row, now): + return None + if await self._count(sql, "SELECT count(*) AS n FROM teaql_blob_binding WHERE key=?", key): + return None + owners = await self._count(sql, "SELECT count(*) AS n FROM teaql_blob_catalog WHERE object_key=?", + row["object_key"]) + if owners != 1: + raise RuntimeError("BLOB_CATALOG_FAILED") + if row["status"] != "deleting": + try: + affected, _ = await sql.execute_sql(_query( + "UPDATE teaql_blob_catalog SET status='deleting' " + "WHERE key=? AND scope=? AND status=?", + Value.Text(key), Value.Text(scope), Value.Text(row["status"]))) + except Exception: + raise RuntimeError("BLOB_CATALOG_FAILED") from None + if affected != 1: + raise RuntimeError("BLOB_CATALOG_FAILED") + return row + return await self._transaction(work) + + async def _capture(self, scope, claim, version, now): + if not _version(version): + raise RuntimeError("BLOB_IDEMPOTENCY_CONFLICT") + async def work(sql): + row = await self._row(sql, scope, claim["key"]) + bound = await self._count(sql, "SELECT count(*) AS n FROM teaql_blob_binding WHERE key=?", + claim["key"]) + if (not self._eligible(row, now) or row["status"] != "deleting" + or row["object_key"] != claim["object_key"] or bound + or row["written_etag"] and row["written_etag"] != version + or row["version"] and row["version"] != version): + raise RuntimeError("BLOB_IDEMPOTENCY_CONFLICT") + if not row["version"]: + try: + affected, _ = await sql.execute_sql(_query( + "UPDATE teaql_blob_catalog SET content_version=? " + "WHERE key=? AND scope=? AND status='deleting' AND content_version=''", + Value.Text(version), Value.Text(claim["key"]), Value.Text(scope))) + except Exception: + raise RuntimeError("BLOB_CATALOG_FAILED") from None + if affected != 1: + raise RuntimeError("BLOB_IDEMPOTENCY_CONFLICT") + await self._transaction(work) + + async def _finish(self, scope, claim, version, now): + async def work(sql): + row = await self._row(sql, scope, claim["key"]) + if row is None or row["object_key"] != claim["object_key"] or row["version"] != version: + raise RuntimeError("BLOB_IDEMPOTENCY_CONFLICT") + if row["status"] == "deleted": + return + bound = await self._count(sql, "SELECT count(*) AS n FROM teaql_blob_binding WHERE key=?", + claim["key"]) + if not self._eligible(row, now) or row["status"] != "deleting" or bound: + raise RuntimeError("BLOB_IDEMPOTENCY_CONFLICT") + try: + affected, _ = await sql.execute_sql(_query( + "UPDATE teaql_blob_catalog SET status='deleted' " + "WHERE key=? AND scope=? AND status='deleting' AND content_version=?", + Value.Text(claim["key"]), Value.Text(scope), Value.Text(version))) + except Exception: + raise RuntimeError("BLOB_CATALOG_FAILED") from None + if affected != 1: + raise RuntimeError("BLOB_CATALOG_FAILED") + await self._transaction(work) + + async def _collect(self, actor, scope, key, store): + try: + await self._authorize(actor, scope) + claim = await self._claim(scope, key, self._now()) + if claim is None: + return BlobMaintenanceOutcome(BlobRef(key), "not_eligible") + await self._authorize(actor, scope) + try: + object_key = ObjectKey(claim["object_key"]) + except StorageError: + raise RuntimeError("BLOB_CATALOG_FAILED") from None + version = claim["version"] + if version and claim["written_etag"] and version != claim["written_etag"]: + raise RuntimeError("BLOB_IDEMPOTENCY_CONFLICT") + if not version: + try: + observed = await asyncio.to_thread(store.head, object_key) + except Exception: + raise RuntimeError("BLOB_STORE_FAILED") from None + if observed is not None: + if (not _version(observed.etag) + or claim["written_etag"] and observed.etag != claim["written_etag"]): + raise RuntimeError("BLOB_IDEMPOTENCY_CONFLICT") + version = observed.etag + await self._authorize(actor, scope) + await self._capture(scope, claim, version, self._now()) + await self._authorize(actor, scope) + result = "already_missing" + if version: + try: + deleted = await asyncio.to_thread(store.delete_if_version, object_key, version) + except Exception: + raise RuntimeError("BLOB_STORE_FAILED") from None + if deleted == "version_mismatch": + raise RuntimeError("BLOB_IDEMPOTENCY_CONFLICT") + result = deleted + await self._authorize(actor, scope) + await self._finish(scope, claim, version, self._now()) + return BlobMaintenanceOutcome(BlobRef(key), result) + except RuntimeError as error: + code = str(error) + if code in ("BLOB_MAINTENANCE_DENIED", "BLOB_TRANSACTION_REQUIRED"): + raise + if code == "BLOB_IDEMPOTENCY_CONFLICT": + return BlobMaintenanceOutcome(BlobRef(key), "version_conflict", code) + if code != "BLOB_CATALOG_FAILED": + code = "BLOB_STORE_FAILED" + return BlobMaintenanceOutcome(BlobRef(key), "failed", code) + + async def run_batch(self, actor, cursor: BlobMaintenanceCursor | None = None) -> BlobMaintenanceBatch: + scope = await self._authorize(actor) + if cursor is not None and (not isinstance(cursor, BlobMaintenanceCursor) + or cursor.scope != scope): + raise RuntimeError("BLOB_MAINTENANCE_DENIED") + actor.require_no_active_graph_sql() + try: + store = actor.object_storage("collect expired Blob upload") + except Exception: + raise RuntimeError("BLOB_CAPABILITY_MISSING") from None + if not store.supports_version_conditional_delete(): + raise RuntimeError("BLOB_CAPABILITY_MISSING") + await self.catalog.validate_schema() + now = self._now() + after = cursor.after if cursor else "" + try: + rows = await self.catalog.transport.fetch_all_sql(_query( + "SELECT DISTINCT c.key FROM teaql_blob_catalog c JOIN teaql_blob_upload u " + "ON u.key=c.key AND u.scope=c.scope WHERE c.scope=? AND c.key>? " + "AND ((u.expires_ms<=? AND u.state IN ('initiated','written') " + "AND c.status IN ('initiated','deleting') " + "AND (c.upload_mode IS NULL OR c.upload_mode='server_stream')) OR " + "(u.state='completed' AND c.upload_mode='server_stream' " + "AND c.policy_profile IS NOT NULL AND c.completed_at_ms IS NOT NULL " + "AND c.content_version<>'' AND (c.status='deleting' OR " + "(c.status='uploaded_unbound' AND c.last_unbound_at_ms IS NULL " + "AND c.completed_at_ms+c.unbound_retention_seconds*1000<=?) OR " + "(c.status='available' AND c.last_unbound_at_ms IS NOT NULL " + "AND c.last_unbound_at_ms+c.effective_unref_seconds*1000<=? " + "AND (c.retention_floor_until_ms IS NULL OR c.retention_floor_until_ms<=?))))) " + "AND NOT EXISTS (SELECT 1 FROM teaql_blob_binding b WHERE b.key=c.key) " + "ORDER BY c.key LIMIT ?", Value.Text(scope), Value.Text(after), + Value.I64(now), Value.I64(now), Value.I64(now), Value.I64(now), + Value.I64(self.batch_size))) + except Exception: + raise RuntimeError("BLOB_CATALOG_FAILED") from None + keys = [] + for row in rows: + key = row.get("key") + try: + BlobRef.from_sql(key) + except (ValueError, TypeError): + raise RuntimeError("BLOB_CATALOG_FAILED") from None + keys.append(key) + outcomes = tuple([await self._collect(actor, scope, key, store) for key in keys]) + next_cursor = BlobMaintenanceCursor(scope, keys[-1]) if len(keys) == self.batch_size else None + return BlobMaintenanceBatch(outcomes, next_cursor) diff --git a/packages/teaql-blob-reference/src/teaql_blob_reference/module.py b/packages/teaql-blob-reference/src/teaql_blob_reference/module.py new file mode 100644 index 00000000..b89b83a1 --- /dev/null +++ b/packages/teaql-blob-reference/src/teaql_blob_reference/module.py @@ -0,0 +1,82 @@ +"""Passive assembly of the optional SQLite Blob reference services.""" + +from teaql.runtime.capability import install_runtime_capability +from teaql.sql.executor import SqlDataServiceExecutor +from teaql.sql.types import DatabaseKind + +from .read_service import SqliteBlobResponseContributor +from .mutation_participant import SqliteBlobMutationParticipant +from .sqlite_catalog import SqliteBlobCatalog +from .upload_service import SqliteBlobUploadService +from .maintenance_service import SqliteBlobMaintenanceService + + +class _BlobCapability: + capability = "blob.v1" + + def __init__(self, module: "BlobReferenceModule") -> None: + self.module = module + + async def validate_ready(self, context) -> None: + if not isinstance(self.module.upload, SqliteBlobUploadService) or \ + self.module.upload.catalog is not self.module.catalog or \ + self.module.read.catalog is not self.module.catalog or \ + self.module.mutation.catalog is not self.module.catalog or \ + context.get_resource("blob_response_contributor") is not self.module.read or \ + context.get_resource("blob_mutation_participant") is not self.module.mutation or \ + context.get_resource("object_storage_provider") is None: + raise RuntimeError("BLOB_CAPABILITY_MISSING") + if self.module.maintenance is not None and \ + (self.module.maintenance.catalog is not self.module.catalog or \ + context.get_resource("blob_maintenance_service") is not self.module.maintenance): + raise RuntimeError("BLOB_CAPABILITY_MISSING") + owner = context.get_resource("dataService") + try: + compatible = (isinstance(owner, SqlDataServiceExecutor) and + owner.dialect.kind() == DatabaseKind.Sqlite and + await owner.uses_sql_database(self.module.catalog.transport)) + except Exception: + compatible = False + if not compatible: + raise RuntimeError("BLOB_CAPABILITY_MISSING") + await self.module.catalog.validate_schema() + verifier_ready = getattr(self.module.upload.verifier, "validate_ready", None) + if callable(verifier_ready): + verifier_ready() + + +class BlobReferenceModule: + """Host installs this module, then separately migrates and checks its catalog.""" + + def __init__(self, catalog: SqliteBlobCatalog, upload: SqliteBlobUploadService, + read_authorization, binding_authorization, maximum_url_seconds: int = 600, + maintenance: SqliteBlobMaintenanceService | None = None): + if not isinstance(catalog, SqliteBlobCatalog) or \ + not isinstance(upload, SqliteBlobUploadService) or upload.catalog is not catalog or \ + maintenance is not None and (not isinstance(maintenance, SqliteBlobMaintenanceService) + or maintenance.catalog is not catalog): + raise RuntimeError("BLOB_CAPABILITY_MISSING") + self.catalog = catalog + self.upload = upload + self.mutation = SqliteBlobMutationParticipant(catalog, binding_authorization) + self.read = SqliteBlobResponseContributor(catalog, read_authorization, + maximum_url_seconds) + self.maintenance = maintenance + + def install(self, context) -> None: + if context is None or context.get_resource("blob_response_contributor") is not None or \ + context.get_resource("blob_mutation_participant") is not None or \ + (self.maintenance is not None and context.get_resource("blob_maintenance_service") is not None) or \ + context.get_resource("teaql.runtime.capability:blob.v1") is not None: + raise RuntimeError("BLOB_CAPABILITY_MISSING") + install_runtime_capability(context, _BlobCapability(self)) + context.insert_resource("blob_response_contributor", self.read) + context.insert_resource("blob_mutation_participant", self.mutation) + if self.maintenance is not None: + context.insert_resource("blob_maintenance_service", self.maintenance) + + async def ready(self, context) -> None: + capability = context.get_resource("teaql.runtime.capability:blob.v1") + if not isinstance(capability, _BlobCapability) or capability.module is not self: + raise RuntimeError("BLOB_CAPABILITY_MISSING") + await capability.validate_ready(context) diff --git a/packages/teaql-blob-reference/src/teaql_blob_reference/mutation_participant.py b/packages/teaql-blob-reference/src/teaql_blob_reference/mutation_participant.py new file mode 100644 index 00000000..0ff51690 --- /dev/null +++ b/packages/teaql-blob-reference/src/teaql_blob_reference/mutation_participant.py @@ -0,0 +1,112 @@ +"""Native SQLite Blob binding in the caller's physical graph transaction.""" + +from teaql.core.blob_ref import BlobRef +from teaql.core.mutation import DeleteCommand, InsertCommand, UpdateCommand +from teaql.core.value import DataType, Value +from teaql.sql.types import CompiledQuery + +from .sqlite_catalog import BlobBindingChange, BlobBindingTarget, SqliteBlobCatalog + + +class SqliteBlobMutationParticipant: + def __init__(self, catalog: SqliteBlobCatalog, authorization): + if not isinstance(catalog, SqliteBlobCatalog) or authorization is None or \ + not callable(getattr(authorization, "scope", None)) or \ + not callable(getattr(authorization, "authorize", None)): + raise RuntimeError("BLOB_CAPABILITY_MISSING") + self.catalog = catalog + self.authorization = authorization + + async def apply(self, context, descriptor, mutation, persisted) -> None: + await self.apply_with_before(context, descriptor, mutation, persisted, None) + + async def apply_with_before(self, context, descriptor, mutation, persisted, before) -> None: + graph = getattr(context, "_graph_session", None) + if graph is None or context is not graph.context: + raise RuntimeError("BLOB_TRANSACTION_REQUIRED") + if not isinstance(mutation, (InsertCommand, UpdateCommand, DeleteCommand)): + raise RuntimeError("BLOB_BIND_DENIED") + id_property = next((prop for prop in descriptor.properties if prop._is_id), None) + version_property = next((prop for prop in descriptor.properties if prop._is_version), None) + if id_property is None or version_property is None: + raise RuntimeError("BLOB_CAPABILITY_MISSING") + if isinstance(mutation, DeleteCommand): + raw_id = mutation.id.val + action = "Delete" + else: + if not isinstance(persisted, dict): + raise RuntimeError("BLOB_BIND_DENIED") + raw_id = persisted.get(id_property.name) + action = "Create" if isinstance(mutation, InsertCommand) else "Update" + if isinstance(raw_id, bool) or not isinstance(raw_id, int) or raw_id <= 0: + raise RuntimeError("BLOB_BIND_DENIED") + entity_id = str(raw_id) + try: + scope = self.authorization.scope(context) + except Exception: + raise RuntimeError("BLOB_BIND_DENIED") from None + if not isinstance(scope, str) or not scope or scope != scope.strip(): + raise RuntimeError("BLOB_BIND_DENIED") + + async def prior_binding(target): + async def query(sql): + rows = await sql.fetch_all_sql(CompiledQuery( + "SELECT key,version FROM teaql_blob_binding WHERE scope = ? AND entity = ? " + "AND entity_id = ? AND field = ?", + [Value.Text(scope), Value.Text(target.entity), Value.Text(target.entity_id), + Value.Text(target.field)])) + if len(rows) > 1: + raise RuntimeError("BLOB_CATALOG_FAILED") + return rows[0] if rows else None + return await graph.with_sql_transport(query) + + for prop in descriptor.properties: + if prop.blob_spec is None: + continue + if prop.blob_spec.entity != descriptor._name or prop.blob_spec.field != prop.name: + raise RuntimeError("BLOB_REFERENCE_INVALID") + if action != "Delete" and prop.name not in mutation.values: + if action == "Create" and not prop.blob_spec.nullable: + raise RuntimeError("BLOB_BIND_DENIED") + if action != "Create": + continue + target = BlobBindingTarget(descriptor._name, entity_id, prop.name) + binding = await prior_binding(target) + prior = binding["version"] if binding is not None else None + if prior is not None and (type(prior) is not int or prior < 0): + raise RuntimeError("BLOB_CATALOG_FAILED") + if action == "Create" and prior is not None: + raise RuntimeError("BLOB_BIND_DENIED") + if action != "Create": + if not isinstance(before, dict) or prop.name not in before or \ + before[prop.name] != (binding["key"] if binding is not None else None): + raise RuntimeError("BLOB_BIND_DENIED") + if action == "Delete": + if prior is None: + # A pre-binding nullable legacy slot has no catalog fact + # to clear. Its authoritative old value must be NULL. + continue + next_ref = None + else: + value = mutation.values.get(prop.name, Value.Null()) + if not isinstance(value, Value): + raise RuntimeError("BLOB_REFERENCE_INVALID") + if value.val is None: + if not prop.blob_spec.nullable: + raise RuntimeError("BLOB_BIND_DENIED") + next_ref = None + elif value._type_hint == DataType.BlobRef: + try: + next_ref = BlobRef.from_sql(value.val) + except ValueError: + raise RuntimeError("BLOB_REFERENCE_INVALID") from None + else: + raise RuntimeError("BLOB_REFERENCE_INVALID") + if persisted.get(prop.name) != (next_ref.key if next_ref else None): + raise RuntimeError("BLOB_BIND_DENIED") + business_version = persisted.get(version_property.name) if persisted else None + next_version = max((prior or 0) + 1, + business_version if type(business_version) is int and business_version > 0 else 0) + await self.catalog.bind_in_graph(graph, context, + BlobBindingChange(target, next_ref, prior or 0, next_version, action), + self.authorization) diff --git a/packages/teaql-blob-reference/src/teaql_blob_reference/read_service.py b/packages/teaql-blob-reference/src/teaql_blob_reference/read_service.py new file mode 100644 index 00000000..a4cf8ef8 --- /dev/null +++ b/packages/teaql-blob-reference/src/teaql_blob_reference/read_service.py @@ -0,0 +1,62 @@ +"""Current-authorized Blob URL projection backed by the optional SQLite catalog.""" + +import asyncio +from datetime import datetime, timedelta, timezone +from typing import Protocol + +from teaql.core.object_storage import ObjectKey, ReadUrlOptions +from teaql.runtime.view_preparation import ( + AuthorizedBlobSlot, AuthorizedResponseGraph, ProjectedBlobURL, ViewPreparationError, +) + +from .sqlite_catalog import SqliteBlobCatalog + + +class BlobReadAuthorization(Protocol): + async def authorize(self, context, slot: AuthorizedBlobSlot) -> bool: ... + + +class SqliteBlobResponseContributor: + """Reauthorizes and rereads before requesting complete temporary storage URLs.""" + + def __init__(self, catalog: SqliteBlobCatalog, + authorization: BlobReadAuthorization, maximum_url_seconds: int = 600): + if catalog is None or authorization is None or type(maximum_url_seconds) is not int \ + or not 1 <= maximum_url_seconds <= 3600: + raise RuntimeError("BLOB_POLICY_INVALID") + self.catalog = catalog + self.authorization = authorization + self.maximum_url_seconds = maximum_url_seconds + + async def project_batch(self, context, graph: AuthorizedResponseGraph, + expires_at: datetime | None) -> tuple[ProjectedBlobURL, ...]: + results = [] + for slot in graph.blob_slots: + try: + allowed = await self.authorization.authorize(context, slot) + except Exception: + raise ViewPreparationError("BLOB_READ_DENIED") from None + if not allowed: + raise ViewPreparationError("BLOB_READ_DENIED") + try: + record = await self.catalog.current_read(slot) + except RuntimeError as error: + code = str(error) if str(error) in ("BLOB_READ_DENIED", "BLOB_CATALOG_FAILED") \ + else "BLOB_CATALOG_FAILED" + raise ViewPreparationError(code) from None + seconds = self.maximum_url_seconds + if expires_at is not None: + seconds = min(seconds, int((expires_at - datetime.now(timezone.utc)).total_seconds()) - 1) + if seconds < 1: + raise ViewPreparationError("BLOB_REFERENCE_EXPIRED") + disposition = "inline" if record.content_type.startswith(("image/", "audio/", "video/")) \ + or record.content_type == "application/pdf" else "attachment" + try: + signed = await asyncio.to_thread( + context.object_storage("prepare authorized Blob view").get_read_url, + ObjectKey(record.object_key), + ReadUrlOptions(timedelta(seconds=seconds), disposition, record.file_name)) + except Exception: + raise ViewPreparationError("BLOB_SIGN_FAILED") from None + results.append(ProjectedBlobURL(signed.url, signed.expires_at)) + return tuple(results) diff --git a/packages/teaql-blob-reference/src/teaql_blob_reference/retention.py b/packages/teaql-blob-reference/src/teaql_blob_reference/retention.py new file mode 100644 index 00000000..b1e7a577 --- /dev/null +++ b/packages/teaql-blob-reference/src/teaql_blob_reference/retention.py @@ -0,0 +1,78 @@ +"""Validated, immutable host policy and time rules for Blob retention.""" + +from dataclasses import dataclass + + +MAX_RETENTION_EPOCH_MS = 8_640_000_000_000_000 + + +def valid_time(value: object) -> bool: + return type(value) is int and 0 <= value <= MAX_RETENTION_EPOCH_MS + + +def valid_seconds(value: object) -> bool: + return type(value) is int and 0 <= value <= MAX_RETENTION_EPOCH_MS // 1000 + + +def retention_deadline(base_ms: int, seconds: int) -> int: + if not valid_time(base_ms): + raise RuntimeError("BLOB_REFERENCE_INVALID") + if not valid_seconds(seconds) or seconds > (MAX_RETENTION_EPOCH_MS - base_ms) // 1000: + raise RuntimeError("BLOB_POLICY_INVALID") + return base_ms + seconds * 1000 + + +def effective_retention(accepted: tuple[int, ...], proposed: int) -> int: + if not valid_seconds(proposed) or any(not valid_seconds(value) for value in accepted): + raise RuntimeError("BLOB_POLICY_INVALID") + return max((proposed, *accepted)) + + +def unreferenced_eligible_at(last_unbound_ms: int, seconds: int, + floor_until_ms: int | None) -> int: + deadline = retention_deadline(last_unbound_ms, seconds) + if floor_until_ms is not None: + if not valid_time(floor_until_ms): + raise RuntimeError("BLOB_REFERENCE_INVALID") + deadline = max(deadline, floor_until_ms) + return deadline + + +def require_bindable(status: str, now_ms: int, completed_ms: int | None, + unbound_seconds: int, unreferenced_eligible_ms: int | None) -> None: + if not valid_time(now_ms): + raise RuntimeError("BLOB_REFERENCE_INVALID") + if status == "initiated": + raise RuntimeError("BLOB_UPLOAD_INCOMPLETE") + if status in ("deleting", "deleted"): + raise RuntimeError("BLOB_REFERENCE_EXPIRED") + if status == "uploaded_unbound": + if completed_ms is None: + raise RuntimeError("BLOB_REFERENCE_INVALID") + if now_ms >= retention_deadline(completed_ms, unbound_seconds): + raise RuntimeError("BLOB_REFERENCE_EXPIRED") + elif status == "available": + if unreferenced_eligible_ms is not None: + if not valid_time(unreferenced_eligible_ms): + raise RuntimeError("BLOB_REFERENCE_INVALID") + if now_ms >= unreferenced_eligible_ms: + raise RuntimeError("BLOB_REFERENCE_EXPIRED") + else: + raise RuntimeError("BLOB_REFERENCE_INVALID") + + +@dataclass(frozen=True) +class BlobRetentionPolicy: + profile: str + upload_mode: str + unbound_retention_seconds: int + unreferenced_retention_seconds: int + minimum_retention_seconds: int + + def __post_init__(self): + from .sqlite_catalog import _scope + if (not _scope(self.profile) or self.upload_mode not in ("server_stream", "direct") + or not all(valid_seconds(value) for value in ( + self.unbound_retention_seconds, self.unreferenced_retention_seconds, + self.minimum_retention_seconds))): + raise RuntimeError("BLOB_POLICY_INVALID") diff --git a/packages/teaql-blob-reference/src/teaql_blob_reference/sqlite_catalog.py b/packages/teaql-blob-reference/src/teaql_blob_reference/sqlite_catalog.py new file mode 100644 index 00000000..701ab5b6 --- /dev/null +++ b/packages/teaql-blob-reference/src/teaql_blob_reference/sqlite_catalog.py @@ -0,0 +1,371 @@ +"""Authoritative SQLite binding component under an explicit native graph owner.""" + +from dataclasses import dataclass +from typing import Protocol +import time +import unicodedata + +from teaql.core.blob_ref import BlobRef +from teaql.core.value import Value +from teaql.sql.types import CompiledQuery +from teaql.runtime.view_preparation import AuthorizedBlobSlot +from .retention import (BlobRetentionPolicy, require_bindable, retention_deadline, + unreferenced_eligible_at, valid_seconds, valid_time) + + +@dataclass(frozen=True) +class BlobBindingTarget: + entity: str + entity_id: str + field: str + + +@dataclass(frozen=True) +class BlobBindingChange: + target: BlobBindingTarget + next: BlobRef | None + expected_version: int + next_version: int + action: str + + +@dataclass(frozen=True) +class BlobCatalogRead: + object_key: str + content_version: str + file_name: str + content_type: str + size_bytes: int + + +class BlobBindingAuthorization(Protocol): + def scope(self, context) -> str: ... + + async def authorize(self, context, target: BlobBindingTarget, + previous: BlobRef | None, next_ref: BlobRef | None, + action: str) -> bool: ... + + +class BlobBindingPolicyAuthorization(Protocol): + async def current_policy_on_lease(self, context, lease, + target: BlobBindingTarget) -> BlobRetentionPolicy: ... + + +def _name(value): + return (isinstance(value, str) and value and value == value.strip() + and len(value) <= 128 and not any(unicodedata.category(ch) == "Cc" for ch in value)) + + +def _scope(value): + if not isinstance(value, str) or not value or value != value.strip(): + return False + try: + return (len(value.encode("utf-8")) <= 256 + and not any(unicodedata.category(ch) == "Cc" for ch in value)) + except UnicodeEncodeError: + return False + + +def _entity_id(value): + return (isinstance(value, str) and 1 <= len(value) <= 20 and value.isascii() + and value.isdecimal() and 0 < int(value) <= 2**64 - 1 + and str(int(value)) == value) + + +class SqliteBlobCatalog: + """SQLite-only catalog; install and schema migration are separate host actions.""" + + def __init__(self, transport, now_ms=None): + if (transport is None or not callable(getattr(transport, "execute_sql", None)) + or not callable(getattr(transport, "fetch_all_sql", None))): + raise RuntimeError("BLOB_CAPABILITY_MISSING") + self.transport = transport + self.now_ms = now_ms or (lambda: time.time_ns() // 1_000_000) + + async def ensure_schema(self): + statements = ( + """CREATE TABLE IF NOT EXISTS teaql_blob_catalog ( + key TEXT PRIMARY KEY, scope TEXT NOT NULL, object_key TEXT NOT NULL, + content_version TEXT NOT NULL, file_name TEXT NOT NULL, + content_type TEXT NOT NULL, size_bytes BIGINT NOT NULL, + completed_at_ms BIGINT, last_unbound_at_ms BIGINT, + policy_profile TEXT, upload_mode TEXT, unbound_retention_seconds BIGINT, + effective_unref_seconds BIGINT, retention_floor_until_ms BIGINT, + status TEXT NOT NULL CHECK (status IN + ('initiated','uploaded_unbound','available','deleting','deleted'))) + """, + """CREATE TABLE IF NOT EXISTS teaql_blob_binding ( + scope TEXT NOT NULL, entity TEXT NOT NULL, entity_id TEXT NOT NULL, + field TEXT NOT NULL, key TEXT, version BIGINT NOT NULL, + PRIMARY KEY (scope,entity,entity_id,field), + FOREIGN KEY (key) REFERENCES teaql_blob_catalog(key)) + """, + "CREATE INDEX IF NOT EXISTS teaql_blob_binding_key ON teaql_blob_binding(key)", + "CREATE UNIQUE INDEX IF NOT EXISTS teaql_blob_catalog_object_key ON teaql_blob_catalog(object_key)", + """CREATE TABLE IF NOT EXISTS teaql_blob_upload ( + upload_id TEXT PRIMARY KEY, scope TEXT NOT NULL, replay_key TEXT NOT NULL, + request_hash TEXT NOT NULL, key TEXT NOT NULL, entity TEXT NOT NULL, + entity_id TEXT NOT NULL, field TEXT NOT NULL, file_name TEXT NOT NULL, + extension TEXT NOT NULL, declared_size BIGINT NOT NULL, + expires_ms BIGINT NOT NULL, state TEXT NOT NULL CHECK + (state IN ('initiated','written','completed')), + written_sha256 TEXT, written_etag TEXT, + UNIQUE(scope,replay_key), FOREIGN KEY(key) REFERENCES teaql_blob_catalog(key)) + """, + ) + for sql in statements: + try: + await self.transport.execute_sql(CompiledQuery(sql, [])) + except Exception: + raise RuntimeError("BLOB_CATALOG_FAILED") from None + # Existing catalogs have no trusted policy provenance. Keep the added + # columns NULL: a migration cannot infer a host authorization decision. + columns = ( + ("completed_at_ms", "BIGINT"), ("last_unbound_at_ms", "BIGINT"), + ("policy_profile", "TEXT"), ("upload_mode", "TEXT"), + ("unbound_retention_seconds", "BIGINT"), + ("effective_unref_seconds", "BIGINT"), + ("retention_floor_until_ms", "BIGINT"), + ) + for name, kind in columns: + try: + rows = await self.transport.fetch_all_sql(CompiledQuery( + "PRAGMA table_info(teaql_blob_catalog)", [])) + if name not in {row["name"] for row in rows}: + try: + await self.transport.execute_sql(CompiledQuery( + f"ALTER TABLE teaql_blob_catalog ADD COLUMN {name} {kind}", [])) + except Exception: + # A concurrent migrator may have added the column. + rows = await self.transport.fetch_all_sql(CompiledQuery( + "PRAGMA table_info(teaql_blob_catalog)", [])) + if name not in {row["name"] for row in rows}: + raise + except Exception: + raise RuntimeError("BLOB_CATALOG_FAILED") from None + + async def validate_schema(self): + for sql in ( + "SELECT key, scope, object_key, content_version, file_name, content_type, size_bytes, " + "completed_at_ms, last_unbound_at_ms, policy_profile, upload_mode, " + "unbound_retention_seconds, effective_unref_seconds, retention_floor_until_ms, status " + "FROM teaql_blob_catalog WHERE 1 = 0", + "SELECT scope, entity, entity_id, field, key, version " + "FROM teaql_blob_binding WHERE 1 = 0", + "SELECT upload_id, scope, replay_key, request_hash, key, entity, entity_id, " + "field, file_name, extension, declared_size, expires_ms, state, " + "written_sha256, written_etag FROM teaql_blob_upload WHERE 1 = 0", + ): + try: + await self.transport.fetch_all_sql(CompiledQuery(sql, [])) + except Exception: + raise RuntimeError("BLOB_SCHEMA_MISSING") from None + try: + columns = await self.transport.fetch_all_sql(CompiledQuery("PRAGMA table_info(teaql_blob_binding)", [])) + names = {row["name"]: row for row in columns} + if names["key"]["notnull"] != 0 or names["version"]["notnull"] != 1: + raise RuntimeError("BLOB_SCHEMA_MISSING") + indexes = await self.transport.fetch_all_sql(CompiledQuery( + "PRAGMA index_list(teaql_blob_catalog)", [])) + if not any(row.get("name") == "teaql_blob_catalog_object_key" + and row.get("unique") == 1 for row in indexes): + raise RuntimeError("BLOB_SCHEMA_MISSING") + index_columns = await self.transport.fetch_all_sql(CompiledQuery( + "PRAGMA index_info(teaql_blob_catalog_object_key)", [])) + if len(index_columns) != 1 or index_columns[0].get("name") != "object_key": + raise RuntimeError("BLOB_SCHEMA_MISSING") + except Exception: + raise RuntimeError("BLOB_SCHEMA_MISSING") from None + + async def current_read(self, slot: AuthorizedBlobSlot) -> BlobCatalogRead: + """Reread the exact current binding; possession of the stable Key grants no read.""" + if (not isinstance(slot, AuthorizedBlobSlot) or not _scope(slot.scope) + or not _name(slot.entity) or not _entity_id(slot.entity_id) + or not _name(slot.field) or type(slot.binding_version) is not int + or slot.binding_version < 1 or not isinstance(slot.reference, BlobRef)): + raise RuntimeError("BLOB_READ_DENIED") + try: + rows = await self.transport.fetch_all_sql(CompiledQuery( + "SELECT c.object_key,c.content_version,c.file_name,c.content_type,c.size_bytes " + "FROM teaql_blob_binding b JOIN teaql_blob_catalog c ON c.key = b.key " + "WHERE b.scope = ? AND b.entity = ? AND b.entity_id = ? AND b.field = ? " + "AND b.key = ? AND b.version = ? AND c.scope = b.scope AND c.status = 'available'", + [Value.Text(slot.scope), Value.Text(slot.entity), Value.Text(slot.entity_id), + Value.Text(slot.field), Value.Text(slot.reference.key), Value.I64(slot.binding_version)])) + except Exception: + raise RuntimeError("BLOB_CATALOG_FAILED") from None + if len(rows) != 1: + raise RuntimeError("BLOB_READ_DENIED") + row = rows[0] + values = (row.get("object_key"), row.get("content_version"), + row.get("file_name"), row.get("content_type")) + size = row.get("size_bytes") + if (any(not isinstance(value, str) or not value for value in values) + or type(size) is not int or size < 0): + raise RuntimeError("BLOB_CATALOG_FAILED") + return BlobCatalogRead(*values, size) + + async def bind_in_graph(self, graph, actor, change: BlobBindingChange, + authorization: BlobBindingAuthorization, *, lease=None) -> int: + if graph is None or not callable(getattr(graph, "with_sql_transport", None)): + raise RuntimeError("BLOB_TRANSACTION_REQUIRED") + + async def bind(sql): + if actor is not graph.context: + raise RuntimeError("BLOB_BIND_DENIED") + if (not isinstance(change, BlobBindingChange) or not isinstance(change.target, BlobBindingTarget) + or not _name(change.target.entity) or not _entity_id(change.target.entity_id) + or not _name(change.target.field) or not isinstance(change.expected_version, int) + or isinstance(change.expected_version, bool) or change.expected_version < 0 + or not isinstance(change.next_version, int) or isinstance(change.next_version, bool) + or change.next_version <= change.expected_version + or change.action not in ("Create", "Update", "Delete") + or change.action == "Create" and change.expected_version != 0 + or change.action == "Delete" and change.next is not None + or change.next is not None and not isinstance(change.next, BlobRef) + or authorization is None): + raise RuntimeError("BLOB_REFERENCE_INVALID") + try: + scope = authorization.scope(actor) + except Exception: + raise RuntimeError("BLOB_BIND_DENIED") from None + if not _scope(scope): + raise RuntimeError("BLOB_BIND_DENIED") + target = change.target + identity = [Value.Text(scope), Value.Text(target.entity), + Value.Text(target.entity_id), Value.Text(target.field)] + try: + rows = await sql.fetch_all_sql(CompiledQuery( + "SELECT key, version FROM teaql_blob_binding WHERE scope = ? AND entity = ? " + "AND entity_id = ? AND field = ?", identity)) + except Exception: + raise RuntimeError("BLOB_CATALOG_FAILED") from None + if len(rows) > 1: + raise RuntimeError("BLOB_CATALOG_FAILED") + row = rows[0] if rows else None + version = row["version"] if row else 0 + if not isinstance(version, int) or version < 0: + raise RuntimeError("BLOB_CATALOG_FAILED") + if version != change.expected_version: + raise RuntimeError("BLOB_BIND_DENIED") + try: + previous = BlobRef.from_sql(row["key"]) if row and row["key"] is not None else None + except ValueError: + raise RuntimeError("BLOB_CATALOG_FAILED") from None + try: + allowed = await authorization.authorize(actor, target, previous, change.next, change.action) + except Exception: + raise RuntimeError("BLOB_BIND_DENIED") from None + if not allowed: + raise RuntimeError("BLOB_BIND_DENIED") + if change.next: + try: + available = await sql.fetch_all_sql(CompiledQuery( + "SELECT key,status,completed_at_ms,last_unbound_at_ms,policy_profile," + "upload_mode,unbound_retention_seconds,effective_unref_seconds," + "retention_floor_until_ms FROM teaql_blob_catalog WHERE key = ? AND scope = ? " + "AND status IN ('uploaded_unbound','available')", + [Value.Text(change.next.key), Value.Text(scope)])) + except Exception: + raise RuntimeError("BLOB_CATALOG_FAILED") from None + if len(available) != 1: + raise RuntimeError("BLOB_BIND_DENIED") + blob = available[0] + current = getattr(authorization, "current_policy_on_lease", None) + if (not callable(current) or not _scope(blob.get("policy_profile")) + or blob.get("upload_mode") != "server_stream" + or not valid_time(blob.get("completed_at_ms")) + or not valid_seconds(blob.get("unbound_retention_seconds")) + or not valid_seconds(blob.get("effective_unref_seconds"))): + raise RuntimeError("BLOB_BIND_DENIED") + try: + policy = await current(actor, sql, target) + except Exception: + raise RuntimeError("BLOB_BIND_DENIED") from None + if not isinstance(policy, BlobRetentionPolicy) or policy.upload_mode != "server_stream": + raise RuntimeError("BLOB_BIND_DENIED") + now = self.now_ms() + if not valid_time(now): + raise RuntimeError("BLOB_POLICY_INVALID") + eligible_at = None + if blob["status"] == "available": + try: + bound = await sql.fetch_all_sql(CompiledQuery( + "SELECT 1 FROM teaql_blob_binding WHERE key = ? LIMIT 1", + [Value.Text(change.next.key)])) + except Exception: + raise RuntimeError("BLOB_CATALOG_FAILED") from None + if not bound: + last_unbound = blob.get("last_unbound_at_ms") + floor = blob.get("retention_floor_until_ms") + if not valid_time(last_unbound) or floor is not None and not valid_time(floor): + raise RuntimeError("BLOB_BIND_DENIED") + eligible_at = unreferenced_eligible_at( + last_unbound, blob["effective_unref_seconds"], floor) + require_bindable(blob["status"], now, blob["completed_at_ms"], + blob["unbound_retention_seconds"], eligible_at) + accepted_floor = retention_deadline(now, policy.minimum_retention_seconds) + key = Value.Text(change.next.key) if change.next else Value.Null() + try: + if row: + affected, _ = await sql.execute_sql(CompiledQuery( + "UPDATE teaql_blob_binding SET key = ?, version = ? WHERE scope = ? " + "AND entity = ? AND entity_id = ? AND field = ? AND version = ?", + [key, Value.I64(change.next_version), *identity, Value.I64(version)])) + else: + affected, _ = await sql.execute_sql(CompiledQuery( + "INSERT INTO teaql_blob_binding (scope, entity, entity_id, field, key, version) " + "VALUES (?, ?, ?, ?, ?, ?)", + [*identity, key, Value.I64(change.next_version)])) + except Exception: + raise RuntimeError("BLOB_CATALOG_FAILED") from None + if affected != 1: + raise RuntimeError("BLOB_BIND_DENIED") + if change.next: + try: + promoted, _ = await sql.execute_sql(CompiledQuery( + "UPDATE teaql_blob_catalog SET status = 'available', last_unbound_at_ms = NULL " + "WHERE key = ? AND scope = ? " + "AND status IN ('uploaded_unbound','available')", + [Value.Text(change.next.key), Value.Text(scope)])) + except Exception: + raise RuntimeError("BLOB_CATALOG_FAILED") from None + if promoted != 1: + raise RuntimeError("BLOB_BIND_DENIED") + try: + retained, _ = await sql.execute_sql(CompiledQuery( + "UPDATE teaql_blob_catalog SET " + "effective_unref_seconds=MAX(effective_unref_seconds,?), " + "retention_floor_until_ms=CASE WHEN retention_floor_until_ms IS NULL " + "OR retention_floor_until_ms str: ... + def field(self, context, target: BlobUploadTarget) -> BlobFieldSpec: ... + async def authorize(self, context, target: BlobUploadTarget, action: str, + input: BlobUploadInput) -> bool: ... + + +class BlobUploadPolicyAuthorization(Protocol): + """Optional trusted policy lookup on the physical begin-upload SQL lease.""" + + async def current_policy_on_lease(self, context, lease: SqlTransactionTransport, + target: BlobUploadTarget) -> BlobRetentionPolicy: ... + + +class BlobUploadLeaseAuthorization(Protocol): + """Optional host authorization on the physical upload transaction.""" + + async def current_on_lease(self, context, lease: SqlTransactionTransport, + target: BlobUploadTarget, action: str, + input: BlobUploadInput) -> BlobUploadAuthorizationState: ... + + +class BlobContentVerifier(Protocol): + def verify(self, stream: BinaryIO, extension: str, maximum_bytes: int) -> BlobVerifiedContent: ... + + +def _token(): + return base64.urlsafe_b64encode(secrets.token_bytes(24)).rstrip(b"=").decode("ascii") + + +def _text(value, maximum): + if not isinstance(value, str) or not value or value != value.strip(): + return False + try: + return len(value.encode("utf-8")) <= maximum and not any( + ord(c) < 32 or 127 <= ord(c) <= 159 for c in value) + except UnicodeEncodeError: + return False + + +def _query(sql, *params): + return CompiledQuery(sql, list(params)) + + +class _CountingReader: + def __init__(self, stream: BinaryIO, maximum: int): + self.stream = stream + self.maximum = maximum + self.digest = sha256() + self.count = 0 + self.eof = False + + def read(self, size=-1): + block = self.stream.read(size) + if not isinstance(block, bytes): + raise RuntimeError("BLOB_UPLOAD_INCOMPLETE") + self.count += len(block) + if self.count > self.maximum: + raise RuntimeError("BLOB_SIZE_EXCEEDED") + self.digest.update(block) + if not block and size != 0: + self.eof = True + return block + + def close(self): + # ObjectStorage.put closes this borrowed reader. The upload service + # owns the caller stream through conflict recovery and SQL promotion. + pass + + +class _ReplayMismatch(Exception): + pass + + +class SqliteBlobUploadService: + """Host-invoked upload path; schema migration remains explicit.""" + + def __init__(self, catalog: SqliteBlobCatalog, authorization: BlobUploadAuthorization, + verifier: BlobContentVerifier, maximum_bytes: int, lease_seconds: int = 900, + now_ms: Callable[[], int] | None = None): + if (not isinstance(catalog, SqliteBlobCatalog) + or not isinstance(catalog.transport, SqlTransactionTransport) + or authorization is None or verifier is None + or not callable(getattr(verifier, "verify", None)) + or type(maximum_bytes) is not int or not 1 <= maximum_bytes <= MAX_PUT_SIZE + or type(lease_seconds) is not int or not 1 <= lease_seconds <= 3600): + raise RuntimeError("BLOB_POLICY_INVALID") + self.catalog = catalog + self.authorization = authorization + self.verifier = verifier + self.maximum_bytes = maximum_bytes + self.lease_seconds = lease_seconds + self.now_ms = now_ms or (lambda: time.time_ns() // 1_000_000) + + def _normalize(self, input: BlobUploadInput): + if (not isinstance(input, BlobUploadInput) or type(input.declared_size) is not int + or not 0 <= input.declared_size <= self.maximum_bytes): + raise RuntimeError("BLOB_SIZE_EXCEEDED") + if not _text(input.file_name, 1024): + raise RuntimeError("BLOB_FORMAT_NOT_ALLOWED") + name = input.file_name.replace("\\", "/").rsplit("/", 1)[-1] + if not _text(name, 512): + raise RuntimeError("BLOB_FORMAT_NOT_ALLOWED") + extension = name.rsplit(".", 1)[-1].lower() + if blob_mime_for_extension(extension) is None: + raise RuntimeError("BLOB_FORMAT_NOT_ALLOWED") + declared = input.declared_content_type.strip().lower() if isinstance( + input.declared_content_type, str) else None + if declared is None or declared and not _text(declared, 128): + raise RuntimeError("BLOB_FORMAT_NOT_ALLOWED") + return BlobUploadInput(name, input.declared_size, declared), extension + + async def _current(self, actor, target, action, input, expected_scope=None, lease=None): + if (actor is None or not isinstance(target, BlobUploadTarget) + or not _name(target.entity) or not _name(target.field) + or target.entity_id != "" and not _entity_id(target.entity_id)): + raise RuntimeError("BLOB_REFERENCE_INVALID") + try: + current_on_lease = getattr(self.authorization, "current_on_lease", None) + if lease is not None and current_on_lease is not None: + state = await current_on_lease(actor, lease, target, action, input) + if not isinstance(state, BlobUploadAuthorizationState): + raise TypeError("invalid upload authorization state") + scope, field, allowed = state.scope, state.field, state.allowed + else: + scope = self.authorization.scope(actor) + field = self.authorization.field(actor, target) + allowed = await self.authorization.authorize(actor, target, action, input) + except Exception: + raise RuntimeError("BLOB_UPLOAD_DENIED") from None + if (not _scope(scope) or expected_scope is not None and scope != expected_scope + or not isinstance(field, BlobFieldSpec) or field.entity != target.entity + or field.field != target.field or type(allowed) is not bool or not allowed): + raise RuntimeError("BLOB_UPLOAD_DENIED") + extension = input.file_name.rsplit(".", 1)[-1].lower() + if extension not in field.allowed_extensions: + raise RuntimeError("BLOB_FORMAT_NOT_ALLOWED") + return scope + + async def _transaction(self, work): + try: + tx = await self.catalog.transport.begin_sql() + except Exception: + raise RuntimeError("BLOB_CATALOG_FAILED") from None + committed = False + try: + result = await work(tx) + try: + await tx.commit_sql() + except Exception: + raise RuntimeError("BLOB_CATALOG_FAILED") from None + committed = True + return result + finally: + if not committed: + with suppress(Exception): + await tx.rollback_sql() + + async def _policy_on_lease(self, actor, tx, target): + current = getattr(self.authorization, "current_policy_on_lease", None) + if current is None: + return None + if not callable(current): + raise RuntimeError("BLOB_POLICY_INVALID") + try: + policy = await current(actor, tx, target) + except Exception: + raise RuntimeError("BLOB_POLICY_INVALID") from None + if not isinstance(policy, BlobRetentionPolicy): + raise RuntimeError("BLOB_POLICY_INVALID") + if policy.upload_mode != "server_stream": + raise RuntimeError("BLOB_CAPABILITY_MISSING") + return policy + + async def begin(self, actor, target: BlobUploadTarget, input: BlobUploadInput, + replay_key: str) -> BlobUploadSession: + normalized, extension = self._normalize(input) + if not _text(replay_key, 256): + raise RuntimeError("BLOB_REFERENCE_INVALID") + scope = await self._current(actor, target, "Begin", normalized) + request = json.dumps([target.entity, target.entity_id, target.field, normalized.file_name, + normalized.declared_size, normalized.declared_content_type], + ensure_ascii=False, separators=(",", ":")).encode("utf-8") + request_hash = sha256(request).hexdigest() + + async def work(tx): + try: + rows = await tx.fetch_all_sql(_query( + "SELECT upload_id,key,request_hash,expires_ms FROM teaql_blob_upload " + "WHERE scope = ? AND replay_key = ?", Value.Text(scope), Value.Text(replay_key))) + except Exception: + raise RuntimeError("BLOB_CATALOG_FAILED") from None + if len(rows) > 1: + raise RuntimeError("BLOB_CATALOG_FAILED") + if rows: + row = rows[0] + if row.get("request_hash") != request_hash: + raise RuntimeError("BLOB_IDEMPOTENCY_CONFLICT") + await self._current(actor, target, "Begin", normalized, scope, tx) + try: + expires_ms = row["expires_ms"] + if type(expires_ms) is not int: + raise RuntimeError("BLOB_CATALOG_FAILED") + if self.now_ms() >= expires_ms: + raise RuntimeError("BLOB_UPLOAD_EXPIRED") + return BlobUploadSession(row["upload_id"], BlobRef(row["key"]), + datetime.fromtimestamp(expires_ms / 1000, timezone.utc)) + except (KeyError, TypeError, ValueError, OverflowError): + raise RuntimeError("BLOB_CATALOG_FAILED") from None + reference = BlobRef("blob_" + _token()) + upload_id = "bu_" + _token() + object_key = "blob/" + _token() + expires_ms = self.now_ms() + self.lease_seconds * 1000 + await self._current(actor, target, "Begin", normalized, scope, tx) + policy = await self._policy_on_lease(actor, tx, target) + profile = Value.Text(policy.profile) if policy else Value.Null() + mode = Value.Text(policy.upload_mode) if policy else Value.Null() + unbound = Value.I64(policy.unbound_retention_seconds) if policy else Value.Null() + unreferenced = Value.I64(policy.unreferenced_retention_seconds) if policy else Value.Null() + try: + await tx.execute_sql(_query( + "INSERT INTO teaql_blob_catalog " + "(key,scope,object_key,content_version,file_name,content_type,size_bytes,status," + "policy_profile,upload_mode,unbound_retention_seconds,effective_unref_seconds) " + "VALUES (?,?,?,?,?,?,?,?,?,?,?,?)", Value.Text(reference.key), Value.Text(scope), + Value.Text(object_key), Value.Text(""), Value.Text(normalized.file_name), + Value.Text(""), Value.I64(0), Value.Text("initiated"), + profile, mode, unbound, unreferenced)) + await tx.execute_sql(_query( + "INSERT INTO teaql_blob_upload " + "(upload_id,scope,replay_key,request_hash,key,entity,entity_id,field,file_name," + "extension,declared_size,expires_ms,state) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?)", + Value.Text(upload_id), Value.Text(scope), Value.Text(replay_key), + Value.Text(request_hash), Value.Text(reference.key), Value.Text(target.entity), + Value.Text(target.entity_id), Value.Text(target.field), + Value.Text(normalized.file_name), Value.Text(extension), + Value.I64(normalized.declared_size), Value.I64(expires_ms), + Value.Text("initiated"))) + except Exception: + raise RuntimeError("BLOB_CATALOG_FAILED") from None + return BlobUploadSession(upload_id, reference, + datetime.fromtimestamp(expires_ms / 1000, timezone.utc)) + + return await self._transaction(work) + + async def _row(self, actor, upload_id: str, action: str): + if not isinstance(upload_id, str) or re.fullmatch(r"bu_[A-Za-z0-9_-]{32}", upload_id) is None: + raise RuntimeError("BLOB_UPLOAD_DENIED") + try: + scope = self.authorization.scope(actor) + except Exception: + raise RuntimeError("BLOB_UPLOAD_DENIED") from None + if not _scope(scope): + raise RuntimeError("BLOB_UPLOAD_DENIED") + try: + rows = await self.catalog.transport.fetch_all_sql(_query( + "SELECT upload_id,scope,key,entity,entity_id,field,file_name,extension," + "declared_size,expires_ms,state,written_sha256,written_etag " + "FROM teaql_blob_upload WHERE scope = ? AND upload_id = ?", + Value.Text(scope), Value.Text(upload_id))) + except Exception: + raise RuntimeError("BLOB_CATALOG_FAILED") from None + if len(rows) != 1: + raise RuntimeError("BLOB_UPLOAD_DENIED") + row = rows[0] + try: + target = BlobUploadTarget(row["entity"], row["entity_id"], row["field"]) + input = BlobUploadInput(row["file_name"], row["declared_size"]) + await self._current(actor, target, action, input, scope) + if type(row["expires_ms"]) is not int or self.now_ms() >= row["expires_ms"]: + raise RuntimeError("BLOB_UPLOAD_EXPIRED") + if type(row["declared_size"]) is not int or not 0 <= row["declared_size"] <= self.maximum_bytes: + raise RuntimeError("BLOB_SIZE_EXCEEDED") + if row["extension"] != row["file_name"].rsplit(".", 1)[-1].lower(): + raise RuntimeError("BLOB_CATALOG_FAILED") + except (KeyError, TypeError, AttributeError): + raise RuntimeError("BLOB_CATALOG_FAILED") from None + return row + + async def _object_key(self, actor, row): + try: + rows = await self.catalog.transport.fetch_all_sql(_query( + "SELECT object_key FROM teaql_blob_catalog WHERE key = ? AND scope = ?", + Value.Text(row["key"]), Value.Text(row["scope"]))) + if len(rows) == 1: + return ObjectKey(rows[0]["object_key"]) + except Exception: + pass + raise RuntimeError("BLOB_CATALOG_FAILED") + + async def write(self, actor, upload_id: str, source: BinaryIO) -> int: + if source is None or not callable(getattr(source, "read", None)): + raise RuntimeError("BLOB_UPLOAD_INCOMPLETE") + source_closed = False + + def close_source(): + nonlocal source_closed + if source_closed: + return + source_closed = True + try: + source.close() + except Exception: + raise RuntimeError("BLOB_STORE_FAILED") from None + + try: + row = await self._row(actor, upload_id, "Write") + key = await self._object_key(actor, row) + mime = blob_mime_for_extension(row["extension"]) + reader = _CountingReader(source, row["declared_size"]) + target = BlobUploadTarget(row["entity"], row["entity_id"], row["field"]) + + async def work(tx): + # BEGIN IMMEDIATE remains owned until the File write and journal + # transition finish. A second writer or collector must recheck + # the authoritative row after this owner commits. + try: + current = await tx.fetch_all_sql(_query( + "SELECT state,expires_ms,key,written_sha256,written_etag FROM teaql_blob_upload " + "WHERE upload_id=? AND scope=?", Value.Text(upload_id), + Value.Text(row["scope"]))) + catalog = await tx.fetch_all_sql(_query( + "SELECT object_key,status,content_version,content_type,size_bytes " + "FROM teaql_blob_catalog WHERE key=? AND scope=?", + Value.Text(row["key"]), Value.Text(row["scope"]))) + except Exception: + raise RuntimeError("BLOB_CATALOG_FAILED") from None + if (len(current) != 1 or current[0]["key"] != row["key"] or + current[0]["expires_ms"] != row["expires_ms"] or + len(catalog) != 1 or + catalog[0]["object_key"] != key.value): + raise RuntimeError("BLOB_IDEMPOTENCY_CONFLICT") + state = current[0]["state"] + if state in ("written", "completed"): + recorded_sha = current[0]["written_sha256"] + recorded_etag = current[0]["written_etag"] + if (not isinstance(recorded_sha, str) or + re.fullmatch(r"[0-9a-f]{64}", recorded_sha) is None or + not isinstance(recorded_etag, str) or not recorded_etag or + (state == "written" and catalog[0]["status"] != "initiated") or + (state == "completed" and ( + catalog[0]["status"] not in ("uploaded_unbound", "available") or + catalog[0]["content_version"] != recorded_etag or + catalog[0]["content_type"] != mime or + catalog[0]["size_bytes"] != row["declared_size"]))): + raise RuntimeError("BLOB_IDEMPOTENCY_CONFLICT") + await self._current(actor, target, "Write", + BlobUploadInput(row["file_name"], row["declared_size"]), + row["scope"], tx) + if self.now_ms() >= row["expires_ms"]: + raise RuntimeError("BLOB_UPLOAD_EXPIRED") + + def verify_replay(): + try: + while reader.read(65536): + pass + if (not reader.eof or reader.count != row["declared_size"] or + reader.digest.hexdigest() != recorded_sha): + raise _ReplayMismatch() + store = actor.object_storage("recheck written Blob upload") + before = store.head(key) + if (before is None or before.size != reader.count or + before.etag != recorded_etag or before.content_type != mime): + raise _ReplayMismatch() + digest, count = sha256(), 0 + with store.get(key) as content: + if (content.metadata.size != before.size or + content.metadata.etag != before.etag or + content.metadata.content_type != mime): + raise _ReplayMismatch() + while True: + block = content.stream.read(65536) + if not isinstance(block, bytes) or count + len(block) > reader.count: + raise _ReplayMismatch() + if not block: + break + digest.update(block) + count += len(block) + after = store.head(key) + if (count != reader.count or digest.hexdigest() != recorded_sha or + after is None or after.size != before.size or + after.etag != before.etag or after.content_type != mime): + raise _ReplayMismatch() + return count + except _ReplayMismatch: + raise RuntimeError("BLOB_IDEMPOTENCY_CONFLICT") from None + except Exception: + raise RuntimeError("BLOB_STORE_FAILED") from None + + count = await asyncio.to_thread(verify_replay) + await self._current(actor, target, "Write", + BlobUploadInput(row["file_name"], row["declared_size"]), + row["scope"], tx) + if self.now_ms() >= row["expires_ms"]: + raise RuntimeError("BLOB_UPLOAD_EXPIRED") + close_source() + return count + if state != "initiated" or catalog[0]["status"] != "initiated": + raise RuntimeError("BLOB_IDEMPOTENCY_CONFLICT") + await self._current(actor, target, "Write", + BlobUploadInput(row["file_name"], row["declared_size"]), + row["scope"], tx) + if self.now_ms() >= row["expires_ms"]: + raise RuntimeError("BLOB_UPLOAD_EXPIRED") + store = actor.object_storage("write Blob upload") + try: + metadata = await asyncio.to_thread( + store.put, key, reader, + PutOptions(row["declared_size"], WriteMode.CREATE_ONLY, mime)) + except StorageError as error: + if error.code != StorageErrorCode.CONFLICT: + raise RuntimeError("BLOB_STORE_FAILED") from None + + def recover_existing(): + # A prior put can have committed while its SQL promotion + # rolled back. Only the same complete source may adopt it. + if not reader.eof or reader.count != row["declared_size"]: + raise RuntimeError("BLOB_STORE_FAILED") + before = store.head(key) + if (before is None or before.size != reader.count or + before.content_type != mime or not before.etag): + raise RuntimeError("BLOB_STORE_FAILED") + digest, count = sha256(), 0 + with store.get(key) as content: + if (content.metadata.size != before.size or + content.metadata.etag != before.etag or + content.metadata.content_type != mime): + raise RuntimeError("BLOB_STORE_FAILED") + while True: + block = content.stream.read(65536) + if not isinstance(block, bytes) or count + len(block) > reader.count: + raise RuntimeError("BLOB_STORE_FAILED") + if not block: + break + digest.update(block) + count += len(block) + after = store.head(key) + if (count != reader.count or digest.hexdigest() != reader.digest.hexdigest() + or after is None or after.size != before.size or + after.etag != before.etag or after.content_type != mime): + raise RuntimeError("BLOB_STORE_FAILED") + return before + + try: + metadata = await asyncio.to_thread(recover_existing) + except Exception: + raise RuntimeError("BLOB_STORE_FAILED") from None + except Exception: + raise RuntimeError("BLOB_STORE_FAILED") from None + await self._current(actor, target, "Write", + BlobUploadInput(row["file_name"], row["declared_size"]), + row["scope"], tx) + now = self.now_ms() + if now >= row["expires_ms"]: + raise RuntimeError("BLOB_UPLOAD_EXPIRED") + if (not reader.eof or reader.count != row["declared_size"] or + metadata.size != reader.count or not metadata.etag): + raise RuntimeError("BLOB_UPLOAD_INCOMPLETE") + close_source() + digest = reader.digest.hexdigest() + try: + affected, _ = await tx.execute_sql(_query( + "UPDATE teaql_blob_upload SET state='written', written_sha256=?, written_etag=? " + "WHERE upload_id=? AND scope=? AND state='initiated' AND expires_ms > ? " + "AND EXISTS (SELECT 1 FROM teaql_blob_catalog " + "WHERE key=teaql_blob_upload.key AND scope=teaql_blob_upload.scope " + "AND status='initiated')", + Value.Text(digest), Value.Text(metadata.etag), + Value.Text(upload_id), Value.Text(row["scope"]), Value.I64(now))) + except Exception: + raise RuntimeError("BLOB_CATALOG_FAILED") from None + if affected != 1: + raise RuntimeError("BLOB_IDEMPOTENCY_CONFLICT") + return reader.count + return await self._transaction(work) + finally: + if not source_closed: + with suppress(Exception): + source.close() + + async def complete(self, actor, upload_id: str) -> BlobRef: + row = await self._row(actor, upload_id, "Confirm") + if row["state"] == "completed": + if (not isinstance(row["written_sha256"], str) or + re.fullmatch(r"[0-9a-f]{64}", row["written_sha256"]) is None or + not row["written_etag"]): + raise RuntimeError("BLOB_UPLOAD_INCOMPLETE") + target = BlobUploadTarget(row["entity"], row["entity_id"], row["field"]) + mime = blob_mime_for_extension(row["extension"]) + input = BlobUploadInput(row["file_name"], row["declared_size"], mime) + + async def replay_work(tx): + try: + current = await tx.fetch_all_sql(_query( + "SELECT state,expires_ms,key,written_sha256,written_etag " + "FROM teaql_blob_upload WHERE upload_id=? AND scope=?", + Value.Text(upload_id), Value.Text(row["scope"]))) + rows = await tx.fetch_all_sql(_query( + "SELECT object_key,content_version,content_type,size_bytes,status " + "FROM teaql_blob_catalog WHERE key=? AND scope=?", + Value.Text(row["key"]), Value.Text(row["scope"]))) + except Exception: + raise RuntimeError("BLOB_CATALOG_FAILED") from None + if (len(current) != 1 or current[0]["state"] != "completed" or + current[0]["expires_ms"] != row["expires_ms"] or + current[0]["key"] != row["key"] or + current[0]["written_sha256"] != row["written_sha256"] or + current[0]["written_etag"] != row["written_etag"] or + len(rows) != 1): + raise RuntimeError("BLOB_UPLOAD_INCOMPLETE") + catalog = rows[0] + if (catalog["status"] not in ("uploaded_unbound", "available") or + catalog["content_version"] != row["written_etag"] or + catalog["content_type"] != mime or + type(catalog["size_bytes"]) is not int or + catalog["size_bytes"] != row["declared_size"]): + raise RuntimeError("BLOB_UPLOAD_INCOMPLETE") + try: + key = ObjectKey(catalog["object_key"]) + except Exception: + raise RuntimeError("BLOB_UPLOAD_INCOMPLETE") from None + await self._current(actor, target, "Confirm", input, row["scope"], tx) + if self.now_ms() >= row["expires_ms"]: + raise RuntimeError("BLOB_UPLOAD_EXPIRED") + + def recheck_stored(): + try: + metadata = actor.object_storage("recheck completed Blob upload").head(key) + if (metadata is None or metadata.size != row["declared_size"] or + metadata.etag != row["written_etag"]): + raise RuntimeError("BLOB_UPLOAD_INCOMPLETE") + except Exception: + raise RuntimeError("BLOB_UPLOAD_INCOMPLETE") from None + + await asyncio.to_thread(recheck_stored) + await self._current(actor, target, "Confirm", input, row["scope"], tx) + if self.now_ms() >= row["expires_ms"]: + raise RuntimeError("BLOB_UPLOAD_EXPIRED") + return BlobRef(row["key"]) + + return await self._transaction(replay_work) + if row["state"] != "written" or not row["written_sha256"] or not row["written_etag"]: + raise RuntimeError("BLOB_UPLOAD_INCOMPLETE") + key = await self._object_key(actor, row) + + def verify_stored(): + try: + store = actor.object_storage("verify Blob upload") + before = store.head(key) + if (before is None or before.size != row["declared_size"] + or before.etag != row["written_etag"]): + raise RuntimeError("BLOB_UPLOAD_INCOMPLETE") + with store.get(key) as content: + if (content.metadata.size != before.size or + content.metadata.etag != before.etag): + raise RuntimeError("BLOB_UPLOAD_INCOMPLETE") + reader = _CountingReader(content.stream, self.maximum_bytes) + verified = self.verifier.verify(reader, row["extension"], self.maximum_bytes) + digest = reader.digest.hexdigest() + mime = blob_mime_for_extension(row["extension"]) + if (not isinstance(verified, BlobVerifiedContent) or not reader.eof + or reader.count != before.size or verified.size_bytes != reader.count + or verified.sha256 != digest or verified.sha256 != row["written_sha256"] + or verified.content_type != mime): + raise RuntimeError("BLOB_UPLOAD_INCOMPLETE") + after = store.head(key) + if after is None or after.size != before.size or after.etag != before.etag: + raise RuntimeError("BLOB_UPLOAD_INCOMPLETE") + return verified + except RuntimeError as error: + if str(error) in ("BLOB_FORMAT_NOT_ALLOWED", "BLOB_SIZE_EXCEEDED", + "BLOB_VERIFICATION_BUSY", "BLOB_CAPABILITY_MISSING"): + raise + raise RuntimeError("BLOB_UPLOAD_INCOMPLETE") from None + except Exception: + raise RuntimeError("BLOB_UPLOAD_INCOMPLETE") from None + + target = BlobUploadTarget(row["entity"], row["entity_id"], row["field"]) + + async def work(tx): + # Keep the same physical SQLite owner through parser I/O and both + # journal promotions. A concurrent confirmation rechecks after it. + try: + current = await tx.fetch_all_sql(_query( + "SELECT state,expires_ms,key,written_sha256,written_etag " + "FROM teaql_blob_upload WHERE upload_id=? AND scope=?", + Value.Text(upload_id), Value.Text(row["scope"]))) + catalog = await tx.fetch_all_sql(_query( + "SELECT object_key,status FROM teaql_blob_catalog WHERE key=? AND scope=?", + Value.Text(row["key"]), Value.Text(row["scope"]))) + except Exception: + raise RuntimeError("BLOB_CATALOG_FAILED") from None + if (len(current) == 1 and current[0]["state"] == "completed" and + current[0]["expires_ms"] == row["expires_ms"] and + current[0]["key"] == row["key"] and + current[0]["written_sha256"] == row["written_sha256"] and + current[0]["written_etag"] == row["written_etag"]): + # Another confirmer won the lease. Reenter the regular replay + # path after releasing this SQL owner so catalog, generation, + # authorization and expiry are checked against current state. + return None + if (len(current) != 1 or current[0]["state"] != "written" or + current[0]["expires_ms"] != row["expires_ms"] or + current[0]["key"] != row["key"] or + current[0]["written_sha256"] != row["written_sha256"] or + current[0]["written_etag"] != row["written_etag"] or + len(catalog) != 1 or catalog[0]["status"] != "initiated" or + catalog[0]["object_key"] != key.value): + raise RuntimeError("BLOB_IDEMPOTENCY_CONFLICT") + await self._current(actor, target, "Confirm", + BlobUploadInput(row["file_name"], row["declared_size"]), + row["scope"], tx) + now = self.now_ms() + if not valid_time(now) or now >= row["expires_ms"]: + raise RuntimeError("BLOB_UPLOAD_EXPIRED") + verified = await asyncio.to_thread(verify_stored) + await self._current(actor, target, "Confirm", + BlobUploadInput(row["file_name"], row["declared_size"], + verified.content_type), row["scope"], tx) + now = self.now_ms() + if not valid_time(now) or now >= row["expires_ms"]: + raise RuntimeError("BLOB_UPLOAD_EXPIRED") + try: + affected, _ = await tx.execute_sql(_query( + "UPDATE teaql_blob_upload SET state='completed' WHERE upload_id=? AND scope=? " + "AND state='written' AND written_sha256=? AND written_etag=? AND expires_ms > ?", + Value.Text(upload_id), Value.Text(row["scope"]), + Value.Text(row["written_sha256"]), Value.Text(row["written_etag"]), Value.I64(now))) + if affected != 1: + raise RuntimeError("BLOB_IDEMPOTENCY_CONFLICT") + affected, _ = await tx.execute_sql(_query( + "UPDATE teaql_blob_catalog SET content_version=?,content_type=?,size_bytes=?, " + "completed_at_ms=?,status='uploaded_unbound' " + "WHERE key=? AND scope=? AND status='initiated'", + Value.Text(row["written_etag"]), Value.Text(verified.content_type), + Value.I64(verified.size_bytes), Value.I64(now), Value.Text(row["key"]), + Value.Text(row["scope"]))) + if affected != 1: + raise RuntimeError("BLOB_IDEMPOTENCY_CONFLICT") + except RuntimeError: + raise + except Exception: + raise RuntimeError("BLOB_CATALOG_FAILED") from None + return BlobRef(row["key"]) + result = await self._transaction(work) + return result if result is not None else await self.complete(actor, upload_id) diff --git a/packages/teaql-secure-document-reference/README.md b/packages/teaql-secure-document-reference/README.md new file mode 100644 index 00000000..525ae098 --- /dev/null +++ b/packages/teaql-secure-document-reference/README.md @@ -0,0 +1,5 @@ +# TeaQL secure document reference + +Optional secure reference protocol and document governance package. The base `teaql` runtime does not import it. Installation, key policy and host authorization remain explicit. + +The current source implements governed tqr1 entity references, tqd1/tqd2 document snapshots, and tqs1 Blob selections with the shared Java/Rust wire vectors. `SecureDocumentReferenceModule.install` passively attaches a trusted snapshot contributor to the runtime's governed view preparation. Full document open/accept/replay, transactional selection consumption, and release gates remain in progress. diff --git a/packages/teaql-secure-document-reference/pyproject.toml b/packages/teaql-secure-document-reference/pyproject.toml new file mode 100644 index 00000000..d2ac86b9 --- /dev/null +++ b/packages/teaql-secure-document-reference/pyproject.toml @@ -0,0 +1,17 @@ +[build-system] +requires = ["setuptools>=61.0"] +build-backend = "setuptools.build_meta" + +[project] +name = "teaql-secure-document-reference" +version = "0.3.1" +description = "Optional TeaQL secure document reference module" +readme = "README.md" +requires-python = ">=3.10" +license = "Apache-2.0" +authors = [{ name = "TeaQL" }] +dependencies = ["teaql==0.3.1"] + +[tool.setuptools.packages.find] +where = ["src"] +include = ["teaql_secure_document_reference*"] diff --git a/packages/teaql-secure-document-reference/src/teaql_secure_document_reference/__init__.py b/packages/teaql-secure-document-reference/src/teaql_secure_document_reference/__init__.py new file mode 100644 index 00000000..d640d308 --- /dev/null +++ b/packages/teaql-secure-document-reference/src/teaql_secure_document_reference/__init__.py @@ -0,0 +1,33 @@ +"""Optional secure-document reference services; no base runtime reverse dependency.""" + +from .round_trip_reference import ( + ContextBoundReferenceRuntime, ReferenceDocumentScope, ReferenceIdentity, + ReferenceKey, RoundTripReferenceError, StaticReferenceKeyProvider, + TrustedReferencePrincipal, VerifiedEntityReference, +) +from .document_round_trip import ( + BlobSelectionClaim, BlobSelectionTarget, BoundDocumentSnapshot, + DocumentBlobBindingSnapshot, DocumentEntitySnapshot, DocumentRelationSnapshot, + DocumentSnapshot, PreparedBoundDocument, SecureDocumentAssembly, + SecureDocumentReferenceModule, VerifiedDocument, +) +from .sqlite_document_service import ( + AcceptedDocument, ContextualDocument, CurrentBoundDocument, + DocumentBlobSelectionSubmission, DocumentBlobSelectionClaim, DocumentCommandReceipt, + VerifiedDocumentBlobSelection, SQLiteContextBoundDocumentService, + SQLiteDocumentCommandJournal, SubmittedDocumentCommand, +) + +__all__ = [ + "ContextBoundReferenceRuntime", "ReferenceDocumentScope", "ReferenceIdentity", + "ReferenceKey", "RoundTripReferenceError", "StaticReferenceKeyProvider", + "TrustedReferencePrincipal", "VerifiedEntityReference", + "BlobSelectionClaim", "BlobSelectionTarget", "BoundDocumentSnapshot", + "DocumentBlobBindingSnapshot", "DocumentEntitySnapshot", "DocumentRelationSnapshot", + "DocumentSnapshot", "PreparedBoundDocument", "SecureDocumentAssembly", + "SecureDocumentReferenceModule", "VerifiedDocument", + "AcceptedDocument", "ContextualDocument", "CurrentBoundDocument", + "DocumentBlobSelectionSubmission", "DocumentBlobSelectionClaim", "DocumentCommandReceipt", + "VerifiedDocumentBlobSelection", "SQLiteContextBoundDocumentService", + "SQLiteDocumentCommandJournal", "SubmittedDocumentCommand", +] diff --git a/packages/teaql-secure-document-reference/src/teaql_secure_document_reference/document_round_trip.py b/packages/teaql-secure-document-reference/src/teaql_secure_document_reference/document_round_trip.py new file mode 100644 index 00000000..3a71d7b7 --- /dev/null +++ b/packages/teaql-secure-document-reference/src/teaql_secure_document_reference/document_round_trip.py @@ -0,0 +1,599 @@ +"""Versioned encrypted document snapshots and document-bound Blob selection.""" + +from __future__ import annotations + +import base64 +import binascii +import hmac +import re +import struct +from dataclasses import dataclass +from datetime import datetime, timezone +from typing import Generic, Protocol, TypeVar + +from cryptography.exceptions import InvalidTag +from cryptography.hazmat.primitives.ciphers.aead import AESGCM +from teaql.runtime.capability import install_runtime_capability +from teaql.runtime.view_preparation import PreparedSecureDocument, ViewPreparationError + +from .round_trip_reference import ( + ContextBoundReferenceRuntime, ReferenceDocumentScope, ReferenceIdentity, + ReferenceKey, RoundTripReferenceError, TrustedReferencePrincipal, + _BASE64URL, _KEY_ID, _i64, _u64, +) + +_MUTATIONS = {"UPDATE": 1, "CREATE_CHILD": 2, "REMOVE_CHILD": 3} +_REVERSE_MUTATIONS = {value: key for key, value in _MUTATIONS.items()} +_MAX_I64 = (1 << 63) - 1 +_BLOB_KEY = re.compile(r"blob_[A-Za-z0-9_-]{1,123}\Z", re.ASCII) + + +def _fail(code="ROUND_TRIP_REFERENCE_INVALID"): + raise RoundTripReferenceError(code) + + +def _text(value: str) -> bytes: + if not isinstance(value, str) or not value.strip(): + _fail() + try: + encoded = value.encode("utf-8", errors="strict") + except UnicodeError: + _fail() + if len(encoded) > 16384: + _fail() + return encoded + + +def _str(value: str) -> bytes: + data = _text(value) + return struct.pack(">I", len(data)) + data + + +def _count(value: int, maximum: int) -> bytes: + if type(value) is not int or value < 0 or value > maximum: + _fail("DOCUMENT_PROJECTION_VIOLATION") + return struct.pack(">I", value) + + +def _flag(value: bool) -> bytes: + if type(value) is not bool: + _fail() + return bytes((int(value),)) + + +def _blob_key(value: str) -> str: + if not isinstance(value, str) or not _BLOB_KEY.fullmatch(value): + _fail() + return value + + +def _scope(scope: ReferenceDocumentScope) -> bytes: + scope.validate() + return b"".join((_str(scope.document_id), _str(scope.purpose), _str(scope.aggregate_type), + _u64(scope.aggregate_id), _i64(scope.aggregate_revision))) + + +def _identity(identity: ReferenceIdentity) -> bytes: + identity.validate() + return b"".join((_str(identity.entity_type), _u64(identity.id), _i64(identity.version))) + + +def _set(values: frozenset[str], maximum: int) -> bytes: + if not isinstance(values, frozenset): + _fail("DOCUMENT_PROJECTION_VIOLATION") + return _count(len(values), maximum) + b"".join(_str(value) for value in sorted(values, key=lambda x: x.encode("utf-8"))) + + +def _mutations(values: frozenset[str]) -> bytes: + if not isinstance(values, frozenset) or any(value not in _MUTATIONS for value in values): + _fail("DOCUMENT_PROJECTION_VIOLATION") + return _count(len(values), 3) + bytes(_MUTATIONS[value] for value in sorted(values, key=_MUTATIONS.get)) + + +@dataclass(frozen=True) +class DocumentEntitySnapshot: + row_key: str + identity: ReferenceIdentity + loaded_fields: frozenset[str] + writable_fields: frozenset[str] + allowed_mutations: frozenset[str] + + +@dataclass(frozen=True) +class DocumentRelationSnapshot: + name: str + row_keys: tuple[str, ...] + completeness: str + writable: bool + allowed_mutations: frozenset[str] + + +@dataclass(frozen=True) +class DocumentSnapshot: + model_fingerprint: str + business_id: str + scope: ReferenceDocumentScope + aggregate: ReferenceIdentity + entities: tuple[DocumentEntitySnapshot, ...] + relations: tuple[DocumentRelationSnapshot, ...] + + +@dataclass(frozen=True) +class DocumentBlobBindingSnapshot: + row_key: str + canonical_field: str + entity_type: str + stable_key: str | None + binding_version: int + nullable: bool + writable: bool + + +@dataclass(frozen=True) +class BoundDocumentSnapshot: + snapshot: DocumentSnapshot + bindings: tuple[DocumentBlobBindingSnapshot, ...] + + +@dataclass(frozen=True) +class BlobSelectionTarget: + scope: ReferenceDocumentScope + target_entity: str + canonical_field: str + kind: str # EXISTING_ROW or NEW_CLIENT_KEY + target_key: str + command_id: str + + +@dataclass(frozen=True) +class BlobSelectionClaim: + target: BlobSelectionTarget + stable_key: str + + +T = TypeVar("T") + + +@dataclass(frozen=True) +class VerifiedDocument(Generic[T]): + document: T + key_id: str + issued_at: datetime + expires_at: datetime + + +@dataclass(frozen=True) +class PreparedBoundDocument: + document: BoundDocumentSnapshot + lifetime_seconds: int + + +class SecureDocumentAssembly(Protocol): + def principal(self, context) -> TrustedReferencePrincipal: ... + + async def snapshot(self, context, graph) -> PreparedBoundDocument: ... + + +class _SecureCapability: + capability = "secure_document.v1" + + def __init__(self, contributor) -> None: + self.contributor = contributor + + def validate_ready(self, context) -> None: + if context.get_resource("secure_document_contributor") is not self.contributor: + raise RuntimeError("DOCUMENT_CONFIGURATION") + + +class _SecureContributor: + def __init__(self, module, context, assembly: SecureDocumentAssembly) -> None: + self.module, self.context, self.assembly = module, context, assembly + + async def prepare(self, context, graph, options) -> PreparedSecureDocument: + if context is not self.context or options.mode != "governed": + raise ViewPreparationError("DOCUMENT_CONFIGURATION") + capability = context.get_resource("teaql.runtime.capability:secure_document.v1") + if not isinstance(capability, _SecureCapability): + raise ViewPreparationError("DOCUMENT_CONFIGURATION") + try: + capability.validate_ready(context) + except Exception: + raise ViewPreparationError("DOCUMENT_CONFIGURATION") from None + principal = self.assembly.principal(context) + prepared = await self.assembly.snapshot(context, graph) + document = prepared.document + try: + for slot in graph.blob_slots: + if not slot.entity_id.isascii() or not slot.entity_id.isdecimal() or \ + slot.entity_id.startswith("0"): + raise ViewPreparationError("DOCUMENT_PROJECTION_VIOLATION") + row = next((entity for entity in document.snapshot.entities + if entity.identity.entity_type == slot.entity and + entity.identity.id == int(slot.entity_id)), None) + binding = next((item for item in document.bindings + if item.row_key == (row.row_key if row else None) and + item.canonical_field == slot.field), None) + if binding is None or binding.stable_key != slot.reference.key or \ + binding.binding_version != slot.binding_version: + raise ViewPreparationError("DOCUMENT_PROJECTION_VIOLATION") + token = self.module.issue_bound_document_snapshot(principal, document, + prepared.lifetime_seconds) + verified = self.module.consume_bound_document_snapshot( + principal, token, document.snapshot.scope.aggregate_type, document.snapshot.scope.purpose) + except RoundTripReferenceError as error: + raise ViewPreparationError(error.code) from None + return PreparedSecureDocument(graph, {"businessId": document.snapshot.business_id, + "documentToken": token}, verified.expires_at) + + +class _Reader: + def __init__(self, data: bytes) -> None: + self.data, self.offset = data, 0 + + def take(self, count: int) -> bytes: + if count < 0 or self.offset + count > len(self.data): + _fail() + value = self.data[self.offset:self.offset + count] + self.offset += count + return value + + def byte(self) -> int: + return self.take(1)[0] + + def flag(self) -> bool: + value = self.byte() + if value not in (0, 1): + _fail() + return value == 1 + + def count(self, maximum: int) -> int: + value = struct.unpack(">I", self.take(4))[0] + if value > maximum: + _fail() + return value + + def text(self) -> str: + value = self.take(self.count(16384)) + try: + result = value.decode("utf-8", errors="strict") + except UnicodeError: + _fail() + _text(result) + return result + + def u64(self) -> int: + return struct.unpack(">Q", self.take(8))[0] + + def i64(self) -> int: + return struct.unpack(">q", self.take(8))[0] + + def end(self) -> None: + if self.offset != len(self.data): + _fail() + + +def _read_scope(reader: _Reader) -> ReferenceDocumentScope: + value = ReferenceDocumentScope(reader.text(), reader.text(), reader.text(), reader.u64(), reader.i64()) + value.validate() + return value + + +def _read_identity(reader: _Reader) -> ReferenceIdentity: + value = ReferenceIdentity(reader.text(), reader.u64(), reader.i64()) + value.validate() + return value + + +def _read_set(reader: _Reader, maximum: int) -> frozenset[str]: + count = reader.count(maximum) + values = [reader.text() for _ in range(count)] + if len(set(values)) != count: + _fail() + return frozenset(values) + + +def _read_mutations(reader: _Reader) -> frozenset[str]: + count = reader.count(3) + values = [] + for _ in range(count): + value = _REVERSE_MUTATIONS.get(reader.byte()) + if value is None or value in values: + _fail() + values.append(value) + return frozenset(values) + + +def _encode_snapshot(snapshot: DocumentSnapshot, version: int, actor: bytes, + issued: int, expires: int) -> bytes: + if snapshot.aggregate.entity_type != snapshot.scope.aggregate_type or \ + snapshot.aggregate.id != snapshot.scope.aggregate_id or \ + snapshot.aggregate.version != snapshot.scope.aggregate_revision: + _fail("DOCUMENT_PROJECTION_VIOLATION") + parts = [bytes((version,)), actor, _i64(issued), _i64(expires), _str(snapshot.model_fingerprint), + _str(snapshot.business_id), _scope(snapshot.scope), _identity(snapshot.aggregate), + _count(len(snapshot.entities), 10000)] + row_keys = set() + for entity in snapshot.entities: + if entity.row_key in row_keys or not entity.writable_fields.issubset(entity.loaded_fields) or \ + not entity.allowed_mutations: + _fail("DOCUMENT_PROJECTION_VIOLATION") + row_keys.add(entity.row_key) + parts.extend((_str(entity.row_key), _identity(entity.identity), + _set(entity.loaded_fields, 1024), _set(entity.writable_fields, 1024), + _mutations(entity.allowed_mutations))) + parts.append(_count(len(snapshot.relations), 1024)) + for relation in snapshot.relations: + if relation.completeness not in ("COMPLETE", "PARTIAL") or \ + (not relation.writable and relation.allowed_mutations) or \ + len(set(relation.row_keys)) != len(relation.row_keys) or \ + any(key not in row_keys for key in relation.row_keys): + _fail("DOCUMENT_PROJECTION_VIOLATION") + parts.extend((_str(relation.name), bytes((1 if relation.completeness == "COMPLETE" else 2,)), + _flag(relation.writable), _mutations(relation.allowed_mutations), + _count(len(relation.row_keys), 10000), + *(_str(key) for key in relation.row_keys))) + return b"".join(parts) + + +def _decode_snapshot(reader: _Reader, version: int): + if reader.byte() != version: + _fail() + actor, issued, expires = reader.take(32), reader.i64(), reader.i64() + model_fingerprint, business_id = reader.text(), reader.text() + scope, aggregate = _read_scope(reader), _read_identity(reader) + entities = [] + for _ in range(reader.count(10000)): + entities.append(DocumentEntitySnapshot(reader.text(), _read_identity(reader), + _read_set(reader, 1024), _read_set(reader, 1024), + _read_mutations(reader))) + relations = [] + for _ in range(reader.count(1024)): + name, completeness = reader.text(), reader.byte() + if completeness not in (1, 2): + _fail() + writable, mutations = reader.flag(), _read_mutations(reader) + rows = tuple(reader.text() for _ in range(reader.count(10000))) + relations.append(DocumentRelationSnapshot(name, rows, + "COMPLETE" if completeness == 1 else "PARTIAL", + writable, mutations)) + snapshot = DocumentSnapshot(model_fingerprint, business_id, scope, aggregate, + tuple(entities), tuple(relations)) + _encode_snapshot(snapshot, version, actor, issued, expires) + return actor, issued, expires, snapshot + + +def _bindings(document: BoundDocumentSnapshot) -> tuple[DocumentBlobBindingSnapshot, ...]: + values = tuple(sorted(document.bindings, key=lambda x: (x.row_key.encode(), x.canonical_field.encode()))) + if len(values) > 10000: + _fail("DOCUMENT_PROJECTION_VIOLATION") + previous = None + for binding in values: + for value in (binding.row_key, binding.canonical_field, binding.entity_type): + _text(value) + if value != value.strip(): + _fail("DOCUMENT_PROJECTION_VIOLATION") + pair = (binding.row_key, binding.canonical_field) + if pair == previous: + _fail("DOCUMENT_PROJECTION_VIOLATION") + previous = pair + entity = next((row for row in document.snapshot.entities if row.row_key == binding.row_key), None) + if entity is None or entity.identity.entity_type != binding.entity_type or \ + binding.canonical_field not in entity.loaded_fields or \ + (binding.writable and binding.canonical_field not in entity.writable_fields): + _fail("DOCUMENT_PROJECTION_VIOLATION") + if binding.stable_key is not None: + _blob_key(binding.stable_key) + _u64(binding.binding_version) + _flag(binding.nullable) + _flag(binding.writable) + return values + + +def _target_allowed(document: BoundDocumentSnapshot, target: BlobSelectionTarget) -> None: + target.scope.validate() + for value in (target.target_entity, target.canonical_field, target.target_key, target.command_id): + _text(value) + if value != value.strip(): + _fail() + if document.snapshot.scope != target.scope: + _fail("ROUND_TRIP_REFERENCE_SCOPE_MISMATCH") + if target.kind == "EXISTING_ROW": + entity = next((row for row in document.snapshot.entities if row.row_key == target.target_key), None) + if entity is None or entity.identity.entity_type != target.target_entity or \ + target.canonical_field not in entity.writable_fields or \ + not any(binding.row_key == target.target_key and + binding.canonical_field == target.canonical_field and binding.writable + for binding in document.bindings): + _fail("DOCUMENT_PROJECTION_VIOLATION") + elif target.kind == "NEW_CLIENT_KEY": + if not any(relation.completeness == "COMPLETE" and relation.writable and + "CREATE_CHILD" in relation.allowed_mutations for relation in document.snapshot.relations): + _fail("DOCUMENT_PROJECTION_VIOLATION") + else: + _fail() + + +class SecureDocumentReferenceModule(ContextBoundReferenceRuntime): + """Protocol implementation. Host document open/accept and SQL replay remain separate gates.""" + + def install(self, context, assembly: SecureDocumentAssembly) -> None: + if context is None or assembly is None or not callable(getattr(assembly, "principal", None)) or \ + not callable(getattr(assembly, "snapshot", None)) or \ + context.get_resource("secure_document_contributor") is not None or \ + context.get_resource("teaql.runtime.capability:secure_document.v1") is not None: + raise RuntimeError("DOCUMENT_CONFIGURATION") + contributor = _SecureContributor(self, context, assembly) + install_runtime_capability(context, _SecureCapability(contributor)) + context.insert_resource("secure_document_contributor", contributor) + + def _seal(self, prefix: str, label: bytes, purpose: str, plaintext: bytes, + key: ReferenceKey) -> str: + nonce = self.nonce() + if not isinstance(nonce, bytes) or len(nonce) != 12: + _fail("ROUND_TRIP_REFERENCE_CONFIGURATION_INVALID") + aad = b"".join(_str(value) for value in (prefix, key.key_id, self.service, self.environment, purpose)) + encrypted = AESGCM(self._derive(key, label)).encrypt(nonce, plaintext, aad) + token = f"{prefix}.{key.key_id}." + base64.urlsafe_b64encode(nonce + encrypted).rstrip(b"=").decode() + if len(token) > 256000: + _fail() + return token + + def _open(self, prefix: str, label: bytes, purpose: str, token: str) -> tuple[ReferenceKey, bytes]: + if not isinstance(token, str) or len(token) > 256000: + _fail() + parts = token.split(".") + if len(parts) != 3 or parts[0] != prefix or not _KEY_ID.fullmatch(parts[1]) or \ + not _BASE64URL.fullmatch(parts[2]): + _fail() + key = self.keys.key_by_id(parts[1]) + if key is None: + _fail() + try: + envelope = base64.urlsafe_b64decode(parts[2] + "=" * (-len(parts[2]) % 4)) + except (ValueError, binascii.Error): + _fail() + if len(envelope) < 28 or base64.urlsafe_b64encode(envelope).rstrip(b"=").decode() != parts[2]: + _fail() + aad = b"".join(_str(value) for value in (prefix, key.key_id, self.service, self.environment, purpose)) + try: + return key, AESGCM(self._derive(key, label)).decrypt(envelope[:12], envelope[12:], aad) + except InvalidTag: + _fail() + + def _times(self, lifetime_seconds: int) -> tuple[int, int]: + if type(lifetime_seconds) is not int or lifetime_seconds <= 0: + _fail() + issued = self._timestamp() + expires = issued + lifetime_seconds + if expires > _MAX_I64: + _fail() + return issued, expires + + def _authorize_snapshot(self, principal: TrustedReferencePrincipal, + snapshot: DocumentSnapshot) -> None: + self._authorize(principal, snapshot.scope, snapshot.aggregate) + for entity in snapshot.entities: + self._authorize(principal, snapshot.scope, entity.identity) + + def _verify(self, principal: TrustedReferencePrincipal, key: ReferenceKey, actor: bytes, + issued: int, expires: int, scope: ReferenceDocumentScope, + aggregate_type: str, purpose: str, document: T) -> VerifiedDocument[T]: + now = self._timestamp() + if expires <= now: + _fail("ROUND_TRIP_REFERENCE_EXPIRED") + if issued > now + 60: + _fail() + if not hmac.compare_digest(actor, self._actor(key, principal)) or \ + scope.purpose != purpose or scope.aggregate_type != aggregate_type: + _fail("ROUND_TRIP_REFERENCE_SCOPE_MISMATCH") + try: + return VerifiedDocument(document, key.key_id, + datetime.fromtimestamp(issued, timezone.utc), + datetime.fromtimestamp(expires, timezone.utc)) + except (OverflowError, OSError, ValueError): + _fail() + + def issue_document_snapshot(self, principal: TrustedReferencePrincipal, + snapshot: DocumentSnapshot, lifetime_seconds: int) -> str: + issued, expires = self._times(lifetime_seconds) + key = self.keys.current_key() + plaintext = _encode_snapshot(snapshot, 1, self._actor(key, principal), issued, expires) + self._authorize_snapshot(principal, snapshot) + return self._seal("tqd1", b"document-snapshot-encryption", snapshot.scope.purpose, plaintext, key) + + def consume_document_snapshot(self, principal: TrustedReferencePrincipal, token: str, + aggregate_type: str, purpose: str) -> VerifiedDocument[DocumentSnapshot]: + key, plaintext = self._open("tqd1", b"document-snapshot-encryption", purpose, token) + reader = _Reader(plaintext) + actor, issued, expires, snapshot = _decode_snapshot(reader, 1) + reader.end() + verified = self._verify(principal, key, actor, issued, expires, snapshot.scope, + aggregate_type, purpose, snapshot) + self._authorize_snapshot(principal, snapshot) + return verified + + def issue_bound_document_snapshot(self, principal: TrustedReferencePrincipal, + document: BoundDocumentSnapshot, lifetime_seconds: int) -> str: + issued, expires = self._times(lifetime_seconds) + key = self.keys.current_key() + bindings = _bindings(document) + parts = [_encode_snapshot(document.snapshot, 2, self._actor(key, principal), issued, expires), + _count(len(bindings), 10000)] + for binding in bindings: + parts.extend((_str(binding.row_key), _str(binding.canonical_field), _str(binding.entity_type), + _flag(binding.stable_key is not None))) + if binding.stable_key is not None: + parts.append(_str(binding.stable_key)) + parts.extend((_u64(binding.binding_version), _flag(binding.nullable), _flag(binding.writable))) + self._authorize_snapshot(principal, document.snapshot) + return self._seal("tqd2", b"document-snapshot-v2-encryption", document.snapshot.scope.purpose, + b"".join(parts), key) + + def consume_bound_document_snapshot(self, principal: TrustedReferencePrincipal, token: str, + aggregate_type: str, purpose: str) -> VerifiedDocument[BoundDocumentSnapshot]: + key, plaintext = self._open("tqd2", b"document-snapshot-v2-encryption", purpose, token) + reader = _Reader(plaintext) + actor, issued, expires, snapshot = _decode_snapshot(reader, 2) + bindings = [] + for _ in range(reader.count(10000)): + row_key, field, entity_type = reader.text(), reader.text(), reader.text() + stable_key = reader.text() if reader.flag() else None + bindings.append(DocumentBlobBindingSnapshot(row_key, field, entity_type, stable_key, + reader.u64(), reader.flag(), reader.flag())) + reader.end() + document = BoundDocumentSnapshot(snapshot, tuple(bindings)) + if document.bindings != _bindings(document): + _fail() + verified = self._verify(principal, key, actor, issued, expires, snapshot.scope, + aggregate_type, purpose, document) + self._authorize_snapshot(principal, snapshot) + return verified + + def issue_blob_selection(self, principal: TrustedReferencePrincipal, document_token: str, + claim: BlobSelectionClaim, lifetime_seconds: int) -> str: + document = self.consume_bound_document_snapshot(principal, document_token, + claim.target.scope.aggregate_type, + claim.target.scope.purpose) + _target_allowed(document.document, claim.target) + _blob_key(claim.stable_key) + issued, expires = self._times(lifetime_seconds) + if expires > int(document.expires_at.timestamp()): + _fail() + key = self.keys.current_key() + payload = b"".join((b"\x01", self._actor(key, principal), _i64(issued), _i64(expires), + _scope(claim.target.scope), _str(claim.target.target_entity), + _str(claim.target.canonical_field), + bytes((1 if claim.target.kind == "EXISTING_ROW" else 2,)), + _str(claim.target.target_key), _str(claim.stable_key), + _str(claim.target.command_id))) + return self._seal("tqs1", b"blob-selection-v1-encryption", claim.target.scope.purpose, + payload, key) + + def consume_blob_selection(self, principal: TrustedReferencePrincipal, document_token: str, + token: str, expected_target: BlobSelectionTarget) -> VerifiedDocument[BlobSelectionClaim]: + document = self.consume_bound_document_snapshot(principal, document_token, + expected_target.scope.aggregate_type, + expected_target.scope.purpose) + _target_allowed(document.document, expected_target) + key, plaintext = self._open("tqs1", b"blob-selection-v1-encryption", + expected_target.scope.purpose, token) + reader = _Reader(plaintext) + if reader.byte() != 1: + _fail() + actor, issued, expires = reader.take(32), reader.i64(), reader.i64() + scope, target_entity, field = _read_scope(reader), reader.text(), reader.text() + kind_byte = reader.byte() + if kind_byte not in (1, 2): + _fail() + target_key, stable_key, command_id = reader.text(), _blob_key(reader.text()), reader.text() + reader.end() + target = BlobSelectionTarget(scope, target_entity, field, + "EXISTING_ROW" if kind_byte == 1 else "NEW_CLIENT_KEY", + target_key, command_id) + if target != expected_target: + _fail("ROUND_TRIP_REFERENCE_SCOPE_MISMATCH") + if expires > int(document.expires_at.timestamp()): + _fail() + return self._verify(principal, key, actor, issued, expires, scope, + expected_target.scope.aggregate_type, expected_target.scope.purpose, + BlobSelectionClaim(target, stable_key)) diff --git a/packages/teaql-secure-document-reference/src/teaql_secure_document_reference/round_trip_reference.py b/packages/teaql-secure-document-reference/src/teaql_secure_document_reference/round_trip_reference.py new file mode 100644 index 00000000..247e7c4a --- /dev/null +++ b/packages/teaql-secure-document-reference/src/teaql_secure_document_reference/round_trip_reference.py @@ -0,0 +1,308 @@ +"""Governed tqr1 entity references compatible with the Java/Rust wire contract.""" + +from __future__ import annotations + +import base64 +import binascii +import hmac +import math +import os +import re +import struct +from dataclasses import dataclass +from datetime import datetime, timezone +from hashlib import sha256 +from typing import Callable + +from cryptography.exceptions import InvalidTag +from cryptography.hazmat.primitives import hashes +from cryptography.hazmat.primitives.ciphers.aead import AESGCM +from cryptography.hazmat.primitives.kdf.hkdf import HKDF + + +_SALT = b"teaql.round-trip-reference.v1" +_KEY_ID = re.compile(r"[A-Za-z0-9_-]{1,64}\Z", re.ASCII) +_BASE64URL = re.compile(r"[A-Za-z0-9_-]+\Z", re.ASCII) +_MAX_U64 = (1 << 64) - 1 +_MAX_I64 = (1 << 63) - 1 + + +class RoundTripReferenceError(ValueError): + def __init__(self, code: str) -> None: + self.code = code + super().__init__(code) + + +def _fail(code: str = "ROUND_TRIP_REFERENCE_INVALID"): + raise RoundTripReferenceError(code) + + +def _text(value: str) -> bytes: + if not isinstance(value, str) or not value or value != value.strip(): + _fail() + try: + encoded = value.encode("utf-8", errors="strict") + except UnicodeError: + _fail() + if len(encoded) > 4096: + _fail() + return encoded + + +def _sized(value: bytes) -> bytes: + if len(value) > 65535: + _fail() + return struct.pack(">H", len(value)) + value + + +def _string(value: str) -> bytes: + return _sized(_text(value)) + + +def _u64(value: int, positive: bool = False) -> bytes: + if type(value) is not int or value < (1 if positive else 0) or value > _MAX_U64: + _fail() + return struct.pack(">Q", value) + + +def _i64(value: int, nonnegative: bool = False) -> bytes: + if type(value) is not int or value < (0 if nonnegative else -1 << 63) or value > _MAX_I64: + _fail() + return struct.pack(">q", value) + + +@dataclass(frozen=True) +class TrustedReferencePrincipal: + authentication_realm: str + subject: str + domain_root_type: str + domain_root_id: int + + def wire(self) -> bytes: + return b"".join((_string(self.authentication_realm), _string(self.subject), + _string(self.domain_root_type), _u64(self.domain_root_id, True))) + + +@dataclass(frozen=True) +class ReferenceDocumentScope: + document_id: str + purpose: str + aggregate_type: str + aggregate_id: int + aggregate_revision: int + + def validate(self) -> None: + _text(self.document_id) + _text(self.purpose) + _text(self.aggregate_type) + _u64(self.aggregate_id, True) + _i64(self.aggregate_revision, True) + + +@dataclass(frozen=True) +class ReferenceIdentity: + entity_type: str + id: int + version: int + + def validate(self) -> None: + _text(self.entity_type) + _u64(self.id, True) + _i64(self.version, True) + + +@dataclass(frozen=True) +class ReferenceKey: + key_id: str + master_key: bytes + + def __post_init__(self) -> None: + if not isinstance(self.key_id, str) or not _KEY_ID.fullmatch(self.key_id) or \ + not isinstance(self.master_key, bytes) or len(self.master_key) != 32: + _fail("ROUND_TRIP_REFERENCE_CONFIGURATION_INVALID") + + def __repr__(self) -> str: + return f"ReferenceKey(key_id={self.key_id!r}, master_key=[REDACTED])" + + +class StaticReferenceKeyProvider: + def __init__(self, current_key_id: str, keys: tuple[ReferenceKey, ...]) -> None: + if not isinstance(current_key_id, str) or not _KEY_ID.fullmatch(current_key_id): + _fail("ROUND_TRIP_REFERENCE_CONFIGURATION_INVALID") + retained: dict[str, ReferenceKey] = {} + for key in keys: + if not isinstance(key, ReferenceKey) or key.key_id in retained: + _fail("ROUND_TRIP_REFERENCE_CONFIGURATION_INVALID") + retained[key.key_id] = key + if current_key_id not in retained: + _fail("ROUND_TRIP_REFERENCE_CONFIGURATION_INVALID") + self._keys = retained + self._current = current_key_id + + def current_key(self) -> ReferenceKey: + return self._keys[self._current] + + def key_by_id(self, key_id: str) -> ReferenceKey | None: + return self._keys.get(key_id) + + +@dataclass(frozen=True) +class VerifiedEntityReference: + identity: ReferenceIdentity + key_id: str + issued_at: datetime + expires_at: datetime + + +class _Reader: + def __init__(self, data: bytes) -> None: + self.data = data + self.offset = 0 + + def take(self, count: int) -> bytes: + if count < 0 or self.offset + count > len(self.data): + _fail() + result = self.data[self.offset:self.offset + count] + self.offset += count + return result + + def text(self) -> str: + length = struct.unpack(">H", self.take(2))[0] + try: + result = self.take(length).decode("utf-8", errors="strict") + except UnicodeError: + _fail() + _text(result) + return result + + def u64(self) -> int: + return struct.unpack(">Q", self.take(8))[0] + + def i64(self) -> int: + return struct.unpack(">q", self.take(8))[0] + + def end(self) -> None: + if self.offset != len(self.data): + _fail() + + +class ContextBoundReferenceRuntime: + def __init__(self, service: str, environment: str, keys: StaticReferenceKeyProvider, + authorize: Callable[[TrustedReferencePrincipal, ReferenceDocumentScope, + ReferenceIdentity], None], + now: Callable[[], datetime] | None = None, + nonce: Callable[[], bytes] | None = None) -> None: + _text(service) + _text(environment) + if not keys or not callable(authorize): + _fail("ROUND_TRIP_REFERENCE_CONFIGURATION_INVALID") + keys.current_key() + self.service, self.environment = service, environment + self.keys, self.authorize = keys, authorize + self.now = now or (lambda: datetime.now(timezone.utc)) + self.nonce = nonce or (lambda: os.urandom(12)) + + def _timestamp(self) -> int: + value = self.now() + if not isinstance(value, datetime) or value.tzinfo is None: + _fail("ROUND_TRIP_REFERENCE_CONFIGURATION_INVALID") + return math.floor(value.timestamp()) + + def _derive(self, key: ReferenceKey, label: bytes) -> bytes: + info = b"".join((_sized(label), _string(self.service), _string(self.environment), + _string(key.key_id))) + return HKDF(algorithm=hashes.SHA256(), + length=32, salt=_SALT, info=info).derive(key.master_key) + + def _actor(self, key: ReferenceKey, principal: TrustedReferencePrincipal) -> bytes: + return hmac.new(self._derive(key, b"actor-fingerprint"), principal.wire(), sha256).digest() + + def _aad(self, key: ReferenceKey, purpose: str) -> bytes: + return b"".join(_string(value) for value in + ("tqr1", key.key_id, self.service, self.environment, purpose)) + + def _authorize(self, principal: TrustedReferencePrincipal, scope: ReferenceDocumentScope, + identity: ReferenceIdentity) -> None: + try: + self.authorize(principal, scope, identity) + except Exception: + _fail("ROUND_TRIP_REFERENCE_AUTHORIZATION_REQUIRED") + + def issue(self, principal: TrustedReferencePrincipal, scope: ReferenceDocumentScope, + identity: ReferenceIdentity, lifetime_seconds: int) -> str: + principal.wire() + scope.validate() + identity.validate() + if type(lifetime_seconds) is not int or lifetime_seconds <= 0: + _fail() + self._authorize(principal, scope, identity) + key = self.keys.current_key() + issued = self._timestamp() + expires = issued + lifetime_seconds + _i64(expires) + plaintext = b"".join((self._actor(key, principal), _string(identity.entity_type), + _u64(identity.id), _i64(identity.version), + _string(scope.document_id), _string(scope.aggregate_type), + _u64(scope.aggregate_id), _i64(scope.aggregate_revision), + _i64(issued), _i64(expires))) + nonce = self.nonce() + if not isinstance(nonce, bytes) or len(nonce) != 12: + _fail("ROUND_TRIP_REFERENCE_CONFIGURATION_INVALID") + encrypted = AESGCM(self._derive(key, b"entity-reference-encryption")).encrypt( + nonce, plaintext, self._aad(key, scope.purpose)) + body = base64.urlsafe_b64encode(nonce + encrypted).rstrip(b"=").decode("ascii") + token = f"tqr1.{key.key_id}.{body}" + if len(token) > 24000: + _fail() + return token + + def consume(self, principal: TrustedReferencePrincipal, scope: ReferenceDocumentScope, + token: str, expected_entity_type: str) -> VerifiedEntityReference: + principal.wire() + scope.validate() + _text(expected_entity_type) + if not isinstance(token, str) or len(token) > 24000: + _fail() + parts = token.split(".") + if len(parts) != 3 or parts[0] != "tqr1" or not _KEY_ID.fullmatch(parts[1]) or \ + not _BASE64URL.fullmatch(parts[2]): + _fail() + key = self.keys.key_by_id(parts[1]) + if key is None: + _fail() + try: + envelope = base64.urlsafe_b64decode(parts[2] + "=" * (-len(parts[2]) % 4)) + except (ValueError, binascii.Error): + _fail() + if len(envelope) < 28 or base64.urlsafe_b64encode(envelope).rstrip(b"=").decode() != parts[2]: + _fail() + try: + plaintext = AESGCM(self._derive(key, b"entity-reference-encryption")).decrypt( + envelope[:12], envelope[12:], self._aad(key, scope.purpose)) + except InvalidTag: + _fail() + reader = _Reader(plaintext) + actor = reader.take(32) + identity = ReferenceIdentity(reader.text(), reader.u64(), reader.i64()) + document_id, aggregate_type = reader.text(), reader.text() + aggregate_id, aggregate_revision = reader.u64(), reader.i64() + issued, expires = reader.i64(), reader.i64() + reader.end() + identity.validate() + now = self._timestamp() + if expires <= now: + _fail("ROUND_TRIP_REFERENCE_EXPIRED") + if issued > now + 60: + _fail() + if not hmac.compare_digest(actor, self._actor(key, principal)) or \ + document_id != scope.document_id or aggregate_type != scope.aggregate_type or \ + aggregate_id != scope.aggregate_id or aggregate_revision != scope.aggregate_revision or \ + identity.entity_type != expected_entity_type: + _fail("ROUND_TRIP_REFERENCE_SCOPE_MISMATCH") + self._authorize(principal, scope, identity) + try: + issued_at = datetime.fromtimestamp(issued, timezone.utc) + expires_at = datetime.fromtimestamp(expires, timezone.utc) + except (OverflowError, OSError, ValueError): + _fail() + return VerifiedEntityReference(identity, key.key_id, issued_at, expires_at) diff --git a/packages/teaql-secure-document-reference/src/teaql_secure_document_reference/sqlite_document_service.py b/packages/teaql-secure-document-reference/src/teaql_secure_document_reference/sqlite_document_service.py new file mode 100644 index 00000000..39c386c2 --- /dev/null +++ b/packages/teaql-secure-document-reference/src/teaql_secure_document_reference/sqlite_document_service.py @@ -0,0 +1,585 @@ +"""Host-backed secure documents and durable command receipts on a physical SQLite graph.""" + +from __future__ import annotations + +import hashlib +import json +import re +import unicodedata +from dataclasses import dataclass +from typing import Any, Protocol + +from teaql.core.value import Value +from teaql.runtime.graph_session import GraphCommittedError +from teaql.sql.types import CompiledQuery + +from .document_round_trip import ( + BlobSelectionTarget, BoundDocumentSnapshot, SecureDocumentReferenceModule, + _bindings, _encode_snapshot, +) +from .round_trip_reference import ReferenceDocumentScope, TrustedReferencePrincipal + +_MAX_BYTES = 256000 +_MAX_SAFE_INTEGER = (1 << 53) - 1 + + +def _invalid(code: str) -> None: + raise RuntimeError(code) + + +def _text(value: Any, maximum: int) -> bool: + return (isinstance(value, str) and bool(value) and value == value.strip() and + all(not unicodedata.category(ch).startswith("C") for ch in value) and + len(value.encode("utf-8")) <= maximum) + + +def _copy_body(value: Any) -> dict[str, Any]: + remaining = 100000 + + def check(item: Any, depth: int) -> None: + nonlocal remaining + remaining -= 1 + if remaining < 0 or depth > 64: + _invalid("DOCUMENT_COMMAND_INVALID") + if item is None or type(item) is bool: + return + if type(item) is str: + if len(item.encode("utf-8")) > _MAX_BYTES: + _invalid("DOCUMENT_COMMAND_INVALID") + return + if type(item) is int: + if abs(item) > _MAX_SAFE_INTEGER: + _invalid("DOCUMENT_COMMAND_INVALID") + return + if type(item) is list: + for child in item: + check(child, depth + 1) + return + if type(item) is dict: + for key, child in item.items(): + if not _text(key, 16384) or key in {"__proto__", "constructor", "prototype"}: + _invalid("DOCUMENT_COMMAND_INVALID") + check(child, depth + 1) + return + _invalid("DOCUMENT_COMMAND_INVALID") + + if type(value) is not dict: + _invalid("DOCUMENT_COMMAND_INVALID") + check(value, 0) + try: + encoded = json.dumps(value, sort_keys=True, ensure_ascii=False, + separators=(",", ":"), allow_nan=False).encode("utf-8") + except (UnicodeError, ValueError, TypeError, RecursionError): + _invalid("DOCUMENT_COMMAND_INVALID") + if len(encoded) > _MAX_BYTES: + _invalid("DOCUMENT_COMMAND_INVALID") + return json.loads(encoded) + + +def _hash_parts(*parts: Any) -> str: + encoded = json.dumps(parts, ensure_ascii=False, separators=(",", ":"), + sort_keys=True).encode("utf-8") + return hashlib.sha256(encoded).hexdigest() + + +@dataclass(frozen=True, repr=False) +class SubmittedDocumentCommand: + business_id: str + document_token: str + command_id: str + body: dict[str, Any] + blob_selections: tuple[DocumentBlobSelectionSubmission, ...] | None = None + + def __repr__(self) -> str: + return "SubmittedDocumentCommand[redacted]" + + +@dataclass(frozen=True, repr=False) +class DocumentBlobSelectionSubmission: + token: str + target_entity: str + canonical_field: str + kind: str + target_key: str + + def __repr__(self) -> str: + return "DocumentBlobSelectionSubmission[redacted]" + + +@dataclass(frozen=True, repr=False) +class DocumentCommandReceipt: + previous_revision: int + new_revision: int + response: str + + def __repr__(self) -> str: + return "DocumentCommandReceipt[redacted]" + + +def _prepared(command: SubmittedDocumentCommand) -> tuple[str, SubmittedDocumentCommand]: + if (not isinstance(command, SubmittedDocumentCommand) or + not _text(command.business_id, 16384) or + not _text(command.document_token, _MAX_BYTES) or + not _text(command.command_id, 256)): + _invalid("DOCUMENT_COMMAND_INVALID") + copied = _copy_body(command.body) + selections = command.blob_selections + if selections is not None: + if type(selections) is not tuple or len(selections) > 100: + _invalid("DOCUMENT_COMMAND_INVALID") + for selection in selections: + if (not isinstance(selection, DocumentBlobSelectionSubmission) or + not all(_text(value, _MAX_BYTES if index == 0 else 16384) + for index, value in enumerate((selection.token, + selection.target_entity, selection.canonical_field, + selection.target_key))) or + selection.kind not in {"EXISTING_ROW", "NEW_CLIENT_KEY"}): + _invalid("DOCUMENT_COMMAND_INVALID") + # Keep the original digest for receipts written without selections. + parts = [command.business_id, command.document_token, command.command_id, copied] + if selections is not None: + parts.append([{"canonicalField": selection.canonical_field, + "kind": selection.kind, + "targetEntity": selection.target_entity, + "targetKey": selection.target_key, + "token": selection.token} for selection in selections]) + encoded = json.dumps(parts, ensure_ascii=False, separators=(",", ":"), + sort_keys=True).encode("utf-8") + if len(encoded) > _MAX_BYTES: + _invalid("DOCUMENT_COMMAND_INVALID") + return hashlib.sha256(encoded).hexdigest(), SubmittedDocumentCommand( + command.business_id, command.document_token, command.command_id, copied, selections) + + +def _legacy_selection_hash(command: SubmittedDocumentCommand) -> str: + """Recognize receipts written before the shared selection-key encoding.""" + parts = [command.business_id, command.document_token, command.command_id, + command.body, [selection.__dict__ for selection in command.blob_selections or ()]] + encoded = json.dumps(parts, ensure_ascii=False, separators=(",", ":"), + sort_keys=True).encode("utf-8") + return hashlib.sha256(encoded).hexdigest() + + +def _receipt(value: DocumentCommandReceipt) -> DocumentCommandReceipt: + if (not isinstance(value, DocumentCommandReceipt) or + type(value.previous_revision) is not int or + type(value.new_revision) is not int or + value.previous_revision < 0 or value.new_revision < value.previous_revision or + not _text(value.response, _MAX_BYTES)): + _invalid("DOCUMENT_COMMAND_INVALID") + return value + + +def _scope_key(scope: ReferenceDocumentScope) -> str: + scope.validate() + return _hash_parts(scope.document_id, scope.purpose, scope.aggregate_type, + str(scope.aggregate_id)) + + +def _actor_key(principal: TrustedReferencePrincipal) -> str: + principal.wire() + return _hash_parts(principal.authentication_realm, principal.subject, + principal.domain_root_type, str(principal.domain_root_id)) + + +class SQLiteDocumentCommandJournal: + """Receipt table installation is explicit; writes borrow a live graph SQL lease.""" + + def __init__(self, transport) -> None: + if transport is None: + _invalid("DOCUMENT_CAPABILITY_MISSING") + self.transport = transport + + async def ensure_schema(self) -> None: + try: + await self.transport.execute_sql(CompiledQuery("""CREATE TABLE IF NOT EXISTS + teaql_document_command_receipt ( + scope_hash TEXT NOT NULL, actor_hash TEXT NOT NULL, command_id TEXT NOT NULL, + request_hash TEXT NOT NULL, previous_revision BIGINT NOT NULL, + new_revision BIGINT NOT NULL, response TEXT NOT NULL, + PRIMARY KEY (scope_hash, actor_hash, command_id))""", [])) + except Exception: + _invalid("DOCUMENT_CATALOG_FAILED") + + async def validate_schema(self) -> None: + try: + rows = await self.transport.fetch_all_sql(CompiledQuery( + "PRAGMA table_info(teaql_document_command_receipt)", [])) + except Exception: + _invalid("DOCUMENT_SCHEMA_MISSING") + names = ("scope_hash", "actor_hash", "command_id", "request_hash", + "previous_revision", "new_revision", "response") + if len(rows) != len(names): + _invalid("DOCUMENT_SCHEMA_MISSING") + for index, name in enumerate(names): + row = rows[index] + if (row.get("name") != name or row.get("notnull") != 1 or + row.get("pk") != (index + 1 if index < 3 else 0)): + _invalid("DOCUMENT_SCHEMA_MISSING") + + async def apply_in_graph(self, graph, scope: ReferenceDocumentScope, + command: SubmittedDocumentCommand, authority, mutate + ) -> tuple[DocumentCommandReceipt, bool]: + if graph is None or not callable(getattr(graph, "with_sql_transport", None)): + _invalid("DOCUMENT_TRANSACTION_REQUIRED") + scope_key = _scope_key(scope) + request_hash, frozen = _prepared(command) + + async def work(sql): + async def current_actor(): + try: + principal = await authority(graph.context, scope, sql) + return _actor_key(principal) + except Exception: + _invalid("DOCUMENT_AUTHORIZATION_REQUIRED") + + actor_key = await current_actor() + params = [Value.Text(scope_key), Value.Text(actor_key), Value.Text(command.command_id)] + try: + rows = await sql.fetch_all_sql(CompiledQuery("""SELECT request_hash, + previous_revision,new_revision,response FROM teaql_document_command_receipt + WHERE scope_hash=? AND actor_hash=? AND command_id=?""", params)) + except Exception: + _invalid("DOCUMENT_CATALOG_FAILED") + if len(rows) > 1: + _invalid("DOCUMENT_CATALOG_FAILED") + if rows: + row = rows[0] + if (row.get("request_hash") != request_hash and + (frozen.blob_selections is None or + row.get("request_hash") != _legacy_selection_hash(frozen))): + _invalid("DOCUMENT_REPLAY_CONFLICT") + try: + found = _receipt(DocumentCommandReceipt(row["previous_revision"], + row["new_revision"], row["response"])) + except (KeyError, RuntimeError): + _invalid("DOCUMENT_CATALOG_FAILED") + if await current_actor() != actor_key: + _invalid("DOCUMENT_AUTHORIZATION_REQUIRED") + return found, True + produced = _receipt(await mutate(sql, frozen)) + if _prepared(frozen)[0] != request_hash or _prepared(command)[0] != request_hash: + _invalid("DOCUMENT_COMMAND_INVALID") + if await current_actor() != actor_key: + _invalid("DOCUMENT_AUTHORIZATION_REQUIRED") + try: + affected, _ = await sql.execute_sql(CompiledQuery("""INSERT INTO + teaql_document_command_receipt + (scope_hash,actor_hash,command_id,request_hash,previous_revision,new_revision,response) + VALUES (?,?,?,?,?,?,?)""", [*params, Value.Text(request_hash), + Value.I64(produced.previous_revision), Value.I64(produced.new_revision), + Value.Text(produced.response)])) + except Exception: + _invalid("DOCUMENT_CATALOG_FAILED") + if affected != 1: + _invalid("DOCUMENT_CATALOG_FAILED") + return produced, False + + return await graph.with_sql_transport(work) + + +@dataclass(frozen=True, repr=False) +class ContextualDocument: + business_id: str + document_token: str + body: dict[str, Any] + + def __repr__(self) -> str: + return "ContextualDocument[redacted]" + + +@dataclass(frozen=True, repr=False) +class AcceptedDocument: + document: ContextualDocument + previous_revision: int + new_revision: int + replayed: bool + + def __repr__(self) -> str: + return "AcceptedDocument[redacted]" + + +@dataclass(frozen=True) +class CurrentBoundDocument: + document: BoundDocumentSnapshot + body: dict[str, Any] + lifetime_seconds: int + + +@dataclass(frozen=True, repr=False) +class VerifiedDocumentBlobSelection: + target: BlobSelectionTarget + stable_key: str + row_key: str + entity_id: str + expected_version: int + next_version: int + + def __repr__(self) -> str: + return "VerifiedDocumentBlobSelection[redacted]" + + +@dataclass(frozen=True, repr=False) +class DocumentBlobSelectionClaim: + target: BlobSelectionTarget + stable_key: str + + def __repr__(self) -> str: + return "DocumentBlobSelectionClaim[redacted]" + + +def _verified_selections(module: SecureDocumentReferenceModule, + principal: TrustedReferencePrincipal, issued: BoundDocumentSnapshot, + command: SubmittedDocumentCommand) -> tuple[DocumentBlobSelectionClaim, ...]: + result = [] + seen = set() + for selected in command.blob_selections or (): + target = BlobSelectionTarget(issued.snapshot.scope, selected.target_entity, + selected.canonical_field, selected.kind, + selected.target_key, command.command_id) + pair = (target.target_key, target.canonical_field) + if pair in seen: + _invalid("DOCUMENT_COMMAND_INVALID") + seen.add(pair) + claim = module.consume_blob_selection(principal, command.document_token, + selected.token, target).document + result.append(DocumentBlobSelectionClaim(target, claim.stable_key)) + return tuple(result) + + +def _mapped_existing(issued: BoundDocumentSnapshot, + selection: DocumentBlobSelectionClaim) -> VerifiedDocumentBlobSelection: + target = selection.target + row = next((item for item in issued.snapshot.entities + if item.row_key == target.target_key), None) + binding = next((item for item in issued.bindings + if item.row_key == target.target_key and + item.canonical_field == target.canonical_field), None) + if (row is None or binding is None or row.identity.entity_type != target.target_entity or + not binding.writable or binding.binding_version >= (1 << 63) - 1): + _invalid("DOCUMENT_PROJECTION_VIOLATION") + return VerifiedDocumentBlobSelection(target, selection.stable_key, target.target_key, + str(row.identity.id), binding.binding_version, + binding.binding_version + 1) + + +def _mapped_new(issued: BoundDocumentSnapshot, selection: DocumentBlobSelectionClaim, + resolved: tuple[str, str]) -> VerifiedDocumentBlobSelection: + if (type(resolved) is not tuple or len(resolved) != 2 or + not _text(resolved[0], 16384) or type(resolved[1]) is not str or + re.fullmatch(r"[1-9][0-9]*", resolved[1], re.ASCII) is None or + len(resolved[1]) > 20 or + int(resolved[1]) > (1 << 64) - 1 or + any(row.row_key == resolved[0] for row in issued.snapshot.entities)): + _invalid("DOCUMENT_PROJECTION_VIOLATION") + return VerifiedDocumentBlobSelection(selection.target, selection.stable_key, + resolved[0], resolved[1], 0, 1) + + +class SQLiteContextBoundDocumentHost(Protocol): + def principal(self, context) -> TrustedReferencePrincipal: ... + async def authorize(self, context, principal: TrustedReferencePrincipal, + scope: ReferenceDocumentScope, sql) -> None: ... + async def load(self, context, business_id: str, sql) -> CurrentBoundDocument: ... + async def apply(self, context, issued: BoundDocumentSnapshot, + submitted: SubmittedDocumentCommand, sql) -> None: ... + async def resolve_new_blob_selection(self, context, issued: BoundDocumentSnapshot, + submitted: SubmittedDocumentCommand, selection: DocumentBlobSelectionClaim, + sql) -> tuple[str, str]: ... + async def present(self, context, document: ContextualDocument) -> None: ... + + +def _valid_current(current: CurrentBoundDocument, business_id: str, + aggregate_type: str, purpose: str) -> None: + if (not isinstance(current, CurrentBoundDocument) or + current.document.snapshot.business_id != business_id or + current.document.snapshot.scope.aggregate_type != aggregate_type or + current.document.snapshot.scope.purpose != purpose or + type(current.lifetime_seconds) is not int or current.lifetime_seconds <= 0): + _invalid("DOCUMENT_PROJECTION_VIOLATION") + try: + _bound_fingerprint(current.document) + _copy_body(current.body) + except Exception: + _invalid("DOCUMENT_PROJECTION_VIOLATION") + + +def _bound_fingerprint(document: BoundDocumentSnapshot): + return (_encode_snapshot(document.snapshot, 2, bytes(32), 0, 1), + _bindings(document)) + + +class SQLiteContextBoundDocumentService: + def __init__(self, context, transport, module: SecureDocumentReferenceModule, + journal: SQLiteDocumentCommandJournal, host: SQLiteContextBoundDocumentHost): + if (context is None or transport is None or not callable(getattr(transport, "begin_sql", None)) + or not isinstance(module, SecureDocumentReferenceModule) or + not isinstance(journal, SQLiteDocumentCommandJournal) or host is None): + _invalid("DOCUMENT_CONFIGURATION") + self.context, self.transport, self.module = context, transport, module + self.journal, self.host = journal, host + + def _principal(self, context) -> TrustedReferencePrincipal: + if context is not self.context: + _invalid("DOCUMENT_CONFIGURATION") + try: + principal = self.host.principal(context) + principal.wire() + return principal + except Exception: + _invalid("DOCUMENT_AUTHORIZATION_REQUIRED") + + async def open(self, context, aggregate_type: str, business_id: str, + purpose: str, lifetime_seconds: int) -> ContextualDocument: + if (not all(_text(v, 16384) for v in (aggregate_type, business_id, purpose)) or + type(lifetime_seconds) is not int or lifetime_seconds <= 0): + _invalid("DOCUMENT_COMMAND_INVALID") + principal = self._principal(context) + tx = await self.transport.begin_sql() + committed = False + try: + current = await self.host.load(context, business_id, tx) + _valid_current(current, business_id, aggregate_type, purpose) + if self._principal(context) != principal: + _invalid("DOCUMENT_AUTHORIZATION_REQUIRED") + try: + await self.host.authorize(context, principal, current.document.snapshot.scope, tx) + except Exception: + _invalid("DOCUMENT_AUTHORIZATION_REQUIRED") + if self._principal(context) != principal: + _invalid("DOCUMENT_AUTHORIZATION_REQUIRED") + token = self.module.issue_bound_document_snapshot( + principal, current.document, min(lifetime_seconds, current.lifetime_seconds)) + result = ContextualDocument(business_id, token, _copy_body(current.body)) + await tx.commit_sql() + committed = True + finally: + if not committed: + try: + await tx.rollback_sql() + except Exception: + pass + await self.host.present(context, result) + return result + + async def accept(self, context, aggregate_type: str, purpose: str, + command: SubmittedDocumentCommand) -> AcceptedDocument: + if not _text(aggregate_type, 16384) or not _text(purpose, 16384): + _invalid("DOCUMENT_COMMAND_INVALID") + principal = self._principal(context) + _prepared(command) + issued = self.module.consume_bound_document_snapshot( + principal, command.document_token, aggregate_type, purpose).document + scope = issued.snapshot.scope + if issued.snapshot.business_id != command.business_id: + _invalid("DOCUMENT_PROJECTION_VIOLATION") + + async def graph_work(graph): + async def authority(_view, expected_scope, sql): + current = self._principal(context) + if current != principal or expected_scope != scope: + _invalid("DOCUMENT_AUTHORIZATION_REQUIRED") + await self.host.authorize(context, current, expected_scope, sql) + return current + + async def mutate(sql, frozen): + before = await self.host.load(context, frozen.business_id, sql) + _valid_current(before, frozen.business_id, aggregate_type, purpose) + if _bound_fingerprint(before.document) != _bound_fingerprint(issued): + _invalid("DOCUMENT_REVISION_CONFLICT") + selections = _verified_selections(self.module, principal, issued, frozen) + binder = getattr(self.host, "bind_blob_selection", None) + if selections and not callable(binder): + _invalid("DOCUMENT_CAPABILITY_MISSING") + resolver = getattr(self.host, "resolve_new_blob_selection", None) + if any(s.target.kind == "NEW_CLIENT_KEY" for s in selections) and not callable(resolver): + _invalid("DOCUMENT_CAPABILITY_MISSING") + await self.host.apply(context, issued, frozen, sql) + mapped = [] + mapped_targets = set() + for selection in selections: + if selection.target.kind == "EXISTING_ROW": + bound = _mapped_existing(issued, selection) + else: + bound = _mapped_new(issued, selection, + await resolver(context, issued, frozen, selection, sql)) + pair = (bound.row_key, bound.target.canonical_field) + if pair in mapped_targets: + _invalid("DOCUMENT_COMMAND_INVALID") + mapped_targets.add(pair) + mapped.append(bound) + for selection in mapped: + await binder(context, graph, selection, sql) + after = await self.host.load(context, frozen.business_id, sql) + _valid_current(after, frozen.business_id, aggregate_type, purpose) + next_scope = after.document.snapshot.scope + if (next_scope.document_id != scope.document_id or + next_scope.aggregate_id != scope.aggregate_id or + next_scope.aggregate_type != scope.aggregate_type or + next_scope.purpose != scope.purpose or + next_scope.aggregate_revision < scope.aggregate_revision): + _invalid("DOCUMENT_PROJECTION_VIOLATION") + for selection in mapped: + binding = next((item for item in after.document.bindings + if item.row_key == selection.row_key and + item.canonical_field == selection.target.canonical_field), None) + if (binding is None or binding.entity_type != selection.target.target_entity or + binding.stable_key != selection.stable_key or + binding.binding_version != selection.next_version): + _invalid("DOCUMENT_PROJECTION_VIOLATION") + if selection.target.kind == "NEW_CLIENT_KEY": + child = next((row for row in after.document.snapshot.entities + if row.row_key == selection.row_key), None) + if (child is None or child.identity.entity_type != selection.target.target_entity or + str(child.identity.id) != selection.entity_id or + selection.target.canonical_field not in child.writable_fields or + not any(relation.completeness == "COMPLETE" and relation.writable and + "CREATE_CHILD" in relation.allowed_mutations and + selection.row_key in relation.row_keys and + any(previous.name == relation.name and + previous.completeness == "COMPLETE" and previous.writable and + "CREATE_CHILD" in previous.allowed_mutations and + selection.row_key not in previous.row_keys + for previous in issued.snapshot.relations) + for relation in after.document.snapshot.relations)): + _invalid("DOCUMENT_PROJECTION_VIOLATION") + try: + await self.host.authorize(context, principal, next_scope, sql) + except Exception: + _invalid("DOCUMENT_AUTHORIZATION_REQUIRED") + token = self.module.issue_bound_document_snapshot( + principal, after.document, after.lifetime_seconds) + body = _copy_body(after.body) + response = json.dumps({"businessId": frozen.business_id, + "documentToken": token, "body": body}, + sort_keys=True, ensure_ascii=False, separators=(",", ":")) + return DocumentCommandReceipt(scope.aggregate_revision, + next_scope.aggregate_revision, response) + + return await self.journal.apply_in_graph(graph, scope, command, authority, mutate) + + receipt, replayed = await context.execute_graph_save( + graph_work, comment="accept protected document command") + try: + data = json.loads(receipt.response) + if (type(data) is not dict or set(data) != {"businessId", "documentToken", "body"} or + data["businessId"] != command.business_id or + not _text(data["documentToken"], _MAX_BYTES)): + _invalid("DOCUMENT_CATALOG_FAILED") + if self._principal(context) != principal: + _invalid("DOCUMENT_AUTHORIZATION_REQUIRED") + verified = self.module.consume_bound_document_snapshot( + principal, data["documentToken"], aggregate_type, purpose).document + if (verified.snapshot.business_id != command.business_id or + verified.snapshot.scope.document_id != scope.document_id or + verified.snapshot.scope.aggregate_id != scope.aggregate_id or + verified.snapshot.scope.aggregate_revision != receipt.new_revision or + receipt.previous_revision != scope.aggregate_revision): + _invalid("DOCUMENT_CATALOG_FAILED") + result = ContextualDocument(command.business_id, data["documentToken"], + _copy_body(data["body"])) + await self.host.present(context, result) + except Exception as error: + raise GraphCommittedError((error,)) from None + return AcceptedDocument(result, receipt.previous_revision, + receipt.new_revision, replayed) diff --git a/packages/teaql-secure-document-reference/tests/test_document_round_trip.py b/packages/teaql-secure-document-reference/tests/test_document_round_trip.py new file mode 100644 index 00000000..d339a1cf --- /dev/null +++ b/packages/teaql-secure-document-reference/tests/test_document_round_trip.py @@ -0,0 +1,245 @@ +from dataclasses import replace +from datetime import datetime, timezone +from hashlib import sha256 +from datetime import timedelta + +import pytest + +from teaql_secure_document_reference import ( + BlobSelectionClaim, BlobSelectionTarget, BoundDocumentSnapshot, + DocumentBlobBindingSnapshot, DocumentEntitySnapshot, DocumentRelationSnapshot, + DocumentSnapshot, ReferenceDocumentScope, ReferenceIdentity, ReferenceKey, + RoundTripReferenceError, SecureDocumentReferenceModule, StaticReferenceKeyProvider, + TrustedReferencePrincipal, PreparedBoundDocument, +) +from teaql.core.blob_ref import BlobRef +from teaql.runtime.context import UserContext +from teaql.runtime.capability import validate_capabilities +from teaql.runtime.view_preparation import ( + AuthorizedBlobSlot, AuthorizedResponseGraph, ProjectedBlobURL, + ViewOptions, ViewPreparationError, ViewPreparationService, +) + + +NOW = datetime.fromtimestamp(1800000000, timezone.utc) +ALICE = TrustedReferencePrincipal("oidc", "alice", "Merchant", 7) +SCOPE = ReferenceDocumentScope("doc-1", "edit-order", "Order", 10, 4) +SNAPSHOT = DocumentSnapshot("model-v1", "ORD-X1", SCOPE, + ReferenceIdentity("Order", 10, 4), + (DocumentEntitySnapshot("item:0", ReferenceIdentity("OrderItem", 20, 2), + frozenset(("attachment", "product_code", "quantity")), + frozenset(("attachment", "quantity")), + frozenset(("UPDATE",))),), + (DocumentRelationSnapshot("items", ("item:0",), "COMPLETE", True, + frozenset(("CREATE_CHILD", "REMOVE_CHILD"))),)) +DOCUMENT = BoundDocumentSnapshot(SNAPSHOT, + (DocumentBlobBindingSnapshot("item:0", "attachment", "OrderItem", + "blob_A", (1 << 64) - 1, True, True),)) +TARGET = BlobSelectionTarget(SCOPE, "OrderItem", "attachment", "EXISTING_ROW", "item:0", "command-1") +CLAIM = BlobSelectionClaim(TARGET, "blob_B") +KEYS = StaticReferenceKeyProvider("k1", (ReferenceKey("k1", bytes([0x42]) * 32),)) + + +def runtime(*, now=NOW, allowed=True, environment="test"): + def authorize(*_args): + if not allowed: + raise PermissionError("revoked") + return SecureDocumentReferenceModule("order-service", environment, KEYS, authorize, + lambda: now, lambda: bytes([0x33]) * 12) + + +def code(call): + with pytest.raises(RoundTripReferenceError) as error: + call() + return error.value.code + + +def test_shared_tqd2_tqs1_vectors_and_current_authorization(): + module = runtime() + token = module.issue_bound_document_snapshot(ALICE, DOCUMENT, 300) + assert sha256(token.encode()).hexdigest() == \ + "e7c81ca089b0a3e0585faefcc275aeb44b456d666d9a4a17949c92b1891089d7" + verified = module.consume_bound_document_snapshot(ALICE, token, "Order", "edit-order") + assert verified.document == DOCUMENT + selection = module.issue_blob_selection(ALICE, token, CLAIM, 120) + assert sha256(selection.encode()).hexdigest() == \ + "31303e3e251dd0e8963bf2a4d8897d1eccde63a485aaf287f88f29d40757ba96" + assert module.consume_blob_selection(ALICE, token, selection, TARGET).document == CLAIM + assert code(lambda: runtime(allowed=False).consume_bound_document_snapshot( + ALICE, token, "Order", "edit-order")) == "ROUND_TRIP_REFERENCE_AUTHORIZATION_REQUIRED" + + +def test_legacy_tqd1_rust_digest_and_v2_separation(): + legacy_entity = replace(SNAPSHOT.entities[0], + loaded_fields=frozenset(("product_code", "quantity")), + writable_fields=frozenset(("quantity",))) + legacy = replace(SNAPSHOT, entities=(legacy_entity,)) + token = runtime().issue_document_snapshot(ALICE, legacy, 600) + assert sha256(token.encode()).hexdigest() == \ + "69fee1ec01cd4a5b69dd3f795aeaffb3a3546e50d48e25cfc9b8f169a527b267" + assert runtime().consume_document_snapshot(ALICE, token, "Order", "edit-order").document == legacy + assert code(lambda: runtime().consume_bound_document_snapshot(ALICE, token, "Order", "edit-order")) == \ + "ROUND_TRIP_REFERENCE_INVALID" + + +def test_tamper_wrong_actor_context_target_and_expiry(): + token = runtime().issue_bound_document_snapshot(ALICE, DOCUMENT, 300) + selection = runtime().issue_blob_selection(ALICE, token, CLAIM, 120) + assert code(lambda: runtime().consume_bound_document_snapshot( + replace(ALICE, subject="bob"), token, "Order", "edit-order")) == \ + "ROUND_TRIP_REFERENCE_SCOPE_MISMATCH" + assert code(lambda: runtime(environment="other").consume_bound_document_snapshot( + ALICE, token, "Order", "edit-order")) == "ROUND_TRIP_REFERENCE_INVALID" + assert code(lambda: runtime().consume_blob_selection( + ALICE, token, selection, replace(TARGET, command_id="other"))) == \ + "ROUND_TRIP_REFERENCE_SCOPE_MISMATCH" + assert code(lambda: runtime(now=datetime.fromtimestamp(1800000120, timezone.utc)).consume_blob_selection( + ALICE, token, selection, TARGET)) == "ROUND_TRIP_REFERENCE_EXPIRED" + assert code(lambda: runtime().consume_blob_selection(ALICE, token, selection + "A", TARGET)) == \ + "ROUND_TRIP_REFERENCE_INVALID" + assert code(lambda: runtime().consume_blob_selection(ALICE, token, "blob_B", TARGET)) == \ + "ROUND_TRIP_REFERENCE_INVALID" + assert code(lambda: runtime().issue_blob_selection(ALICE, token, + replace(CLAIM, stable_key="https://example.test/file"), 120)) == \ + "ROUND_TRIP_REFERENCE_INVALID" + + +@pytest.mark.asyncio +async def test_passive_install_prepares_governed_view_and_checks_blob_binding(): + now = datetime.now(timezone.utc) + module = runtime(now=now) + bound = replace(DOCUMENT, bindings=(replace(DOCUMENT.bindings[0], binding_version=4),)) + reference = BlobRef("blob_A") + graph = AuthorizedResponseGraph({"attachment": reference}, + (AuthorizedBlobSlot(("attachment",), "OrderItem", "20", + "attachment", "tenant-a", 4, reference),)) + + class Authorizer: + async def authorize(self, _context, _result, _options): + return graph + + class Projector: + async def project_batch(self, _context, _graph, _expiry): + return (ProjectedBlobURL("https://files.example.invalid/private/asset", + datetime.now(timezone.utc) + timedelta(seconds=30)),) + + class Assembly: + def __init__(self, document): + self.document = document + + def principal(self, _context): + return ALICE + + async def snapshot(self, _context, _graph): + return PreparedBoundDocument(self.document, 300) + + context = UserContext().insert_resource("response_graph_authorizer", Authorizer()) + context.insert_resource("blob_response_contributor", Projector()) + module.install(context, Assembly(bound)) + context._required_capabilities = frozenset(("secure_document.v1",)) + await validate_capabilities(context) + with pytest.raises(RuntimeError, match="DOCUMENT_CONFIGURATION"): + module.install(context, Assembly(bound)) + view = await ViewPreparationService().prepare( + context, None, ViewOptions("governed", "committed")) + assert view.presentation == {"attachment": "https://files.example.invalid/private/asset"} + assert view.document["documentToken"].startswith("tqd2.k1.") + assert view.expires_at <= datetime.now(timezone.utc) + timedelta(seconds=30) + context.insert_resource("teaql.runtime.capability:secure_document.v1", None) + with pytest.raises(ViewPreparationError) as missing: + await ViewPreparationService().prepare(context, None, ViewOptions("governed", "committed")) + assert missing.value.code == "DOCUMENT_CONFIGURATION" + + wrong = UserContext().insert_resource("response_graph_authorizer", Authorizer()) + module.install(wrong, Assembly(replace(bound, bindings=(replace(bound.bindings[0], + stable_key="blob_other"),)))) + with pytest.raises(ViewPreparationError) as error: + await ViewPreparationService().prepare(wrong, None, ViewOptions("governed", "committed")) + assert (error.value.code, error.value.business_state, error.value.retry) == \ + ("DOCUMENT_PROJECTION_VIOLATION", "committed", "read") + + +@pytest.mark.parametrize("scenario", ["writable-unloaded", "duplicate-row", "orphan-relation", + "duplicate-relation-row", "readonly-relation-mutation", "binding-unloaded", + "binding-readonly", "binding-foreign-entity", "duplicate-binding"]) +def test_rejects_inconsistent_issued_projection(scenario): + row, relation, binding = SNAPSHOT.entities[0], SNAPSHOT.relations[0], DOCUMENT.bindings[0] + if scenario == "writable-unloaded": row = replace(row, writable_fields=frozenset({"hidden"})) + if scenario == "orphan-relation": relation = replace(relation, row_keys=("absent",)) + if scenario == "duplicate-relation-row": relation = replace(relation, row_keys=("item:0", "item:0")) + if scenario == "readonly-relation-mutation": relation = replace(relation, writable=False) + if scenario == "binding-unloaded": + row = replace(row, loaded_fields=frozenset({"quantity"}), writable_fields=frozenset({"quantity"})) + binding = replace(binding, writable=False) + if scenario == "binding-readonly": row = replace(row, writable_fields=frozenset({"quantity"})) + if scenario == "binding-foreign-entity": binding = replace(binding, entity_type="OtherItem") + snapshot = replace(SNAPSHOT, entities=(row, row) if scenario == "duplicate-row" else (row,), + relations=(relation,)) + document = replace(DOCUMENT, snapshot=snapshot, + bindings=(binding, binding) if scenario == "duplicate-binding" else (binding,)) + assert code(lambda: runtime().issue_bound_document_snapshot(ALICE, document, 60)) == \ + "DOCUMENT_PROJECTION_VIOLATION" + + +@pytest.mark.parametrize("scenario", ["readonly-existing", "hidden-existing", "partial-child", + "readonly-child", "no-create-child"]) +def test_selection_cannot_expand_the_issued_surface(scenario): + document, target = DOCUMENT, TARGET + if scenario == "readonly-existing": + row = replace(SNAPSHOT.entities[0], writable_fields=frozenset({"quantity"})) + document = replace(DOCUMENT, snapshot=replace(SNAPSHOT, entities=(row,)), + bindings=(replace(DOCUMENT.bindings[0], writable=False),)) + elif scenario == "hidden-existing": target = replace(TARGET, canonical_field="hidden_attachment") + else: + target = replace(TARGET, kind="NEW_CLIENT_KEY", target_key="client-hidden") + relation = SNAPSHOT.relations[0] + if scenario == "partial-child": relation = replace(relation, completeness="PARTIAL") + if scenario == "readonly-child": relation = replace(relation, writable=False, allowed_mutations=frozenset()) + if scenario == "no-create-child": relation = replace(relation, allowed_mutations=frozenset({"REMOVE_CHILD"})) + document = replace(DOCUMENT, snapshot=replace(SNAPSHOT, relations=(relation,))) + module = runtime() + token = module.issue_bound_document_snapshot(ALICE, document, 60) + assert module.consume_bound_document_snapshot(ALICE, token, "Order", "edit-order").document == document + assert code(lambda: module.issue_blob_selection(ALICE, token, replace(CLAIM, target=target), 30)) == \ + "DOCUMENT_PROJECTION_VIOLATION" + + +@pytest.mark.asyncio +async def test_governed_view_hides_unlisted_blob_and_bounds_url_by_document_deadline(): + now = datetime.now(timezone.utc).replace(microsecond=0) + deadline = now + timedelta(seconds=60) + visible, hidden = BlobRef("blob_A"), BlobRef("blob_hidden") + source = {"attachment": visible, "hidden_attachment": hidden} + graph = AuthorizedResponseGraph({"attachment": visible}, ( + AuthorizedBlobSlot(("attachment",), "OrderItem", "20", "attachment", "tenant-a", 4, visible),)) + projected = [] + late = False + + class Authorizer: + async def authorize(self, *_): return graph + class Assembly: + def principal(self, _): return ALICE + async def snapshot(self, *_): + return PreparedBoundDocument(replace(DOCUMENT, + bindings=(replace(DOCUMENT.bindings[0], binding_version=4),)), 60) + class Projector: + async def project_batch(self, _context, authorized, ceiling): + assert ceiling == deadline + projected.extend(slot.reference.key for slot in authorized.blob_slots) + return (ProjectedBlobURL("https://files.example.invalid/asset", + deadline + timedelta(milliseconds=1) if late else deadline - timedelta(seconds=5)),) + + context = UserContext().insert_resource("response_graph_authorizer", Authorizer()) + context.insert_resource("blob_response_contributor", Projector()) + runtime(now=now).install(context, Assembly()) + options = ViewOptions("governed", "committed", now + timedelta(seconds=120)) + view = await ViewPreparationService().prepare(context, source, options) + assert view.presentation == {"attachment": "https://files.example.invalid/asset"} + assert view.expires_at == deadline - timedelta(seconds=5) + assert projected == ["blob_A"] and source["hidden_attachment"] is hidden + late = True + with pytest.raises(ViewPreparationError) as failure: + await ViewPreparationService().prepare(context, source, options) + assert failure.value.code == "BLOB_SIGN_FAILED" + assert failure.value.business_state == "committed" and failure.value.retry == "read" + assert projected == ["blob_A", "blob_A"] diff --git a/packages/teaql-secure-document-reference/tests/test_round_trip_reference.py b/packages/teaql-secure-document-reference/tests/test_round_trip_reference.py new file mode 100644 index 00000000..19db94f6 --- /dev/null +++ b/packages/teaql-secure-document-reference/tests/test_round_trip_reference.py @@ -0,0 +1,76 @@ +from datetime import datetime, timezone + +import pytest + +from teaql_secure_document_reference import ( + ContextBoundReferenceRuntime, ReferenceDocumentScope, ReferenceIdentity, + ReferenceKey, RoundTripReferenceError, StaticReferenceKeyProvider, + TrustedReferencePrincipal, +) + + +VECTOR = ("tqr1.k2.MzMzMzMzMzMzMzMzmhCET15vBdSc1BbFJcrvMMnRP1wAyjunShYlFPnDZL7Gk5V1LuptWyrbiQ8LQSDHFUmN3o6vb-MEgXZuh3N-YkeO55CYLeVgiyBBYslzuSu-fvWC2KMZcnZ5D8BRmWpGYA2Ok5sWe1Fg1aIyadTjVoKB-H2vPW_0okj-") +ALICE = TrustedReferencePrincipal("oidc", "alice", "Platform", 7) +SCOPE = ReferenceDocumentScope("doc-100", "edit-order", "Order", 100, 9) +IDENTITY = ReferenceIdentity("OrderItem", 42, 3) +NOW = datetime(2026, 9, 27, tzinfo=timezone.utc) +KEYS = StaticReferenceKeyProvider("k2", (ReferenceKey("k1", bytes([0x11]) * 32), + ReferenceKey("k2", bytes([0x22]) * 32))) + + +def runtime(*, now=NOW, allowed=True, environment="test-a"): + def authorize(principal, _scope, identity): + if not allowed or principal.subject != "alice" or identity.entity_type != "OrderItem" or identity.id != 42: + raise PermissionError("denied") + return ContextBoundReferenceRuntime("order-service", environment, KEYS, authorize, + lambda: now, lambda: bytes([0x33]) * 12) + + +def code(call): + with pytest.raises(RoundTripReferenceError) as error: + call() + return error.value.code + + +def test_tqr1_matches_rust_fixed_wire_vector_and_reauthorizes(): + assert runtime().issue(ALICE, SCOPE, IDENTITY, 600) == VECTOR + resolved = runtime().consume(ALICE, SCOPE, VECTOR, "OrderItem") + assert resolved.identity == IDENTITY + assert resolved.key_id == "k2" + assert resolved.issued_at == NOW + assert resolved.expires_at == datetime(2026, 9, 27, 0, 10, tzinfo=timezone.utc) + assert code(lambda: runtime(allowed=False).consume(ALICE, SCOPE, VECTOR, "OrderItem")) == \ + "ROUND_TRIP_REFERENCE_AUTHORIZATION_REQUIRED" + + +def test_tqr1_rejects_wrong_identity_scope_environment_and_expiry(): + assert code(lambda: runtime().consume(TrustedReferencePrincipal("oidc", "bob", "Platform", 7), + SCOPE, VECTOR, "OrderItem")) == \ + "ROUND_TRIP_REFERENCE_SCOPE_MISMATCH" + assert code(lambda: runtime().consume(ALICE, + ReferenceDocumentScope("other", "edit-order", "Order", 100, 9), + VECTOR, "OrderItem")) == "ROUND_TRIP_REFERENCE_SCOPE_MISMATCH" + assert code(lambda: runtime().consume(ALICE, + ReferenceDocumentScope("doc-100", "view-order", "Order", 100, 9), + VECTOR, "OrderItem")) == "ROUND_TRIP_REFERENCE_INVALID" + assert code(lambda: runtime(environment="other").consume(ALICE, SCOPE, VECTOR, "OrderItem")) == \ + "ROUND_TRIP_REFERENCE_INVALID" + assert code(lambda: runtime().consume(ALICE, SCOPE, VECTOR, "Order")) == \ + "ROUND_TRIP_REFERENCE_SCOPE_MISMATCH" + assert code(lambda: runtime(now=datetime(2026, 9, 27, 0, 10, tzinfo=timezone.utc)).consume( + ALICE, SCOPE, VECTOR, "OrderItem")) == "ROUND_TRIP_REFERENCE_EXPIRED" + + +def test_tqr1_rejects_tamper_raw_id_noncanonical_and_bad_config(): + assert code(lambda: runtime().consume(ALICE, SCOPE, VECTOR[:-1] + "a", "OrderItem")) == \ + "ROUND_TRIP_REFERENCE_INVALID" + assert code(lambda: runtime().consume(ALICE, SCOPE, "42", "OrderItem")) == \ + "ROUND_TRIP_REFERENCE_INVALID" + assert code(lambda: runtime().consume(ALICE, SCOPE, VECTOR + "=", "OrderItem")) == \ + "ROUND_TRIP_REFERENCE_INVALID" + assert code(lambda: runtime().issue(ALICE, SCOPE, ReferenceIdentity("OrderItem", 0, 3), 600)) == \ + "ROUND_TRIP_REFERENCE_INVALID" + assert code(lambda: runtime().issue(ALICE, SCOPE, IDENTITY, 0)) == \ + "ROUND_TRIP_REFERENCE_INVALID" + assert code(lambda: ReferenceKey("bad.key", bytes(32))) == \ + "ROUND_TRIP_REFERENCE_CONFIGURATION_INVALID" diff --git a/packages/teaql-secure-document-reference/tests/test_sqlite_document_service.py b/packages/teaql-secure-document-reference/tests/test_sqlite_document_service.py new file mode 100644 index 00000000..0b843446 --- /dev/null +++ b/packages/teaql-secure-document-reference/tests/test_sqlite_document_service.py @@ -0,0 +1,583 @@ +"""A real SQLite application host exercises the optional secure-document service.""" + +from dataclasses import replace +from datetime import datetime, timezone +import hashlib +import json +import time + +import aiosqlite +import pytest + +from teaql.core.value import Value +from teaql.core.blob_ref import BlobRef +from teaql.provider.sqlite.dialect import SqliteDialect +from teaql.provider.sqlite.transport import SqliteTransport +from teaql.runtime import RuntimeModule +from teaql.runtime.graph_session import GraphCommittedError +from teaql.sql.executor import SqlDataServiceExecutor +from teaql.sql.types import CompiledQuery +from teaql.provider.sqlite import SimpleSchemaProvider +from teaql_secure_document_reference import ( + BlobSelectionClaim, BlobSelectionTarget, BoundDocumentSnapshot, + ContextualDocument, CurrentBoundDocument, DocumentBlobBindingSnapshot, + DocumentBlobSelectionSubmission, + DocumentEntitySnapshot, DocumentRelationSnapshot, DocumentSnapshot, ReferenceDocumentScope, + ReferenceIdentity, ReferenceKey, RoundTripReferenceError, + SecureDocumentReferenceModule, + SQLiteContextBoundDocumentService, SQLiteDocumentCommandJournal, + StaticReferenceKeyProvider, SubmittedDocumentCommand, TrustedReferencePrincipal, +) +from teaql_blob_reference import (BlobBindingChange, BlobBindingTarget, + BlobRetentionPolicy, SqliteBlobCatalog) + + +class OrderHost: + def __init__(self): + self.actor = TrustedReferencePrincipal("oidc", "alice", "Merchant", 7) + self.mutations = 0 + self.fail_present = False + self.switch_actor_on_apply = False + + def principal(self, _context): + return self.actor + + async def authorize(self, _context, principal, _scope, sql): + rows = await sql.fetch_all_sql(CompiledQuery( + "SELECT can_edit FROM document_permission WHERE subject=?", + [Value.Text(principal.subject)])) + if len(rows) != 1 or rows[0]["can_edit"] != 1: + raise PermissionError("revoked") + + async def load(self, _context, business_id, sql): + rows = await sql.fetch_all_sql(CompiledQuery( + "SELECT id,version,quantity FROM document_order WHERE business_id=?", + [Value.Text(business_id)])) + if len(rows) != 1: + raise LookupError("order missing") + row = rows[0] + identity = ReferenceIdentity("Order", row["id"], row["version"]) + scope = ReferenceDocumentScope("edit-10", "edit-order", "Order", + row["id"], row["version"]) + snapshot = DocumentSnapshot("model-v1", business_id, scope, identity, + (DocumentEntitySnapshot( + "order:10", identity, frozenset({"quantity"}), + frozenset({"quantity"}), frozenset({"UPDATE"})),), ()) + return CurrentBoundDocument(BoundDocumentSnapshot(snapshot, ()), + {"quantity": row["quantity"]}, 300) + + async def apply(self, _context, issued, submitted, sql): + self.mutations += 1 + if submitted.body != {"quantity": 3}: + raise ValueError("invalid quantity") + affected, _ = await sql.execute_sql(CompiledQuery( + "UPDATE document_order SET quantity=?,version=version+1 WHERE id=? AND version=?", + [Value.I64(3), Value.I64(10), + Value.I64(issued.snapshot.scope.aggregate_revision)])) + if affected != 1: + raise RuntimeError("stale order") + if self.switch_actor_on_apply: + self.actor = replace(self.actor, subject="bob") + + async def present(self, _context, document: ContextualDocument): + assert document.business_id == "ORD-10" + if self.fail_present: + raise RuntimeError("presentation failed") + + +async def fixture(tmp_path, transport_factory=SqliteTransport): + path = str(tmp_path / "secure-document.sqlite") + async with aiosqlite.connect(path) as db: + await db.execute("""CREATE TABLE document_order (id INTEGER PRIMARY KEY, + business_id TEXT NOT NULL UNIQUE,version INTEGER NOT NULL,quantity INTEGER NOT NULL)""") + await db.execute("CREATE TABLE document_permission(subject TEXT PRIMARY KEY,can_edit INTEGER NOT NULL)") + await db.execute("INSERT INTO document_order VALUES (10,'ORD-10',1,1)") + await db.execute("INSERT INTO document_permission VALUES ('alice',1)") + await db.commit() + transport = transport_factory(path) + context = RuntimeModule.new().into_context() + context.insert_resource("dataService", SqlDataServiceExecutor( + SqliteDialect(), transport, SimpleSchemaProvider())) + keys = StaticReferenceKeyProvider("k1", (ReferenceKey("k1", bytes([0x42]) * 32),)) + module = SecureDocumentReferenceModule( + "order-service", "test", keys, lambda *_: None, + lambda: datetime.fromtimestamp(1800000000, timezone.utc), + lambda: bytes([0x33]) * 12) + host = OrderHost() + journal = SQLiteDocumentCommandJournal(transport) + await journal.ensure_schema() + await journal.validate_schema() + service = SQLiteContextBoundDocumentService(context, transport, module, journal, host) + return path, context, host, service + + +async def row(path): + async with aiosqlite.connect(path) as db: + async with db.execute("SELECT version,quantity FROM document_order WHERE id=10") as cursor: + return await cursor.fetchone() + + +async def receipts(path): + async with aiosqlite.connect(path) as db: + async with db.execute("SELECT COUNT(*) FROM teaql_document_command_receipt") as cursor: + return (await cursor.fetchone())[0] + + +@pytest.mark.asyncio +async def test_document_commit_unknown_replays_durable_receipt_under_current_authority(tmp_path): + class LostReplyTransport(SqliteTransport): + lose_next = False + + async def begin_sql(self): + transaction = await super().begin_sql() + if not self.lose_next: + return transaction + self.lose_next = False + + class CommitThenFail: + def __getattr__(self, name): + return getattr(transaction, name) + + async def commit_sql(self): + await transaction.commit_sql() + raise OSError("document commit acknowledgement unavailable") + + return CommitThenFail() + + transports = [] + def make_transport(path): + transport = LostReplyTransport(path) + transports.append(transport) + return transport + + path, context, host, service = await fixture(tmp_path, make_transport) + async with aiosqlite.connect(path) as db: + await db.execute("ALTER TABLE document_order ADD COLUMN hidden_note TEXT NOT NULL DEFAULT 'server-only'") + await db.commit() + opened = await service.open(context, "Order", "ORD-10", "edit-order", 60) + command = SubmittedDocumentCommand("ORD-10", opened.document_token, + "command-commit-unknown", {"quantity": 3}) + transports[0].lose_next = True + with pytest.raises(OSError, match="commit acknowledgement unavailable"): + await service.accept(context, "Order", "edit-order", command) + assert await row(path) == (2, 3) and await receipts(path) == 1 + assert host.mutations == 1 + + async def durable_receipt(): + async with aiosqlite.connect(path) as db: + async with db.execute("SELECT * FROM teaql_document_command_receipt") as cursor: + return await cursor.fetchall() + + original = await durable_receipt() + async with aiosqlite.connect(path) as db: + await db.execute("UPDATE document_permission SET can_edit=0 WHERE subject='alice'") + await db.commit() + with pytest.raises(RuntimeError, match="DOCUMENT_AUTHORIZATION_REQUIRED"): + await service.accept(context, "Order", "edit-order", command) + async with aiosqlite.connect(path) as db: + await db.execute("UPDATE document_permission SET can_edit=1 WHERE subject='alice'") + await db.commit() + with pytest.raises(RuntimeError, match="DOCUMENT_REPLAY_CONFLICT"): + await service.accept(context, "Order", "edit-order", replace(command, body={"quantity": 4})) + recovered = await service.accept(context, "Order", "edit-order", command) + assert recovered.replayed and recovered.new_revision == 2 + assert host.mutations == 1 and await row(path) == (2, 3) + assert await durable_receipt() == original + async with aiosqlite.connect(path) as db: + async with db.execute("SELECT hidden_note FROM document_order WHERE id=10") as cursor: + assert await cursor.fetchone() == ("server-only",) + + +@pytest.mark.asyncio +async def test_sqlite_open_accept_replay_authority_and_rollback(tmp_path): + path, context, host, service = await fixture(tmp_path) + opened = await service.open(context, "Order", "ORD-10", "edit-order", 60) + assert opened.document_token.startswith("tqd2.") and opened.body == {"quantity": 1} + assert opened.document_token not in repr(opened) + command = SubmittedDocumentCommand("ORD-10", opened.document_token, + "command-1", {"quantity": 3}) + first = await service.accept(context, "Order", "edit-order", command) + assert (first.previous_revision, first.new_revision, first.replayed) == (1, 2, False) + assert first.document.body == {"quantity": 3} + assert first.document.document_token not in repr(first) + assert await row(path) == (2, 3) and await receipts(path) == 1 + + replay = await service.accept(context, "Order", "edit-order", command) + assert replay.replayed and replay.document.document_token == first.document.document_token + assert host.mutations == 1 + with pytest.raises(RuntimeError, match="DOCUMENT_REPLAY_CONFLICT"): + await service.accept(context, "Order", "edit-order", + replace(command, body={"quantity": 4})) + with pytest.raises(RuntimeError, match="DOCUMENT_REVISION_CONFLICT"): + await service.accept(context, "Order", "edit-order", + replace(command, command_id="command-stale")) + + async with aiosqlite.connect(path) as db: + await db.execute("UPDATE document_permission SET can_edit=0 WHERE subject='alice'") + await db.commit() + with pytest.raises(RuntimeError, match="DOCUMENT_AUTHORIZATION_REQUIRED"): + await service.accept(context, "Order", "edit-order", command) + with pytest.raises(RuntimeError, match="DOCUMENT_AUTHORIZATION_REQUIRED"): + await service.open(context, "Order", "ORD-10", "edit-order", 60) + async with aiosqlite.connect(path) as db: + await db.execute("UPDATE document_permission SET can_edit=1 WHERE subject='alice'") + await db.commit() + + latest = await service.open(context, "Order", "ORD-10", "edit-order", 60) + followup = SubmittedDocumentCommand("ORD-10", latest.document_token, + "command-2", {"quantity": 3}) + host.fail_present = True + with pytest.raises(GraphCommittedError): + await service.accept(context, "Order", "edit-order", followup) + host.fail_present = False + recovered = await service.accept(context, "Order", "edit-order", followup) + assert recovered.replayed and recovered.new_revision == 3 and host.mutations == 2 + + async with aiosqlite.connect(path) as db: + await db.execute("""CREATE TRIGGER reject_document_receipt BEFORE INSERT + ON teaql_document_command_receipt WHEN NEW.command_id='command-trigger' + BEGIN SELECT RAISE(ABORT, 'receipt rejected'); END""") + await db.commit() + latest = await service.open(context, "Order", "ORD-10", "edit-order", 60) + with pytest.raises(RuntimeError, match="DOCUMENT_CATALOG_FAILED"): + await service.accept(context, "Order", "edit-order", SubmittedDocumentCommand( + "ORD-10", latest.document_token, "command-trigger", {"quantity": 3})) + assert await row(path) == (3, 3) and await receipts(path) == 2 + + host.switch_actor_on_apply = True + with pytest.raises(RuntimeError, match="DOCUMENT_AUTHORIZATION_REQUIRED"): + await service.accept(context, "Order", "edit-order", SubmittedDocumentCommand( + "ORD-10", latest.document_token, "command-actor-drift", {"quantity": 3})) + host.actor = replace(host.actor, subject="alice") + assert await row(path) == (3, 3) and await receipts(path) == 2 + + +@pytest.mark.asyncio +async def test_sqlite_document_journal_rejects_missing_replay_primary_key(tmp_path): + path = str(tmp_path / "broken-receipt.sqlite") + transport = SqliteTransport(path) + await transport.execute_sql(CompiledQuery("""CREATE TABLE teaql_document_command_receipt ( + scope_hash TEXT NOT NULL,actor_hash TEXT NOT NULL,command_id TEXT NOT NULL, + request_hash TEXT NOT NULL,previous_revision BIGINT NOT NULL, + new_revision BIGINT NOT NULL,response TEXT NOT NULL)""", [])) + journal = SQLiteDocumentCommandJournal(transport) + await journal.ensure_schema() + with pytest.raises(RuntimeError, match="DOCUMENT_SCHEMA_MISSING"): + await journal.validate_schema() + + +class BlobOrderHost(OrderHost): + def __init__(self, catalog): + super().__init__() + self.catalog = catalog + + async def load(self, context, business_id, sql): + current = await super().load(context, business_id, sql) + rows = await sql.fetch_all_sql(CompiledQuery( + "SELECT blob_key FROM document_order WHERE business_id=?", [Value.Text(business_id)])) + bindings = await sql.fetch_all_sql(CompiledQuery( + "SELECT key,version FROM teaql_blob_binding WHERE scope=? AND entity=? " + "AND entity_id=? AND field=?", [Value.Text("merchant:7"), Value.Text("Order"), + Value.Text("10"), Value.Text("attachment")])) + assert len(rows) == 1 and len(bindings) <= 1 + binding = bindings[0] if bindings else {"key": None, "version": 0} + assert rows[0]["blob_key"] == binding["key"] + entity = current.document.snapshot.entities[0] + entity = replace(entity, loaded_fields=frozenset({"attachment", "quantity"}), + writable_fields=frozenset({"attachment", "quantity"})) + snapshot = replace(current.document.snapshot, entities=(entity,)) + bound = DocumentBlobBindingSnapshot("order:10", "attachment", "Order", + binding["key"], binding["version"], True, True) + return replace(current, document=BoundDocumentSnapshot(snapshot, (bound,))) + + async def bind_blob_selection(self, _context, graph, selection, sql): + assert (selection.target.target_entity, selection.target.target_key, + selection.target.canonical_field, selection.entity_id) == ( + "Order", "order:10", "attachment", "10") + affected, _ = await sql.execute_sql(CompiledQuery( + "UPDATE document_order SET blob_key=? WHERE id=?", + [Value.Text(selection.stable_key), Value.I64(10)])) + assert affected == 1 + + class Authorization: + def scope(self, _actor): + return "merchant:7" + + async def current_policy_on_lease(self, _actor, lease, _target): + assert lease is sql + return BlobRetentionPolicy("default", "server_stream", 86400, 604800, 3600) + + async def authorize(self, _actor, target, _previous, next_ref, _action): + return (target == BlobBindingTarget("Order", "10", "attachment") and + next_ref == BlobRef.from_sql(selection.stable_key)) + + await self.catalog.bind_in_graph(graph, graph.context, + BlobBindingChange(BlobBindingTarget("Order", "10", "attachment"), + BlobRef.from_sql(selection.stable_key), + selection.expected_version, selection.next_version, "Update"), + Authorization(), lease=sql) + + +class ChildBlobOrderHost(BlobOrderHost): + def __init__(self, catalog): + super().__init__(catalog) + self.wrong_resolve = False + + async def load(self, context, business_id, sql): + current = await super().load(context, business_id, sql) + children = await sql.fetch_all_sql(CompiledQuery( + "SELECT id,version,blob_key FROM document_order_line WHERE order_id=? ORDER BY id", + [Value.I64(10)])) + entities = list(current.document.snapshot.entities) + bindings = list(current.document.bindings) + row_keys = [] + for child in children: + row_key = f"line:{child['id']}" + row_keys.append(row_key) + actual = await sql.fetch_all_sql(CompiledQuery( + "SELECT key,version FROM teaql_blob_binding WHERE scope=? AND entity=? " + "AND entity_id=? AND field=?", [Value.Text("merchant:7"), Value.Text("OrderLine"), + Value.Text(str(child["id"])), Value.Text("attachment")])) + assert len(actual) <= 1 + binding = actual[0] if actual else {"key": None, "version": 0} + assert child["blob_key"] == binding["key"] + entities.append(DocumentEntitySnapshot(row_key, + ReferenceIdentity("OrderLine", child["id"], child["version"]), + frozenset({"attachment"}), frozenset({"attachment"}), + frozenset({"UPDATE"}))) + bindings.append(DocumentBlobBindingSnapshot(row_key, "attachment", "OrderLine", + binding["key"], binding["version"], True, True)) + relation = DocumentRelationSnapshot("lines", tuple(row_keys), "COMPLETE", True, + frozenset({"CREATE_CHILD"})) + snapshot = replace(current.document.snapshot, entities=tuple(entities), + relations=(relation,)) + return replace(current, document=BoundDocumentSnapshot(snapshot, tuple(bindings))) + + async def apply(self, context, issued, submitted, sql): + await super().apply(context, issued, submitted, sql) + for selection in submitted.blob_selections or (): + if selection.kind != "NEW_CLIENT_KEY": + continue + assert selection.target_entity == "OrderLine" and selection.canonical_field == "attachment" + assert selection.target_key in {"client-line-1", "client-line-2"} + child_id = 21 if selection.target_key == "client-line-1" else 22 + affected, _ = await sql.execute_sql(CompiledQuery( + "INSERT INTO document_order_line(id,order_id,client_key,version,blob_key) " + "VALUES (?,?,?,?,NULL)", [Value.I64(child_id), Value.I64(10), + Value.Text(selection.target_key), Value.I64(1)])) + assert affected == 1 + + async def resolve_new_blob_selection(self, _context, _issued, _submitted, selection, sql): + if self.wrong_resolve: + return "order:10", "10" + children = await sql.fetch_all_sql(CompiledQuery( + "SELECT id FROM document_order_line WHERE order_id=? AND client_key=?", + [Value.I64(10), Value.Text(selection.target.target_key)])) + assert len(children) == 1 + return f"line:{children[0]['id']}", str(children[0]["id"]) + + async def bind_blob_selection(self, context, graph, selection, sql): + if selection.target.kind != "NEW_CLIENT_KEY": + return await super().bind_blob_selection(context, graph, selection, sql) + assert selection.target.target_entity == "OrderLine" and selection.row_key == f"line:{selection.entity_id}" + affected, _ = await sql.execute_sql(CompiledQuery( + "UPDATE document_order_line SET blob_key=? WHERE id=?", + [Value.Text(selection.stable_key), Value.I64(int(selection.entity_id))])) + assert affected == 1 + + class Authorization: + def scope(self, _actor): + return "merchant:7" + + async def current_policy_on_lease(self, _actor, lease, _target): + assert lease is sql + return BlobRetentionPolicy("default", "server_stream", 86400, 604800, 3600) + + async def authorize(self, _actor, target, _previous, next_ref, action): + return (target == BlobBindingTarget("OrderLine", selection.entity_id, "attachment") + and next_ref == BlobRef.from_sql(selection.stable_key) and action == "Create") + + await self.catalog.bind_in_graph(graph, graph.context, + BlobBindingChange(BlobBindingTarget("OrderLine", selection.entity_id, "attachment"), + BlobRef.from_sql(selection.stable_key), + selection.expected_version, selection.next_version, "Create"), + Authorization(), lease=sql) + + +@pytest.mark.asyncio +async def test_sqlite_new_child_blob_selection_commits_and_receipt_failure_rolls_back(tmp_path): + path, context, _old_host, old_service = await fixture(tmp_path) + async with aiosqlite.connect(path) as db: + await db.execute("ALTER TABLE document_order ADD COLUMN blob_key TEXT") + await db.execute("""CREATE TABLE document_order_line ( + id INTEGER PRIMARY KEY, order_id INTEGER NOT NULL, client_key TEXT UNIQUE NOT NULL, + version INTEGER NOT NULL, blob_key TEXT)""") + await db.commit() + catalog = SqliteBlobCatalog(old_service.transport) + await catalog.ensure_schema() + await catalog.validate_schema() + async with aiosqlite.connect(path) as db: + for key in ("blob_C", "blob_D"): + await db.execute("""INSERT INTO teaql_blob_catalog + (key,scope,object_key,content_version,file_name,content_type,size_bytes,status, + completed_at_ms,policy_profile,upload_mode,unbound_retention_seconds,effective_unref_seconds) + VALUES (?,?,?,?,?,?,?,'uploaded_unbound',?,?,?,?,?)""", + (key, "merchant:7", "objects/" + key, "v1", key + ".pdf", "application/pdf", 3, + time.time_ns() // 1_000_000, "default", "server_stream", 86400, 604800)) + await db.commit() + host = ChildBlobOrderHost(catalog) + service = SQLiteContextBoundDocumentService(context, old_service.transport, + old_service.module, old_service.journal, host) + + async def selected(opened, command_id, client_key, blob_key): + issued = service.module.consume_bound_document_snapshot(host.actor, + opened.document_token, "Order", "edit-order").document + target = BlobSelectionTarget(issued.snapshot.scope, "OrderLine", "attachment", + "NEW_CLIENT_KEY", client_key, command_id) + token = service.module.issue_blob_selection(host.actor, opened.document_token, + BlobSelectionClaim(target, blob_key), 60) + return DocumentBlobSelectionSubmission(token, "OrderLine", "attachment", + "NEW_CLIENT_KEY", client_key) + + opened = await service.open(context, "Order", "ORD-10", "edit-order", 60) + command = SubmittedDocumentCommand("ORD-10", opened.document_token, "command-child-1", + {"quantity": 3}, (await selected(opened, "command-child-1", "client-line-1", "blob_C"),)) + host.wrong_resolve = True + with pytest.raises(RuntimeError, match="DOCUMENT_PROJECTION_VIOLATION"): + await service.accept(context, "Order", "edit-order", command) + assert await row(path) == (1, 1) and await receipts(path) == 0 + host.wrong_resolve = False + first = await service.accept(context, "Order", "edit-order", command) + assert (first.previous_revision, first.new_revision, first.replayed) == (1, 2, False) + assert (await service.accept(context, "Order", "edit-order", command)).replayed + async with aiosqlite.connect(path) as db: + async with db.execute("SELECT id,version,blob_key FROM document_order_line") as cursor: + assert await cursor.fetchone() == (21, 1, "blob_C") + async with db.execute("SELECT key,version FROM teaql_blob_binding WHERE entity='OrderLine'") as cursor: + assert await cursor.fetchone() == ("blob_C", 1) + async with db.execute("SELECT status FROM teaql_blob_catalog WHERE key='blob_C'") as cursor: + assert await cursor.fetchone() == ("available",) + assert await row(path) == (2, 3) and await receipts(path) == 1 + + opened = await service.open(context, "Order", "ORD-10", "edit-order", 60) + failed = SubmittedDocumentCommand("ORD-10", opened.document_token, "command-child-2", + {"quantity": 3}, (await selected(opened, "command-child-2", "client-line-2", "blob_D"),)) + async with aiosqlite.connect(path) as db: + await db.execute("""CREATE TRIGGER reject_child_receipt BEFORE INSERT + ON teaql_document_command_receipt WHEN NEW.command_id='command-child-2' + BEGIN SELECT RAISE(ABORT, 'receipt rejected'); END""") + await db.commit() + with pytest.raises(RuntimeError, match="DOCUMENT_CATALOG_FAILED"): + await service.accept(context, "Order", "edit-order", failed) + async with aiosqlite.connect(path) as db: + async with db.execute("SELECT COUNT(*) FROM document_order_line") as cursor: + assert await cursor.fetchone() == (1,) + async with db.execute("SELECT COUNT(*) FROM teaql_blob_binding WHERE entity_id='22'") as cursor: + assert await cursor.fetchone() == (0,) + async with db.execute("SELECT status FROM teaql_blob_catalog WHERE key='blob_D'") as cursor: + assert await cursor.fetchone() == ("uploaded_unbound",) + assert await row(path) == (2, 3) and await receipts(path) == 1 + + +@pytest.mark.asyncio +async def test_sqlite_document_blob_selection_shares_business_binding_and_receipt(tmp_path): + path, context, _old_host, old_service = await fixture(tmp_path) + async with aiosqlite.connect(path) as db: + await db.execute("ALTER TABLE document_order ADD COLUMN blob_key TEXT") + await db.commit() + catalog = SqliteBlobCatalog(old_service.transport) + await catalog.ensure_schema() + await catalog.validate_schema() + async with aiosqlite.connect(path) as db: + for key in ("blob_A", "blob_B"): + await db.execute("""INSERT INTO teaql_blob_catalog + (key,scope,object_key,content_version,file_name,content_type,size_bytes,status, + completed_at_ms,policy_profile,upload_mode,unbound_retention_seconds,effective_unref_seconds) + VALUES (?,?,?,?,?,?,?,'uploaded_unbound',?,?,?,?,?)""", + (key, "merchant:7", "objects/" + key, "v1", key + ".pdf", "application/pdf", 3, + time.time_ns() // 1_000_000, "default", "server_stream", 86400, 604800)) + await db.commit() + host = BlobOrderHost(catalog) + service = SQLiteContextBoundDocumentService(context, old_service.transport, + old_service.module, old_service.journal, host) + + async def open_document(): + return await service.open(context, "Order", "ORD-10", "edit-order", 60) + + def selection(opened, command_id, key): + issued = service.module.consume_bound_document_snapshot(host.actor, + opened.document_token, "Order", "edit-order").document + target = BlobSelectionTarget(issued.snapshot.scope, "Order", "attachment", + "EXISTING_ROW", "order:10", command_id) + token = service.module.issue_blob_selection(host.actor, opened.document_token, + BlobSelectionClaim(target, key), 60) + return DocumentBlobSelectionSubmission(token, "Order", "attachment", + "EXISTING_ROW", "order:10") + + opened = await open_document() + command = SubmittedDocumentCommand("ORD-10", opened.document_token, "command-1", + {"quantity": 3}, (selection(opened, "command-1", "blob_A"),)) + first = await service.accept(context, "Order", "edit-order", command) + assert (first.previous_revision, first.new_revision, first.replayed) == (1, 2, False) + async with aiosqlite.connect(path) as db: + async with db.execute("SELECT version,quantity,blob_key FROM document_order") as cursor: + assert await cursor.fetchone() == (2, 3, "blob_A") + async with db.execute("SELECT key,version FROM teaql_blob_binding") as cursor: + assert await cursor.fetchone() == ("blob_A", 1) + async with db.execute("SELECT status FROM teaql_blob_catalog WHERE key='blob_A'") as cursor: + assert await cursor.fetchone() == ("available",) + assert await receipts(path) == 1 + replayed = await service.accept(context, "Order", "edit-order", command) + assert replayed.replayed and host.mutations == 1 + legacy_parts = [command.business_id, command.document_token, command.command_id, + command.body, [item.__dict__ for item in command.blob_selections]] + legacy_hash = hashlib.sha256(json.dumps(legacy_parts, ensure_ascii=False, + separators=(",", ":"), sort_keys=True).encode("utf-8")).hexdigest() + async with aiosqlite.connect(path) as db: + async with db.execute("SELECT request_hash FROM teaql_document_command_receipt") as cursor: + assert (await cursor.fetchone())[0] != legacy_hash + await db.execute("UPDATE teaql_document_command_receipt SET request_hash=?", (legacy_hash,)) + await db.commit() + old_replay = await service.accept(context, "Order", "edit-order", command) + assert old_replay.replayed and host.mutations == 1 + with pytest.raises(RuntimeError, match="DOCUMENT_REPLAY_CONFLICT"): + await service.accept(context, "Order", "edit-order", replace(command, blob_selections=None)) + oversized = DocumentBlobSelectionSubmission("tqs1." + "A" * (256000 - 5), + "Order", "attachment", "EXISTING_ROW", "order:10") + with pytest.raises(RuntimeError, match="DOCUMENT_COMMAND_INVALID"): + await service.accept(context, "Order", "edit-order", + replace(command, blob_selections=(oversized,))) + + fresh = await open_document() + wrong = SubmittedDocumentCommand("ORD-10", fresh.document_token, "command-wrong", + {"quantity": 3}, (selection(fresh, "other-command", "blob_A"),)) + with pytest.raises(RoundTripReferenceError, match="ROUND_TRIP_REFERENCE_SCOPE_MISMATCH"): + await service.accept(context, "Order", "edit-order", wrong) + duplicate = selection(fresh, "command-duplicate", "blob_A") + with pytest.raises(RuntimeError, match="DOCUMENT_COMMAND_INVALID"): + await service.accept(context, "Order", "edit-order", SubmittedDocumentCommand( + "ORD-10", fresh.document_token, "command-duplicate", {"quantity": 3}, + (duplicate, duplicate))) + with pytest.raises(RuntimeError, match="BLOB_BIND_DENIED"): + await service.accept(context, "Order", "edit-order", SubmittedDocumentCommand( + "ORD-10", fresh.document_token, "command-missing", {"quantity": 3}, + (selection(fresh, "command-missing", "blob_missing"),))) + assert await row(path) == (2, 3) and await receipts(path) == 1 + + async with aiosqlite.connect(path) as db: + await db.execute("""CREATE TRIGGER reject_selected_receipt BEFORE INSERT + ON teaql_document_command_receipt WHEN NEW.command_id='command-receipt-fail' + BEGIN SELECT RAISE(ABORT, 'receipt rejected'); END""") + await db.commit() + with pytest.raises(RuntimeError, match="DOCUMENT_CATALOG_FAILED"): + await service.accept(context, "Order", "edit-order", SubmittedDocumentCommand( + "ORD-10", fresh.document_token, "command-receipt-fail", {"quantity": 3}, + (selection(fresh, "command-receipt-fail", "blob_B"),))) + async with aiosqlite.connect(path) as db: + async with db.execute("SELECT version,quantity,blob_key FROM document_order") as cursor: + assert await cursor.fetchone() == (2, 3, "blob_A") + async with db.execute("SELECT key,version FROM teaql_blob_binding") as cursor: + assert await cursor.fetchone() == ("blob_A", 1) + async with db.execute("SELECT status FROM teaql_blob_catalog WHERE key='blob_B'") as cursor: + assert await cursor.fetchone() == ("uploaded_unbound",) + assert await receipts(path) == 1 diff --git a/scripts/test_verify_public_release_absence.py b/scripts/test_verify_public_release_absence.py new file mode 100644 index 00000000..2057991d --- /dev/null +++ b/scripts/test_verify_public_release_absence.py @@ -0,0 +1,178 @@ +#!/usr/bin/env python3 +"""Exercise PyPI release-absence decisions without external network access.""" + +import importlib.util +import hashlib +from io import BytesIO +import json +from pathlib import Path +import tempfile +import unittest +from urllib.error import HTTPError, URLError + + +SCRIPT = Path(__file__).with_name("verify_public_release_absence.py") +SPEC = importlib.util.spec_from_file_location("teaql_pypi_absence", SCRIPT) +MODULE = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(MODULE) +IDENTITIES = (("teaql", "9.8.7"), ("teaql-blob-reference", "9.8.7"), + ("teaql-secure-document-reference", "9.8.7")) + + +class Response(BytesIO): + def __init__(self, status, body=b""): + super().__init__(body) + self.status = status + + +class PyPIAbsenceTests(unittest.TestCase): + def setUp(self): + self.temporary = tempfile.TemporaryDirectory(prefix="teaql-pypi-absence-") + self.dist = Path(self.temporary.name) + for name, version in IDENTITIES: + (self.dist / f"{name.replace('-', '_')}-{version}-py3-none-any.whl").write_bytes(b"wheel") + (self.dist / f"{name}-{version}.tar.gz").write_bytes(b"sdist") + + def tearDown(self): + self.temporary.cleanup() + + def mock_opener(self, statuses, observed): + def open_request(request, timeout): + self.assertEqual(timeout, 15) + observed.append(request.full_url) + status = statuses.get(request.full_url, 404) + if isinstance(status, Exception): + raise status + if status >= 400: + raise HTTPError(request.full_url, status, "mock", None, None) + return Response(status) + return open_request + + def release_payload(self, name, version): + files = [path for path in self.dist.iterdir() + if path.name.startswith(name.replace("-", "_") + f"-{version}-") + or path.name in (f"{name}-{version}.tar.gz", + f"{name.replace('-', '_')}-{version}.tar.gz")] + return {"info": {"name": name, "version": version}, + "urls": [{"filename": path.name, + "digests": {"sha256": hashlib.sha256(path.read_bytes()).hexdigest()}} + for path in files]} + + def published_opener(self, observed, replacements=None): + replacements = replacements or {} + def open_request(request, timeout): + self.assertEqual(timeout, 15) + observed.append(request.full_url) + parts = request.full_url.removeprefix("https://pypi.org/pypi/").split("/") + name, version = parts[0], parts[1] + value = replacements.get(request.full_url) + if isinstance(value, list): + value = value[min(observed.count(request.full_url) - 1, len(value) - 1)] + if isinstance(value, Exception): + raise value + if value is None: + value = self.release_payload(name, version) + return Response(200, json.dumps(value).encode()) + return open_request + + def test_all_three_absent(self): + MODULE.require_exact_distribution_names(self.dist, IDENTITIES) + observed = [] + MODULE.require_absent(IDENTITIES, self.mock_opener({}, observed)) + self.assertEqual(observed, [f"https://pypi.org/pypi/{name}/{version}/json" + for name, version in IDENTITIES]) + + def test_occupied_non_base_project_blocks_publication(self): + observed = [] + url = "https://pypi.org/pypi/teaql-blob-reference/9.8.7/json" + with self.assertRaisesRegex(RuntimeError, "already exists"): + MODULE.require_absent(IDENTITIES, self.mock_opener({url: 200}, observed)) + self.assertEqual(observed[-1], url) + self.assertEqual(len(observed), 2) + + def test_registry_failure_fails_closed(self): + url = "https://pypi.org/pypi/teaql/9.8.7/json" + with self.assertRaisesRegex(RuntimeError, "Could not prove"): + MODULE.require_absent(IDENTITIES, self.mock_opener({url: 503}, [])) + + def test_network_failure_fails_closed(self): + url = "https://pypi.org/pypi/teaql/9.8.7/json" + with self.assertRaisesRegex(RuntimeError, "Could not prove"): + MODULE.require_absent(IDENTITIES, self.mock_opener({url: URLError("offline")}, [])) + + def test_unexpected_success_status_fails_closed(self): + url = "https://pypi.org/pypi/teaql/9.8.7/json" + with self.assertRaisesRegex(RuntimeError, "HTTP 204"): + MODULE.require_absent(IDENTITIES, self.mock_opener({url: 204}, [])) + + def test_exact_six_distributions_required(self): + (self.dist / "teaql-blob-reference-9.8.7.tar.gz").unlink() + with self.assertRaisesRegex(ValueError, "exactly three wheels"): + MODULE.require_exact_distribution_names(self.dist, IDENTITIES) + + def test_current_source_identifies_three_matching_projects(self): + identities = MODULE.release_identities() + self.assertEqual([name for name, _ in identities], + [name for name, _ in IDENTITIES]) + self.assertEqual(len({version for _, version in identities}), 1) + + def test_exact_three_published_release_file_sets(self): + observed = [] + MODULE.require_published_exact(self.dist, IDENTITIES, + self.published_opener(observed), lambda _: None) + self.assertEqual(len(observed), 3) + + def test_eventually_visible_release_files(self): + observed = [] + name, version = IDENTITIES[0] + url = f"https://pypi.org/pypi/{name}/{version}/json" + partial = self.release_payload(name, version) + partial["urls"] = partial["urls"][:1] + MODULE.require_published_exact(self.dist, IDENTITIES, + self.published_opener(observed, {url: [partial, self.release_payload(name, version)]}), + lambda _: None) + self.assertEqual(observed.count(url), 2) + + def test_published_digest_mismatch_rejected(self): + observed = [] + name, version = IDENTITIES[0] + url = f"https://pypi.org/pypi/{name}/{version}/json" + wrong = self.release_payload(name, version) + wrong["urls"][0]["digests"]["sha256"] = "0" * 64 + with self.assertRaisesRegex(RuntimeError, "digest mismatch"): + MODULE.require_published_exact(self.dist, IDENTITIES, + self.published_opener(observed, {url: wrong}), lambda _: None) + + def test_unexpected_published_file_rejected(self): + observed = [] + name, version = IDENTITIES[0] + url = f"https://pypi.org/pypi/{name}/{version}/json" + extra = self.release_payload(name, version) + extra["urls"].append({"filename": "unrecognized.whl", "digests": {"sha256": "0" * 64}}) + with self.assertRaisesRegex(RuntimeError, "Unexpected extra"): + MODULE.require_published_exact(self.dist, IDENTITIES, + self.published_opener(observed, {url: extra}), lambda _: None) + + def test_duplicate_published_filename_rejected(self): + observed = [] + name, version = IDENTITIES[0] + url = f"https://pypi.org/pypi/{name}/{version}/json" + duplicate = self.release_payload(name, version) + duplicate["urls"].append(dict(duplicate["urls"][0])) + with self.assertRaisesRegex(RuntimeError, "Duplicate"): + MODULE.require_published_exact(self.dist, IDENTITIES, + self.published_opener(observed, {url: duplicate}), lambda _: None) + + def test_published_release_never_appears_fails_closed(self): + observed = [] + name, version = IDENTITIES[0] + url = f"https://pypi.org/pypi/{name}/{version}/json" + absent = HTTPError(url, 404, "mock", None, None) + with self.assertRaisesRegex(RuntimeError, "incomplete after retries"): + MODULE.require_published_exact(self.dist, IDENTITIES, + self.published_opener(observed, {url: absent}), lambda _: None) + self.assertEqual(observed.count(url), 5) + + +if __name__ == "__main__": + unittest.main(verbosity=2) diff --git a/scripts/test_verify_signed_release_tag.py b/scripts/test_verify_signed_release_tag.py new file mode 100644 index 00000000..ba1ba95d --- /dev/null +++ b/scripts/test_verify_signed_release_tag.py @@ -0,0 +1,83 @@ +#!/usr/bin/env python3 +"""Exercise the actual signed-tag gate with valid and invalid Git objects.""" + +import importlib.util +from pathlib import Path +import unittest + + +path = Path(__file__).with_name("verify_signed_release_tag.py") +spec = importlib.util.spec_from_file_location("verify_signed_release_tag", path) +subject = importlib.util.module_from_spec(spec) +spec.loader.exec_module(subject) + +COMMIT = "a" * 40 +TAG_SHA = "b" * 40 +TAG = "python-v0.3.2" +BASE = "https://api.github.com/repos/teaql/teaql-python" + + +def valid_responses(): + return { + BASE + "/git/ref/tags/" + TAG: { + "ref": "refs/tags/" + TAG, "object": {"type": "tag", "sha": TAG_SHA}}, + BASE + "/git/tags/" + TAG_SHA: { + "sha": TAG_SHA, "tag": TAG, + "object": {"type": "commit", "sha": COMMIT}, + "verification": {"verified": True, "reason": "valid", "signature": "test-signature"}}, + } + + +class SignedReleaseTagTests(unittest.TestCase): + def run_case(self, responses): + requested = [] + def get(url): + requested.append(url) + return responses[url] + return subject.verify("teaql/teaql-python", TAG, COMMIT, get), requested + + def test_verified_annotated_tag_points_to_exact_release_commit(self): + tag_sha, requested = self.run_case(valid_responses()) + self.assertEqual(tag_sha, TAG_SHA) + self.assertEqual(requested, list(valid_responses())) + + def test_lightweight_tag_cannot_publish(self): + responses = valid_responses() + responses[BASE + "/git/ref/tags/" + TAG]["object"] = {"type": "commit", "sha": COMMIT} + with self.assertRaisesRegex(ValueError, "annotated tag"): + self.run_case(responses) + + def test_unsigned_annotated_tag_cannot_publish(self): + responses = valid_responses() + responses[BASE + "/git/tags/" + TAG_SHA]["verification"] = { + "verified": False, "reason": "unsigned", "signature": None} + with self.assertRaisesRegex(ValueError, "verified a signature"): + self.run_case(responses) + + def test_tag_for_another_commit_cannot_publish(self): + responses = valid_responses() + responses[BASE + "/git/tags/" + TAG_SHA]["object"]["sha"] = "c" * 40 + with self.assertRaisesRegex(ValueError, "release commit"): + self.run_case(responses) + + def test_changed_tag_reference_cannot_publish(self): + responses = valid_responses() + responses[BASE + "/git/ref/tags/" + TAG]["ref"] = "refs/tags/vother" + with self.assertRaisesRegex(ValueError, "different tag reference"): + self.run_case(responses) + + def test_unverified_signature_cannot_publish(self): + responses = valid_responses() + responses[BASE + "/git/tags/" + TAG_SHA]["verification"]["verified"] = False + with self.assertRaisesRegex(ValueError, "verified a signature"): + self.run_case(responses) + + def test_verified_flag_without_signature_cannot_publish(self): + responses = valid_responses() + responses[BASE + "/git/tags/" + TAG_SHA]["verification"]["signature"] = None + with self.assertRaisesRegex(ValueError, "verified a signature"): + self.run_case(responses) + + +if __name__ == "__main__": + unittest.main(verbosity=2) diff --git a/scripts/verify-reference-release-artifacts.py b/scripts/verify-reference-release-artifacts.py new file mode 100644 index 00000000..97cf35f2 --- /dev/null +++ b/scripts/verify-reference-release-artifacts.py @@ -0,0 +1,116 @@ +#!/usr/bin/env python3 +"""Check the exact three-module wheel/sdist set before artifact upload.""" + +from email.parser import Parser +from pathlib import Path +import sys +import tarfile +from zipfile import ZipFile + + +ROOT = Path(__file__).resolve().parents[1] +if len(sys.argv) != 2 or not Path(sys.argv[1]).is_dir(): + raise SystemExit("Usage: verify-reference-release-artifacts.py ") +DIST = Path(sys.argv[1]).resolve() + +try: + import tomllib +except ImportError: # Local Python 3.10; release CI uses Python 3.12. + import tomli as tomllib + +projects = { + "teaql": (ROOT, "teaql"), + "teaql-blob-reference": (ROOT / "packages/teaql-blob-reference", "teaql_blob_reference"), + "teaql-secure-document-reference": ( + ROOT / "packages/teaql-secure-document-reference", "teaql_secure_document_reference" + ), +} +versions = {} +for name, (source, _) in projects.items(): + metadata = tomllib.loads((source / "pyproject.toml").read_text())['project'] + if metadata['name'] != name: + raise SystemExit(f"Unexpected project name in {source}") + versions[name] = metadata['version'] +if len(set(versions.values())) != 1: + raise SystemExit("Reference module versions differ from the base package") + +artifacts = sorted(DIST.iterdir()) +wheels = [p for p in artifacts if p.name.endswith(".whl")] +sdists = [p for p in artifacts if p.name.endswith(".tar.gz")] +if len(artifacts) != 6 or len(wheels) != 3 or len(sdists) != 3: + raise SystemExit("Expected exactly three wheels and three source distributions") + + +def identity(raw: bytes) -> tuple[str, str]: + data = Parser().parsestr(raw.decode("utf-8")) + return data.get("Name", ""), data.get("Version", "") + + +def source_files(source: Path, package: str) -> dict[str, bytes]: + root = source / "src" / package + return { + str(path.relative_to(source / "src")): path.read_bytes() + for path in root.rglob("*") + if path.is_file() and "__pycache__" not in path.parts and path.suffix != ".pyc" + } + + +seen_wheels = set() +for wheel in wheels: + with ZipFile(wheel) as archive: + metadata_files = [entry for entry in archive.namelist() + if entry.endswith(".dist-info/METADATA")] + if len(metadata_files) != 1: + raise SystemExit(f"Unexpected wheel metadata in {wheel.name}") + name, version = identity(archive.read(metadata_files[0])) + if name not in projects or version != versions[name] or name in seen_wheels: + raise SystemExit(f"Unexpected wheel identity in {wheel.name}: {name} {version}") + expected_name = name.replace("-", "_") + f"-{version}-py3-none-any.whl" + if wheel.name != expected_name: + raise SystemExit(f"Unexpected wheel filename: {wheel.name}") + source, package = projects[name] + entries = set(archive.namelist()) + for other in {item[1] for item in projects.values()} - {package}: + if any(entry.startswith(other + "/") for entry in entries): + raise SystemExit(f"Other module leaked into {wheel.name}: {other}") + files = source_files(source, package) + if not files or any(path not in entries or archive.read(path) != data + for path, data in files.items()): + raise SystemExit(f"Source bytes missing or changed in {wheel.name}") + seen_wheels.add(name) + print(f"PASS exact {name} {version} wheel, {len(files)} source files") + +seen_sdists = set() +for sdist in sdists: + with tarfile.open(sdist, "r:gz") as archive: + entries = {entry.name: entry for entry in archive.getmembers() if entry.isfile()} + metadata_files = [path for path in entries if path.endswith("/PKG-INFO") + and path.count("/") == 1] + if len(metadata_files) != 1: + raise SystemExit(f"Unexpected sdist metadata in {sdist.name}") + extracted = archive.extractfile(entries[metadata_files[0]]) + if extracted is None: + raise SystemExit(f"Missing sdist metadata bytes in {sdist.name}") + name, version = identity(extracted.read()) + if name not in projects or version != versions[name] or name in seen_sdists: + raise SystemExit(f"Unexpected sdist identity in {sdist.name}: {name} {version}") + source, package = projects[name] + root = metadata_files[0].split("/", 1)[0] + if sdist.name not in (f"{name}-{version}.tar.gz", + f"{name.replace('-', '_')}-{version}.tar.gz"): + raise SystemExit(f"Unexpected sdist filename: {sdist.name}") + for other in {item[1] for item in projects.values()} - {package}: + if any(path.startswith(f"{root}/src/{other}/") for path in entries): + raise SystemExit(f"Other module leaked into {sdist.name}: {other}") + files = source_files(source, package) + for path, data in files.items(): + member = entries.get(f"{root}/src/{path}") + content = None if member is None else archive.extractfile(member) + if content is None or content.read() != data: + raise SystemExit(f"Source bytes missing or changed in {sdist.name}: {path}") + seen_sdists.add(name) + print(f"PASS exact {name} {version} sdist, {len(files)} source files") + +if seen_wheels != set(projects) or seen_sdists != set(projects): + raise SystemExit("Missing reference module distribution") +print("PASS exact three-package Python release set: 3 wheels and 3 sdists") diff --git a/scripts/verify-reference-release-consumers.py b/scripts/verify-reference-release-consumers.py new file mode 100644 index 00000000..0e3df1a4 --- /dev/null +++ b/scripts/verify-reference-release-consumers.py @@ -0,0 +1,77 @@ +#!/usr/bin/env python3 +"""Import exact release wheels in four isolated TeaQL package installations.""" + +from pathlib import Path +import json +import subprocess +import sys +import tempfile + + +if len(sys.argv) != 2 or not Path(sys.argv[1]).is_dir(): + raise SystemExit("Usage: verify-reference-release-consumers.py ") +dist = Path(sys.argv[1]).resolve() +wheels = { + "base": list(dist.glob("teaql-*-py3-none-any.whl")), + "blob": list(dist.glob("teaql_blob_reference-*-py3-none-any.whl")), + "secure": list(dist.glob("teaql_secure_document_reference-*-py3-none-any.whl")), +} +if any(len(files) != 1 for files in wheels.values()): + raise SystemExit("Expected exactly one wheel per reference package") +variants = { + "base": ("base",), + "secure-only": ("base", "secure"), + "blob-only": ("base", "blob"), + "combined": ("base", "blob", "secure"), +} +third_party = [path for path in sys.path + if "site-packages" in path and Path(path).is_dir()] + +consumer = r''' +import importlib.util +import json +from pathlib import Path +import sys + +target = Path(sys.argv[1]).resolve() +third_party = json.loads(sys.argv[2]) +expected_blob = sys.argv[3] == "1" +expected_secure = sys.argv[4] == "1" +sys.path = [str(target)] + [path for path in sys.path if "site-packages" not in path] + third_party + +from teaql.runtime import RuntimeModule +assert RuntimeModule.__name__ == "RuntimeModule" +assert Path(importlib.util.find_spec("teaql").origin).resolve().is_relative_to(target) +for module, expected, symbol in ( + ("teaql_blob_reference", expected_blob, "BlobReferenceModule"), + ("teaql_secure_document_reference", expected_secure, "SecureDocumentReferenceModule"), +): + spec = importlib.util.find_spec(module) + assert (spec is not None) == expected, (module, expected) + if expected: + package = __import__(module) + assert getattr(package, symbol).__name__ == symbol + assert Path(spec.origin).resolve().is_relative_to(target) +print("PASS isolated wheel consumer") +''' + +with tempfile.TemporaryDirectory(prefix="teaql-python-release-consumers-") as scratch: + for variant, packages in variants.items(): + target = Path(scratch) / variant + target.mkdir() + command = [sys.executable, "-m", "pip", "install", "--no-index", "--no-deps", + "--disable-pip-version-check", "--target", str(target)] + command.extend(str(wheels[package][0]) for package in packages) + installed = subprocess.run(command, capture_output=True, text=True) + if installed.returncode: + raise SystemExit(f"{variant}: exact wheel install failed: " + + (installed.stdout + installed.stderr)[-900:]) + checked = subprocess.run( + [sys.executable, "-S", "-c", consumer, str(target), json.dumps(third_party), + "1" if "blob" in packages else "0", "1" if "secure" in packages else "0"], + cwd=scratch, capture_output=True, text=True, + ) + if checked.returncode or "PASS isolated wheel consumer" not in checked.stdout: + raise SystemExit(f"{variant}: isolated wheel import failed: " + + (checked.stdout + checked.stderr)[-900:]) + print(f"PASS {variant} exact-wheel consumer") diff --git a/scripts/verify-reference-release-dependencies.py b/scripts/verify-reference-release-dependencies.py new file mode 100644 index 00000000..f33c863a --- /dev/null +++ b/scripts/verify-reference-release-dependencies.py @@ -0,0 +1,149 @@ +#!/usr/bin/env python3 +"""Install the exact reference release wheels and declared extras in clean venvs.""" + +from pathlib import Path +import hashlib +import os +import subprocess +import sys +import tempfile + + +if len(sys.argv) != 2 or not Path(sys.argv[1]).is_dir(): + raise SystemExit("Usage: verify-reference-release-dependencies.py ") +DIST = Path(sys.argv[1]).resolve() + + +def run(args, *, cwd, env, timeout=900): + result = subprocess.run(args, cwd=cwd, env=env, capture_output=True, text=True, + timeout=timeout) + if result.returncode: + raise RuntimeError(" ".join(map(str, args)) + " failed:\n" + + (result.stdout + result.stderr)[-5000:]) + return result.stdout + result.stderr + + +def exact_wheels(): + names = { + "base": "teaql-", + "blob": "teaql_blob_reference-", + "secure": "teaql_secure_document_reference-", + } + result = {} + for name, prefix in names.items(): + found = [path for path in DIST.glob("*.whl") + if path.name.startswith(prefix) and path.name.endswith("-py3-none-any.whl")] + if len(found) != 1: + raise RuntimeError("expected exactly one " + name + " release wheel") + result[name] = found[0] + return result + + +CHECK = r''' +from importlib import metadata, util +from pathlib import Path +import sys + +root = Path(sys.prefix).resolve() +variant = sys.argv[1] +blob = variant in ("blob-only", "combined") +secure = variant in ("secure-only", "combined") +for distribution, expected in ( + ("teaql", True), + ("teaql-blob-reference", blob), + ("teaql-secure-document-reference", secure), +): + try: + installed = metadata.distribution(distribution) + except metadata.PackageNotFoundError: + installed = None + assert (installed is not None) == expected, (variant, distribution) + if installed is not None: + assert Path(installed.locate_file("")).resolve().is_relative_to(root), distribution +from teaql.runtime import RuntimeModule +assert RuntimeModule.__name__ == "RuntimeModule" +assert (util.find_spec("teaql_blob_reference") is not None) == blob +assert (util.find_spec("teaql_secure_document_reference") is not None) == secure +if blob: + from teaql_blob_reference import BlobReferenceModule + from teaql_blob_reference.content_pillow import PillowBlobContentVerifier + from teaql_blob_reference.content_pypdf import PypdfBlobContentVerifier + from teaql_blob_reference.content_office import OfficeBlobContentVerifier + from teaql.provider.s3 import S3ObjectStorage + import boto3 + assert BlobReferenceModule.__name__ == "BlobReferenceModule" + assert S3ObjectStorage.__name__ == "S3ObjectStorage" + assert boto3.__version__ + PillowBlobContentVerifier().validate_ready() + PypdfBlobContentVerifier().validate_ready() + OfficeBlobContentVerifier().validate_ready() +else: + assert util.find_spec("boto3") is None + assert util.find_spec("PIL") is None +if secure: + from teaql_secure_document_reference import SecureDocumentReferenceModule + assert SecureDocumentReferenceModule.__name__ == "SecureDocumentReferenceModule" +if variant == "combined": + from teaql.runtime.opentelemetry import OpenTelemetryRuntimeTelemetry + import opentelemetry.sdk + assert OpenTelemetryRuntimeTelemetry.__name__ == "OpenTelemetryRuntimeTelemetry" +print("PASS isolated dependency consumer " + variant) +''' + + +def main(): + wheels = exact_wheels() + with tempfile.TemporaryDirectory(prefix="teaql-python-release-dependencies-") as temporary: + work = Path(temporary) + wheelhouse = work / "wheelhouse" + wheelhouse.mkdir() + environment = dict(os.environ, PIP_CACHE_DIR=str(work / "pip-cache"), + PIP_DISABLE_PIP_VERSION_CHECK="1", PIP_NO_INPUT="1", + PIP_PROGRESS_BAR="off", PYTHONNOUSERSITE="1") + environment.pop("PYTHONPATH", None) + inputs = [ + str(wheels["base"]) + "[object-storage-s3,telemetry]", + str(wheels["blob"]) + "[image,pdf,office]", + str(wheels["secure"]), + ] + run([sys.executable, "-m", "pip", "download", "--only-binary=:all:", + "--dest", str(wheelhouse), *inputs], cwd=work, env=environment) + for name, candidate in wheels.items(): + copy = wheelhouse / candidate.name + if not copy.is_file() or copy.read_bytes() != candidate.read_bytes(): + raise RuntimeError(name + " candidate wheel not retained exactly") + print("PASS exact " + name + " wheel_sha256=" + + hashlib.sha256(candidate.read_bytes()).hexdigest(), flush=True) + resolved = sorted(wheelhouse.glob("*.whl")) + manifest = "\n".join( + path.name + " " + hashlib.sha256(path.read_bytes()).hexdigest() + for path in resolved + ).encode() + print("PASS resolved wheelhouse count=" + str(len(resolved)) + + " manifest_sha256=" + hashlib.sha256(manifest).hexdigest(), flush=True) + cases = { + "base": (str(wheels["base"]),), + "secure-only": (str(wheels["base"]), str(wheels["secure"])), + "blob-only": (str(wheels["base"]) + "[object-storage-s3]", + str(wheels["blob"]) + "[image,pdf,office]"), + "combined": (str(wheels["base"]) + "[object-storage-s3,telemetry]", + str(wheels["blob"]) + "[image,pdf,office]", str(wheels["secure"])), + } + for name, packages in cases.items(): + venv = work / name + run([sys.executable, "-m", "venv", str(venv)], cwd=work, env=environment) + python = venv / "bin/python" + run([str(python), "-I", "-m", "pip", "install", "--no-index", + "--find-links", str(wheelhouse), "--only-binary=:all:", + "--no-cache-dir", *packages], cwd=work, env=environment) + run([str(python), "-I", "-m", "pip", "check"], cwd=work, env=environment) + output = run([str(python), "-I", "-c", CHECK, name], + cwd=work, env=environment) + if "PASS isolated dependency consumer " + name not in output: + raise RuntimeError(name + " application did not run") + print("PASS isolated dependency consumer " + name, flush=True) + print("PASS exact Python release-wheel dependency closure 4/4", flush=True) + + +if __name__ == "__main__": + main() diff --git a/scripts/verify_public_release_absence.py b/scripts/verify_public_release_absence.py new file mode 100644 index 00000000..8efc3062 --- /dev/null +++ b/scripts/verify_public_release_absence.py @@ -0,0 +1,148 @@ +#!/usr/bin/env python3 +"""Fail closed if any exact TeaQL Python release version is already on PyPI.""" + +import hashlib +import json +from pathlib import Path +import re +import sys +import time +from urllib.error import HTTPError, URLError +from urllib.request import Request, urlopen + +try: + import tomllib +except ImportError: # Local Python 3.10; release CI uses Python 3.12. + import tomli as tomllib + + +ROOT = Path(__file__).resolve().parents[1] +PROJECTS = ( + ROOT / "pyproject.toml", + ROOT / "packages/teaql-blob-reference/pyproject.toml", + ROOT / "packages/teaql-secure-document-reference/pyproject.toml", +) + + +def release_identities(project_files=PROJECTS): + identities = [] + for project_file in project_files: + project = tomllib.loads(project_file.read_text())["project"] + name, version = project["name"], project["version"] + if (not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9._-]*", name) + or not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9._+-]*", version)): + raise ValueError(f"Invalid release identity in {project_file}") + identities.append((name, version)) + if len({name for name, _ in identities}) != 3 or len({version for _, version in identities}) != 1: + raise ValueError("Expected three distinct TeaQL projects with one release version") + return identities + + +def require_exact_distribution_names(dist, identities): + if not dist.is_dir(): + raise ValueError(f"Missing distribution directory: {dist}") + names = [path.name for path in dist.iterdir() + if path.is_file() and not path.is_symlink()] + if len(names) != 6 or len(list(dist.iterdir())) != 6: + raise ValueError("Expected exactly three wheels and three source distributions") + for name, version in identities: + wheel = name.replace("-", "_") + f"-{version}-py3-none-any.whl" + sdist = {f"{name}-{version}.tar.gz", f"{name.replace('-', '_')}-{version}.tar.gz"} + if names.count(wheel) != 1 or sum(names.count(item) for item in sdist) != 1: + raise ValueError(f"Missing exact distribution filenames for {name} {version}") + + +def require_absent(identities, opener=urlopen): + for name, version in identities: + url = f"https://pypi.org/pypi/{name}/{version}/json" + request = Request(url, headers={"Accept": "application/json", + "User-Agent": "teaql-release-preflight/1"}) + try: + with opener(request, timeout=15) as response: + status = response.status + except HTTPError as exc: + if exc.code == 404: + exc.close() + continue + raise RuntimeError(f"Could not prove {name} {version} absent on PyPI: HTTP {exc.code}") from exc + except (URLError, TimeoutError) as exc: + raise RuntimeError(f"Could not prove {name} {version} absent on PyPI") from exc + if status == 200: + raise RuntimeError(f"PyPI version already exists: {name} {version}") + raise RuntimeError(f"Could not prove {name} {version} absent on PyPI: HTTP {status}") + + +def require_published_exact(dist, identities, opener=urlopen, sleeper=time.sleep): + for name, version in identities: + expected = { + path.name: hashlib.sha256(path.read_bytes()).hexdigest() + for path in dist.iterdir() + if path.name.startswith(name.replace("-", "_") + f"-{version}-") + or path.name in (f"{name}-{version}.tar.gz", + f"{name.replace('-', '_')}-{version}.tar.gz") + } + if len(expected) != 2: + raise ValueError(f"Expected exact wheel and source distribution for {name} {version}") + url = f"https://pypi.org/pypi/{name}/{version}/json" + request = Request(url, headers={"Accept": "application/json", + "User-Agent": "teaql-release-integrity/1"}) + for attempt in range(5): + try: + with opener(request, timeout=15) as response: + if response.status != 200: + raise RuntimeError(f"Unexpected PyPI HTTP {response.status}: {name} {version}") + raw = response.read(1_000_001) + if len(raw) > 1_000_000: + raise RuntimeError(f"Oversized PyPI release metadata: {name} {version}") + payload = json.loads(raw) + actual = {item["filename"]: item["digests"]["sha256"] + for item in payload["urls"]} + if len(actual) != len(payload["urls"]): + raise RuntimeError(f"Duplicate PyPI release filename: {name} {version}") + if (payload["info"]["name"], payload["info"]["version"]) != (name, version): + raise RuntimeError(f"Unexpected PyPI release identity: {name} {version}") + if actual == expected: + break + if any(filename in actual and actual[filename] != digest + for filename, digest in expected.items()): + raise RuntimeError(f"PyPI release byte digest mismatch: {name} {version}") + if set(actual) - set(expected): + raise RuntimeError(f"Unexpected extra PyPI release file: {name} {version}") + except HTTPError as exc: + if exc.code != 404: + exc.close() + raise RuntimeError(f"Could not verify PyPI release: {name} {version}, HTTP {exc.code}") from exc + exc.close() + except (URLError, TimeoutError) as exc: + raise RuntimeError(f"Could not verify PyPI release: {name} {version}") from exc + except (KeyError, TypeError, ValueError) as exc: + raise RuntimeError(f"Malformed PyPI release metadata: {name} {version}") from exc + if attempt == 4: + raise RuntimeError(f"PyPI release incomplete after retries: {name} {version}") + sleeper(3) + print(f"PASS exact published PyPI bytes: {name} {version}") + + +def main(argv): + if len(argv) == 2: + published = False + dist = Path(argv[1]) + elif len(argv) == 3 and argv[1] == "--published": + published = True + dist = Path(argv[2]) + else: + raise SystemExit("Usage: verify_public_release_absence.py [--published] ") + identities = release_identities() + require_exact_distribution_names(dist, identities) + if published: + require_published_exact(dist, identities) + else: + require_absent(identities) + print("PASS all three TeaQL Python release versions are absent on PyPI") + + +if __name__ == "__main__": + try: + main(sys.argv) + except (OSError, KeyError, TypeError, ValueError, RuntimeError) as exc: + raise SystemExit(str(exc)) from exc diff --git a/scripts/verify_signed_release_tag.py b/scripts/verify_signed_release_tag.py new file mode 100644 index 00000000..fc209958 --- /dev/null +++ b/scripts/verify_signed_release_tag.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python3 +"""Require the selected release tag to be a signed tag verified by GitHub.""" + +import json +import os +import re +import sys +from urllib.parse import quote +from urllib.request import Request, urlopen + + +def _sha(value): + return isinstance(value, str) and re.fullmatch(r"[0-9a-f]{40}|[0-9a-f]{64}", value) is not None + + +def fetch_json(url, token): + request = Request(url, headers={ + "Accept": "application/vnd.github+json", + "Authorization": "Bearer " + token, + "X-GitHub-Api-Version": "2022-11-28", + }) + with urlopen(request, timeout=20) as response: + if response.status != 200: + raise ValueError("GitHub tag API did not return HTTP 200") + return json.load(response) + + +def verify(repository, tag, commit, get_json): + if re.fullmatch(r"[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+", repository or "") is None: + raise ValueError("invalid GitHub repository") + if not tag or not tag.startswith("python-v") or any(ord(c) < 32 for c in tag): + raise ValueError("invalid release tag") + if not _sha(commit): + raise ValueError("invalid release commit SHA") + + base = "https://api.github.com/repos/" + repository + reference = get_json(base + "/git/ref/tags/" + quote(tag, safe="")) + if reference.get("ref") != "refs/tags/" + tag: + raise ValueError("GitHub returned a different tag reference") + target = reference.get("object") or {} + if target.get("type") != "tag" or not _sha(target.get("sha")): + raise ValueError("release tag is not an annotated tag object") + + tag_object = get_json(base + "/git/tags/" + target["sha"]) + if tag_object.get("sha") != target["sha"] or tag_object.get("tag") != tag: + raise ValueError("annotated tag object does not match its reference") + tagged_commit = tag_object.get("object") or {} + if tagged_commit.get("type") != "commit" or tagged_commit.get("sha") != commit: + raise ValueError("signed tag does not point to the release commit") + verification = tag_object.get("verification") or {} + if verification.get("verified") is not True or not verification.get("signature"): + raise ValueError("GitHub has not verified a signature on the release tag") + return target["sha"] + + +def main(): + try: + repository = os.environ["GITHUB_REPOSITORY"] + tag = os.environ["GITHUB_REF_NAME"] + commit = os.environ["GITHUB_SHA"] + token = os.environ["GITHUB_TOKEN"] + if not token: + raise ValueError("missing GitHub token") + tag_sha = verify(repository, tag, commit, lambda url: fetch_json(url, token)) + except (KeyError, ValueError, OSError) as error: + print("Release tag verification failed: " + str(error), file=sys.stderr) + return 1 + print("PASS verified signed tag " + tag + " at " + commit + " (" + tag_sha + ")") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/src/teaql/core/__init__.py b/src/teaql/core/__init__.py index 688c9db7..dabf482a 100644 --- a/src/teaql/core/__init__.py +++ b/src/teaql/core/__init__.py @@ -1,4 +1,5 @@ from .value import Value, DataType, Timestamp +from .blob_ref import BlobRef, BlobFieldSpec from .entity import BaseEntityData, EntityKey, EntityChangeSet, EntityRoot from .expr import ( Expr, ExprBuilder, BinaryOp, ExprFunction, @@ -46,7 +47,7 @@ __all__ = [ "QueryIntent", "MutationIntent", "RequestIntentError", "QueryOperation", - "Value", "DataType", "Timestamp", + "Value", "DataType", "Timestamp", "BlobRef", "BlobFieldSpec", "BaseEntityData", "EntityKey", "EntityChangeSet", "EntityRoot", "Expr", "ExprBuilder", "BinaryOp", "ExprFunction", "SelectQuery", "SortDirection", "OrderBy", "Aggregate", "AggregateFunction", diff --git a/src/teaql/core/blob_ref.py b/src/teaql/core/blob_ref.py new file mode 100644 index 00000000..94b69cbd --- /dev/null +++ b/src/teaql/core/blob_ref.py @@ -0,0 +1,105 @@ +"""Stable Blob catalog values shared with the Java/Rust reference contract.""" + +from __future__ import annotations + +import re + +_KEY = re.compile(r"blob_[A-Za-z0-9_-]{1,123}", re.ASCII) +_MIME = { + "jpg": "image/jpeg", "jpeg": "image/jpeg", "png": "image/png", + "mp4": "video/mp4", "mp3": "audio/mpeg", + "doc": "application/msword", + "docx": "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + "xls": "application/vnd.ms-excel", + "xlsx": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + "pdf": "application/pdf", +} + + +def blob_mime_for_extension(extension: str) -> str | None: + """Resolve only the frozen Blob format set; never trust client MIME.""" + return _MIME.get(extension) if isinstance(extension, str) else None + + +class BlobRef: + """A Key identifies catalog state; it never conveys read or bind authority.""" + + __slots__ = ("_key",) + + def __init__(self, key: str) -> None: + if not isinstance(key, str) or _KEY.fullmatch(key) is None: + raise ValueError("BLOB_REFERENCE_INVALID") + object.__setattr__(self, "_key", key) + + def __setattr__(self, name: str, value: object) -> None: + raise AttributeError("BlobRef is immutable") + + @property + def key(self) -> str: + return self._key + + @classmethod + def from_json(cls, value: object) -> "BlobRef": + if not isinstance(value, str): + raise ValueError("BLOB_REFERENCE_INVALID") + return cls(value) + + @classmethod + def from_sql(cls, value: object) -> "BlobRef": + if not isinstance(value, str): + raise ValueError("BLOB_REFERENCE_INVALID") + return cls(value) + + def to_sql(self) -> str: + return self._key + + def __repr__(self) -> str: + return "BlobRef[masked]" + + __str__ = __repr__ + + def __eq__(self, other: object) -> bool: + return isinstance(other, BlobRef) and self._key == other._key + + def __hash__(self) -> int: + return hash(self._key) + + +class BlobFieldSpec: + """Trusted model restriction; server-selected policy may narrow formats.""" + + __slots__ = ("entity", "field", "nullable", "policy_profile", "allowed_extensions", + "allowed_content_types") + CAPABILITY = "blob.v1" + + def __init__(self, entity: str, field: str, extensions: list[str] | tuple[str, ...], + nullable: bool, policy_profile: str) -> None: + if (not isinstance(entity, str) or not entity or entity != entity.strip() + or not isinstance(field, str) or not field or field != field.strip() + or not isinstance(policy_profile, str) or not policy_profile + or policy_profile != policy_profile.strip()): + raise ValueError("BLOB_POLICY_INVALID") + try: + normalized = tuple(sorted(set(extensions))) + except (TypeError, ValueError): + raise ValueError("BLOB_FORMAT_NOT_ALLOWED") from None + if not normalized or any(extension not in _MIME for extension in normalized): + raise ValueError("BLOB_FORMAT_NOT_ALLOWED") + object.__setattr__(self, "entity", entity) + object.__setattr__(self, "field", field) + object.__setattr__(self, "nullable", nullable) + object.__setattr__(self, "policy_profile", policy_profile) + object.__setattr__(self, "allowed_extensions", normalized) + object.__setattr__(self, "allowed_content_types", + tuple(sorted({_MIME[extension] for extension in normalized}))) + + def __setattr__(self, name: str, value: object) -> None: + raise AttributeError("BlobFieldSpec is immutable") + + @property + def protocol_version(self) -> str: + return self.CAPABILITY + + @property + def max_files(self) -> int: + return 1 diff --git a/src/teaql/core/meta.py b/src/teaql/core/meta.py index 50c9f3dd..dbf25524 100644 --- a/src/teaql/core/meta.py +++ b/src/teaql/core/meta.py @@ -7,6 +7,17 @@ def __init__(self, name: str, property_type: str = "String"): self._is_version = False self.nullable = True self.log_policy_val = 'unknown' + self.blob_spec = None + def blob(self, spec): + from .blob_ref import BlobFieldSpec + from .value import DataType + if not isinstance(spec, BlobFieldSpec) or spec.field != self.name: + raise ValueError("BLOB_POLICY_INVALID") + self.blob_spec = spec + self.property_type = DataType.Text + self.nullable = spec.nullable + self.log_policy_val = 'credential' + return self def log_policy(self, policy): """Trusted schema policy; incoming requests cannot override it.""" if policy not in ('plain', 'masked', 'credential', 'unknown'): @@ -59,6 +70,8 @@ def table_name(self, name): self.table_name_val = name return self def property(self, prop): + if prop.blob_spec is not None and prop.blob_spec.entity != self._name: + raise ValueError("BLOB_POLICY_INVALID") self.properties.append(prop) return self diff --git a/src/teaql/core/object_storage.py b/src/teaql/core/object_storage.py index d5137367..13cd51ba 100644 --- a/src/teaql/core/object_storage.py +++ b/src/teaql/core/object_storage.py @@ -4,6 +4,7 @@ from enum import Enum from types import MappingProxyType from typing import BinaryIO, Mapping, Optional +from urllib.parse import quote import unicodedata MAX_PUT_SIZE = 5 * 1024**3 @@ -116,14 +117,25 @@ def __post_init__(self): if self.disposition not in (None, "inline", "attachment"): raise StorageError(StorageErrorCode.INVALID_INPUT) if self.filename is not None: - _header(self.filename) - if self.disposition is None or any(c in self.filename for c in '/\\"'): + try: + valid_length = 0 < len(self.filename.encode("utf-8")) <= 1024 + except (AttributeError, UnicodeEncodeError): + valid_length = False + if (self.disposition is None or not valid_length or + any(c in '/\\"' or ord(c) < 32 or 127 <= ord(c) <= 159 + for c in self.filename)): raise StorageError(StorageErrorCode.INVALID_INPUT) def content_disposition(self): if self.disposition is None: return None - return self.disposition + (f'; filename="{self.filename}"' if self.filename is not None else "") + if self.filename is None: + return self.disposition + if all(ord(c) <= 126 for c in self.filename): + return f'{self.disposition}; filename="{self.filename}"' + fallback = "".join(c if ord(c) <= 126 else "_" for c in self.filename) + return (f'{self.disposition}; filename="{fallback}"; ' + f"filename*=UTF-8''{quote(self.filename, safe='')}") def __repr__(self): return "ReadUrlOptions[details=masked]" diff --git a/src/teaql/core/value.py b/src/teaql/core/value.py index d3344ffc..535e4da9 100644 --- a/src/teaql/core/value.py +++ b/src/teaql/core/value.py @@ -17,6 +17,7 @@ class DataType(Enum): Date = auto() Time = auto() Timestamp = auto() + BlobRef = auto() @dataclass class Timestamp: @@ -85,6 +86,13 @@ def Decimal(v: Decimal) -> 'Value': def Text(v: str) -> 'Value': return Value(v) + @staticmethod + def Blob(v: 'BlobRef') -> 'Value': + from .blob_ref import BlobRef + if not isinstance(v, BlobRef): + raise ValueError("BLOB_REFERENCE_INVALID") + return Value(v.to_sql(), DataType.BlobRef) + @staticmethod def Json(v: Any) -> 'Value': return Value(v, DataType.Json) @@ -126,8 +134,11 @@ def __eq__(self, other): @classmethod def from_any(cls, value: Any) -> 'Value': + from .blob_ref import BlobRef if isinstance(value, cls): return value + if isinstance(value, BlobRef): + return cls.Blob(value) if value is None: return cls.Null() if isinstance(value, bool): diff --git a/src/teaql/provider/s3/__init__.py b/src/teaql/provider/s3/__init__.py index d852dd19..71c75cb7 100644 --- a/src/teaql/provider/s3/__init__.py +++ b/src/teaql/provider/s3/__init__.py @@ -121,6 +121,26 @@ def head(self, invocation, key): def delete(self, invocation, key): _sdk_call(lambda: self._client.delete_object(Bucket=self._bucket, Key=key.value)) + def delete_if_version(self, invocation, key, content_version: str): + if (not isinstance(content_version, str) or not content_version + or len(content_version) > 256 or content_version == "*" + or content_version.strip() != content_version + or any(ord(char) < 32 or 127 <= ord(char) <= 159 + for char in content_version)): + raise StorageError(StorageErrorCode.INVALID_INPUT) + try: + self._client.delete_object(Bucket=self._bucket, Key=key.value, + IfMatch=content_version) + return "deleted" + except Exception as error: + status = getattr(error, "response", {}).get( + "ResponseMetadata", {}).get("HTTPStatusCode") + if status in (404, 412): + return ("version_mismatch" if self.head(invocation, key) is not None + else "already_missing") + safe = _translate(error) + raise safe from None + def get_read_url(self, invocation, key, options): params = dict(Bucket=self._bucket, Key=key.value) disposition = options.content_disposition() diff --git a/src/teaql/provider/sqlite/transport.py b/src/teaql/provider/sqlite/transport.py index e35ff175..af8bce45 100644 --- a/src/teaql/provider/sqlite/transport.py +++ b/src/teaql/provider/sqlite/transport.py @@ -1,5 +1,6 @@ import json import aiosqlite +from pathlib import Path from decimal import Decimal from datetime import date, time from typing import List, Dict, Any, Optional, AsyncIterator @@ -36,6 +37,26 @@ def __init__(self, db_path: str): async def enable_soundex(self) -> None: self._soundex_enabled = True + async def shares_database(self, transport) -> bool: + """Compare SQLite's actual main file, including native file URI paths. + + Empty filenames (private or shared memory) cannot prove this provider's + independent per-operation connections share a durable database. + """ + if not isinstance(transport, SqliteTransport): + return False + + async def main_file(source): + rows = await source.fetch_all_sql(CompiledQuery("PRAGMA database_list", [])) + scalar = lambda value: getattr(value, "val", value) + files = [scalar(row.get("file")) for row in rows if scalar(row.get("name")) == "main"] + if len(files) != 1 or not isinstance(files[0], str) or not files[0]: + return None + return Path(files[0]).resolve() + + own_file = await main_file(self) + return own_file is not None and own_file == await main_file(transport) + async def _prepare(self, db: aiosqlite.Connection) -> None: if self._soundex_enabled: await db.create_function("soundex", 1, _soundex, deterministic=True) diff --git a/src/teaql/runtime/__init__.py b/src/teaql/runtime/__init__.py index e8146ed4..df17f0e9 100644 --- a/src/teaql/runtime/__init__.py +++ b/src/teaql/runtime/__init__.py @@ -8,7 +8,14 @@ Tools, ToolToken, ToolUnavailableError, ) from .module import RuntimeModule, DefaultEntityDataServiceBehavior +from .capability import install_runtime_capability, validate_capabilities from .graph_session import GraphMutationSession, GraphCommittedError +from .view_preparation import ( + AuthorizedBlobSlot, AuthorizedResponseGraph, BlobResponseContributor, + PreparedSecureDocument, PreparedView, ProjectedBlobURL, + ResponseGraphAuthorizer, SecureDocumentContributor, ViewOptions, + ViewPreparationError, ViewPreparationService, +) from .store import DataStore from .audit import RawAuditEvent, SafeAuditEvent, MutationAuditKind from .business_clock import BusinessClock, FixedBusinessClock, SystemBusinessClock @@ -66,6 +73,11 @@ __all__ = ["WireFieldMetadata", "WireEntityMetadata", "NormalizedWireInput", "WireInputError", "create_wire_entity_metadata", "normalize_wire_input", "encode_wire_output", "retain_submitted_paths", "EntityKey", "EntityChangeSet", "EntityRoot", "ContextEntityRef", "ContextRootError", "CheckException", "CheckResult", "I18nCatalog", "JsonFieldNamingProfile", "Locale", "ObjectLocation", "UnsupportedLocaleError", "UserContext", "TeaqlRuntime", "SqlLogEntry", "SqlLogOperation", "DiagnosticSqlLogSink", "TextDiagnosticSqlLogSink", "ServiceRuntimeFromEnv", "RuntimeModule", "DataStore", "RawAuditEvent", "SafeAuditEvent", "MutationAuditKind", "BusinessClock", "FixedBusinessClock", "SystemBusinessClock", "InternalIdAllocator", "AeadEntityReferenceCodec", "EntityReferenceClaims", "EntityReferenceCodec", "EntityReferenceTokenError", "ENTITY_REFERENCE_AAD", "UNSAFE_RAW_ENTITY_REFERENCES_ACKNOWLEDGEMENT", "UNSAFE_RAW_ENTITY_REFERENCES_ENVIRONMENT", "BUSINESS_ID_PERMUTATION_V1_ALPHABET", "BUSINESS_ID_PERMUTATION_V1_DOMAIN_SIZE", "BUSINESS_ID_PERMUTATION_V1_MAX_SEQUENCE", "BUSINESS_ID_PERMUTATION_V1_WIDTH", "DefaultBusinessIdProfileFactory", "DefaultBusinessIdService", "InMemoryBusinessIdAllocator", "PermutedDailyBusinessIdProfile", "StaticBusinessIdKeyProvider", "encode_business_id_permutation_v1", "ContextTools", "ExecutableHttpTool", "HTTP_TOOL", "HttpIntentPhase", "HttpTool", "HttpToolProvider", "ToolDeniedError", "ToolError", "ToolPolicy", "ToolRisk", "Tools", "ToolToken", "ToolUnavailableError"] __all__ += [ + "AuthorizedBlobSlot", "AuthorizedResponseGraph", "BlobResponseContributor", + "PreparedSecureDocument", "PreparedView", "ProjectedBlobURL", + "ResponseGraphAuthorizer", "SecureDocumentContributor", "ViewOptions", + "ViewPreparationError", "ViewPreparationService", + "install_runtime_capability", "validate_capabilities", "Checker", "CheckerRegistry", "DiagnosticTelemetryAdapter", "RequestPolicy", "RequestPolicyInput", "RuntimeTelemetryAdapter", "RuntimeTelemetryInput", "SafeAuditEventSink", "SchemaCapability", diff --git a/src/teaql/runtime/audit.py b/src/teaql/runtime/audit.py index 7e90bce1..82815d6b 100644 --- a/src/teaql/runtime/audit.py +++ b/src/teaql/runtime/audit.py @@ -30,14 +30,15 @@ class RawAuditEvent: mutation_governance: Any = None _intent_privacy: Any = field(default=None, repr=False, compare=False) - def safe(self, mask_fields: List[str], max_length: Optional[int]) -> "SafeAuditEvent": + def safe(self, mask_fields: List[str], max_length: Optional[int], + credential_fields=()) -> "SafeAuditEvent": from .log_privacy import REDACTED, credential_name, payload_has_credentials, plaintext_enabled, scrub, value_strings allow = plaintext_enabled() secrets = [] fields = [] for change in self.changes: value = None if change.new_value is None else str(getattr(change.new_value, "val", change.new_value)) - credential = (credential_name(change.field) + credential = (change.field in credential_fields or credential_name(change.field) or payload_has_credentials(change.new_value) or payload_has_credentials(change.old_value)) masked = credential or (change.field in mask_fields and not allow) diff --git a/src/teaql/runtime/capability.py b/src/teaql/runtime/capability.py new file mode 100644 index 00000000..3b833a52 --- /dev/null +++ b/src/teaql/runtime/capability.py @@ -0,0 +1,48 @@ +"""Passive optional-module resource slots and explicit readiness checks.""" + +from __future__ import annotations + +from inspect import isawaitable +from typing import Protocol, TYPE_CHECKING + +if TYPE_CHECKING: + from .context import UserContext + + +class RuntimeCapabilityResource(Protocol): + @property + def capability(self) -> str: ... + + def validate_ready(self, context: "UserContext") -> object: ... + + +def _slot(name: str) -> str: + return f"teaql.runtime.capability:{name}" + + +def install_runtime_capability(context: "UserContext", resource: RuntimeCapabilityResource) -> None: + name = getattr(resource, "capability", None) + if not isinstance(name, str) or not name or name != name.strip() \ + or not callable(getattr(resource, "validate_ready", None)): + raise ValueError("RUNTIME_CAPABILITY_INVALID") + if context.get_resource(_slot(name)) is not None: + raise ValueError("RUNTIME_CAPABILITY_DUPLICATE") + context.insert_resource(_slot(name), resource) + + +async def validate_capabilities(context: "UserContext") -> None: + for name in sorted(context._required_capabilities): + resource = context.get_resource(_slot(name)) + if resource is None or getattr(resource, "capability", None) != name: + raise RuntimeError(_missing(name)) + try: + result = resource.validate_ready(context) + if isawaitable(result): + await result + except Exception: + raise RuntimeError(_missing(name)) from None + + +def _missing(name: str) -> str: + return {"blob.v1": "BLOB_CAPABILITY_MISSING", + "secure_document.v1": "DOCUMENT_CONFIGURATION"}.get(name, "RUNTIME_CAPABILITY_MISSING") diff --git a/src/teaql/runtime/context.py b/src/teaql/runtime/context.py index f816b70a..03d2fa45 100644 --- a/src/teaql/runtime/context.py +++ b/src/teaql/runtime/context.py @@ -101,6 +101,7 @@ def invalidate(self, query_key: str) -> None: class UserContext: def __init__(self): self._resources: Dict[str, Any] = {} + self._required_capabilities: frozenset[str] = frozenset() self._resources["sql_log_options"] = SqlLogOptions.all() self._resources["diagnostic_sql_log_sink"] = TextDiagnosticSqlLogSink() self._metadata: Optional[Any] = None @@ -219,6 +220,12 @@ def decode_entity_reference( ) return claims + def require_no_active_graph_sql(self) -> None: + """Reject maintenance that would wait on its own graph SQL owner.""" + guard = self._graph_save_guard.get() + if self._graph_session is not None or (guard is not None and guard["active"]): + raise RuntimeError("BLOB_TRANSACTION_REQUIRED") + async def execute_graph_save(self, work, *, comment=None): """Run ``work(session)``; never implicitly join another root save.""" from teaql.core.request_intent import MutationIntent @@ -237,6 +244,8 @@ async def execute_graph_save(self, work, *, comment=None): lambda: self._execute_graph_transaction(work, intent, GraphMutationSession)) async def _execute_graph_transaction(self, work, intent, session_type): + from .capability import validate_capabilities + await validate_capabilities(self) provider = self.require_resource("dataService") begin = getattr(provider, "begin", None) if not callable(begin): @@ -618,6 +627,8 @@ def set_schema_provider(self, provider: SchemaCapability): self.with_schema_provider(provider) async def ensure_schema(self): + from .capability import validate_capabilities + await validate_capabilities(self) provider = self.get_resource("schema_provider") if provider: from ._schema_capability import SCHEMA_CAPABILITY @@ -786,7 +797,9 @@ async def send_audit_event(self, event: Any): descriptor = self.entity(event.entity) mask_fields = getattr(descriptor, "audit_mask_fields_val", []) if descriptor else [] max_len = getattr(descriptor, "audit_value_max_len_val", None) if descriptor else None - safe_event = event.safe(mask_fields, max_len) + credential_fields = {prop.name for prop in descriptor.properties + if getattr(prop, "blob_spec", None) is not None} if descriptor else set() + safe_event = event.safe(mask_fields, max_len, credential_fields) if self._graph_session is not None: self._graph_session.queue_audit(event, safe_event) return diff --git a/src/teaql/runtime/graph_session.py b/src/teaql/runtime/graph_session.py index af050bd3..bad77acc 100644 --- a/src/teaql/runtime/graph_session.py +++ b/src/teaql/runtime/graph_session.py @@ -49,6 +49,20 @@ def _require_active(self): if not self._active or self._committed: raise RuntimeError('graph mutation session is no longer writable') + async def with_sql_transport(self, work): + """Borrow this graph's physical SQL transaction for an optional module.""" + self._require_active() + owner = getattr(self.transaction, 'with_physical_sql', None) + if not callable(owner): + raise RuntimeError('BLOB_TRANSACTION_REQUIRED') + return await owner(work) + + def is_current_sql_lease(self, lease): + """Check an already borrowed owner lease without entering the owner again.""" + return (self._active and not self._committed and + callable(getattr(self.transaction, 'is_current_physical_sql_lease', None)) and + self.transaction.is_current_physical_sql_lease(lease)) + def scope(self, key, parent=None, local_reason=None): self._require_active() if parent is None: diff --git a/src/teaql/runtime/module.py b/src/teaql/runtime/module.py index 14cbe9d9..ca012a7e 100644 --- a/src/teaql/runtime/module.py +++ b/src/teaql/runtime/module.py @@ -11,6 +11,7 @@ def __init__(self): self._generated_bootstraps: List[Any] = [] self._schema_entities: List[Any] = [] self._wire_metadata: Dict[str, Any] = {} + self._required_capabilities: set[str] = set() @classmethod def new(cls) -> 'RuntimeModule': @@ -60,6 +61,16 @@ def wire_metadata(self, entity: str, metadata: Any) -> 'RuntimeModule': self._wire_metadata[entity] = metadata return self + def require_capability(self, name: str) -> 'RuntimeModule': + if not isinstance(name, str) or not name or name != name.strip() or any(ch.isspace() for ch in name): + raise ValueError("invalid runtime capability") + self._required_capabilities.add(name) + return self + + @property + def required_capabilities(self) -> tuple[str, ...]: + return tuple(sorted(self._required_capabilities)) + def schema_entity(self, descriptor: Any) -> 'RuntimeModule': """Register generated storage metadata without embedding a provider.""" self._schema_entities.append(descriptor) @@ -78,6 +89,7 @@ def and_module(self, other: 'RuntimeModule') -> 'RuntimeModule': ] combined._schema_entities = [*self._schema_entities, *other._schema_entities] combined._wire_metadata = {**self._wire_metadata, **other._wire_metadata} + combined._required_capabilities = self._required_capabilities | other._required_capabilities combined._initial_graphs = [ *getattr(self, '_initial_graphs', []), *getattr(other, '_initial_graphs', []), @@ -89,6 +101,7 @@ def and_module(self, other: 'RuntimeModule') -> 'RuntimeModule': return combined def apply_to(self, context: UserContext): + context._required_capabilities = frozenset(self._required_capabilities) for name, dep in self._dependencies.items(): context.insert_resource(name, dep) installed_entities = self._schema_entities or self._entities diff --git a/src/teaql/runtime/object_storage.py b/src/teaql/runtime/object_storage.py index c88492b6..f105a2fc 100644 --- a/src/teaql/runtime/object_storage.py +++ b/src/teaql/runtime/object_storage.py @@ -224,5 +224,25 @@ def exists(self, key): def delete(self, key): return self._call("delete", lambda: self._provider.delete(self._invocation, key)) + def delete_if_version(self, key, content_version: str): + if (not isinstance(content_version, str) or not content_version + or len(content_version) > 256 or content_version == "*" + or content_version.strip() != content_version + or any(ord(char) < 32 or 127 <= ord(char) <= 159 + for char in content_version)): + raise StorageError(StorageErrorCode.INVALID_INPUT) + conditional = getattr(self._provider, "delete_if_version", None) + if not callable(conditional): + raise StorageError(StorageErrorCode.UNSUPPORTED) + def delete(): + result = conditional(self._invocation, key, content_version) + if result not in ("deleted", "already_missing", "version_mismatch"): + raise StorageError(StorageErrorCode.UNAVAILABLE) + return result + return self._call("delete_if_version", delete) + + def supports_version_conditional_delete(self) -> bool: + return callable(getattr(self._provider, "delete_if_version", None)) + def get_read_url(self, key, options=ReadUrlOptions()): return self._call("read_url", lambda: self._provider.get_read_url(self._invocation, key, options)) diff --git a/src/teaql/runtime/view_preparation.py b/src/teaql/runtime/view_preparation.py new file mode 100644 index 00000000..fd61203f --- /dev/null +++ b/src/teaql/runtime/view_preparation.py @@ -0,0 +1,243 @@ +"""Detached, context-authorized view preparation after business completion.""" + +from __future__ import annotations + +from dataclasses import dataclass +from datetime import datetime, timezone +import re +from typing import Any, Protocol +from urllib.parse import urlsplit + +from teaql.core.blob_ref import BlobRef + +from .context import UserContext + + +@dataclass(frozen=True) +class ViewOptions: + mode: str # ordinary or governed; selected by the trusted host + business_outcome: str # read_only or committed + expires_at: datetime | None = None + + +@dataclass(frozen=True) +class AuthorizedBlobSlot: + path: tuple[str | int, ...] + entity: str + entity_id: str + field: str + scope: str + binding_version: int + reference: BlobRef + + +@dataclass(frozen=True) +class AuthorizedResponseGraph: + presentation: Any + blob_slots: tuple[AuthorizedBlobSlot, ...] + + +class ResponseGraphAuthorizer(Protocol): + async def authorize(self, context: UserContext, result: Any, + options: ViewOptions) -> AuthorizedResponseGraph: ... + + +@dataclass(frozen=True) +class PreparedSecureDocument: + graph: AuthorizedResponseGraph + envelope: Any + expires_at: datetime + + +class SecureDocumentContributor(Protocol): + async def prepare(self, context: UserContext, graph: AuthorizedResponseGraph, + options: ViewOptions) -> PreparedSecureDocument: ... + + +@dataclass(frozen=True) +class ProjectedBlobURL: + url: str + expires_at: datetime + + +class BlobResponseContributor(Protocol): + async def project_batch(self, context: UserContext, graph: AuthorizedResponseGraph, + expires_at: datetime | None) -> tuple[ProjectedBlobURL, ...]: ... + + +@dataclass(frozen=True) +class PreparedView: + presentation: Any + document: Any + expires_at: datetime | None + cache_control: str = "private, no-store" + + +_SAFE_CODES = frozenset({ + "VIEW_OPTIONS_INVALID", "VIEW_AUTHORIZER_MISSING", "VIEW_GRAPH_INVALID", + "DOCUMENT_CONFIGURATION", "BLOB_CAPABILITY_MISSING", "BLOB_SIGN_FAILED", + "BLOB_READ_DENIED", "BLOB_REFERENCE_EXPIRED", "BLOB_SCHEMA_MISSING", + "BLOB_CATALOG_FAILED", "DOCUMENT_PROJECTION_VIOLATION", + "ROUND_TRIP_REFERENCE_INVALID", "ROUND_TRIP_REFERENCE_EXPIRED", + "ROUND_TRIP_REFERENCE_SCOPE_MISMATCH", "ROUND_TRIP_REFERENCE_AUTHORIZATION_REQUIRED", + "ROUND_TRIP_REFERENCE_CONFIGURATION_INVALID", +}) + + +class ViewPreparationError(Exception): + def __init__(self, code: str, business_state: str = "read_only") -> None: + self.code = code + self.business_state = business_state + super().__init__(code) + + @property + def retry(self) -> str: + return "read" if self.business_state == "committed" else "prepare" + + +def _fail(code: str) -> None: + raise ViewPreparationError(code) + + +def _detached(value: Any, depth: int = 0, budget: list[int] | None = None) -> Any: + if budget is None: + budget = [100_000] + budget[0] -= 1 + if budget[0] < 0 or depth > 64: + _fail("VIEW_GRAPH_INVALID") + if value is None or type(value) in (bool, int, float, str, BlobRef, datetime): + return value + if type(value) is dict: + if len(value) > budget[0] or any(type(key) is not str for key in value): + _fail("VIEW_GRAPH_INVALID") + return {key: _detached(child, depth + 1, budget) for key, child in value.items()} + if type(value) is list: + if len(value) > budget[0]: + _fail("VIEW_GRAPH_INVALID") + return [_detached(child, depth + 1, budget) for child in value] + _fail("VIEW_GRAPH_INVALID") + + +def _path_key(path: tuple[str | int, ...]) -> tuple[tuple[str, str | int], ...]: + if not path or len(path) > 64: + _fail("VIEW_GRAPH_INVALID") + key = [] + for part in path: + if type(part) is str and part: + key.append(("s", part)) + elif type(part) is int and part >= 0: + key.append(("i", part)) + else: + _fail("VIEW_GRAPH_INVALID") + return tuple(key) + + +def _at_path(root: Any, path: tuple[str | int, ...]) -> Any: + value = root + for part in path: + if type(part) is str and type(value) is dict and part in value: + value = value[part] + elif type(part) is int and type(value) is list and 0 <= part < len(value): + value = value[part] + else: + _fail("VIEW_GRAPH_INVALID") + return value + + +def _replace(root: Any, path: tuple[str | int, ...], value: str) -> None: + _at_path(root, path) + parent = _at_path(root, path[:-1]) + parent[path[-1]] = value + + +def _contains_blob(value: Any, depth: int = 0, budget: list[int] | None = None) -> bool: + if budget is None: + budget = [100_000] + budget[0] -= 1 + if budget[0] < 0 or depth > 64: + return True + if type(value) is BlobRef: + return True + if type(value) is dict: + return any(_contains_blob(child, depth + 1, budget) for child in value.values()) + if type(value) is list: + return any(_contains_blob(child, depth + 1, budget) for child in value) + return False + + +def _earlier(left: datetime | None, right: datetime) -> datetime: + if left is None: + return right + return min(left, right) + + +def _future(value: datetime) -> bool: + return value.tzinfo is not None and value > datetime.now(timezone.utc) + + +class ViewPreparationService: + async def prepare(self, context: UserContext, result: Any, + options: ViewOptions) -> PreparedView: + state = "committed" if getattr(options, "business_outcome", None) == "committed" else "read_only" + try: + if (context is None or not isinstance(options, ViewOptions) + or options.mode not in ("ordinary", "governed") + or options.business_outcome not in ("read_only", "committed") + or (options.expires_at is not None and not _future(options.expires_at))): + _fail("VIEW_OPTIONS_INVALID") + authorizer = context.get_resource("response_graph_authorizer") + if authorizer is None: + _fail("VIEW_AUTHORIZER_MISSING") + graph = await authorizer.authorize(context, result, options) + if type(graph.blob_slots) not in (tuple, list) or len(graph.blob_slots) > 10_000: + _fail("VIEW_GRAPH_INVALID") + graph = AuthorizedResponseGraph(_detached(graph.presentation), + tuple(graph.blob_slots)) + expiry = options.expires_at + document = None + if options.mode == "governed": + secure = context.get_resource("secure_document_contributor") + if secure is None: + _fail("DOCUMENT_CONFIGURATION") + prepared = await secure.prepare(context, graph, options) + if not _future(prepared.expires_at): + _fail("DOCUMENT_CONFIGURATION") + graph, document = prepared.graph, prepared.envelope + if type(graph.blob_slots) not in (tuple, list) or len(graph.blob_slots) > 10_000: + _fail("VIEW_GRAPH_INVALID") + expiry = _earlier(expiry, prepared.expires_at) + presentation = _detached(graph.presentation) + paths: set[tuple[tuple[str, str | int], ...]] = set() + for slot in graph.blob_slots: + path = _path_key(slot.path) + if path in paths or _at_path(presentation, slot.path) != slot.reference or \ + type(_at_path(presentation, slot.path)) is not BlobRef: + _fail("VIEW_GRAPH_INVALID") + paths.add(path) + if graph.blob_slots: + contributor = context.get_resource("blob_response_contributor") + if contributor is None: + _fail("BLOB_CAPABILITY_MISSING") + ceiling = expiry + urls = await contributor.project_batch(context, graph, ceiling) + if len(urls) != len(graph.blob_slots): + _fail("BLOB_SIGN_FAILED") + for slot, signed in zip(graph.blob_slots, urls): + # urlsplit strips or normalizes raw whitespace; reject the original value. + if (not isinstance(signed.url, str) + or re.search(r"[\s\x00-\x1f\x7f-\x9f]", signed.url)): + _fail("BLOB_SIGN_FAILED") + parsed = urlsplit(signed.url) + if (parsed.scheme not in ("http", "https") or not parsed.hostname + or not _future(signed.expires_at) + or (ceiling is not None and signed.expires_at > ceiling)): + _fail("BLOB_SIGN_FAILED") + _replace(presentation, slot.path, signed.url) + expiry = _earlier(expiry, signed.expires_at) + if _contains_blob(presentation): + _fail("BLOB_CAPABILITY_MISSING") + return PreparedView(presentation, document, expiry) + except Exception as exc: + code = exc.code if isinstance(exc, ViewPreparationError) and exc.code in _SAFE_CODES \ + else "VIEW_PREPARATION_FAILED" + raise ViewPreparationError(code, state) from None diff --git a/src/teaql/sql/executor.py b/src/teaql/sql/executor.py index 28f571cd..f33291b0 100644 --- a/src/teaql/sql/executor.py +++ b/src/teaql/sql/executor.py @@ -5,6 +5,7 @@ import asyncio import hashlib import logging +from inspect import isawaitable import time import threading from array import array @@ -160,6 +161,19 @@ def __init__(self, dialect: SqlDialect, transport: SqlTransport, schema_provider # executor still needs a stable, non-recycled cache isolation identity. self._id_set_source_identity = uuid4().hex + async def uses_sql_database(self, transport) -> bool: + """Ask the native transport for resource identity; no Blob dependency.""" + shares = getattr(self.transport, "shares_database", None) + if not callable(shares): + return False + try: + result = shares(transport) + if isawaitable(result): + result = await result + return result is True + except Exception: + return False + def _backend_name(self) -> str: kind = self.dialect.kind() name = getattr(kind, 'name', str(kind)).lower() @@ -1082,6 +1096,39 @@ async def _mutate(self, context: 'UserContext', request: MutationRequest, privac except SqlCompileError as e: raise CompileError(e) + # A legacy row can predate the Blob binding table. Read its modeled Blob + # columns inside this mutation's SQL owner before an explicit update or + # delete, so a NULL slot can be handled without accepting a non-NULL + # orphan Key after the business row has changed or disappeared. + blob_fields = [prop for prop in entity_desc.properties + if getattr(prop, 'blob_spec', None) is not None] + before_blob_record = None + blob_participant = context.get_resource('blob_mutation_participant') if context is not None else None + if callable(getattr(blob_participant, 'apply_with_before', None)) and blob_fields and id_prop is not None and ( + isinstance(req_data, DeleteCommand) or + isinstance(req_data, UpdateCommand) and any( + prop.name in req_data.values for prop in blob_fields)): + columns = ", ".join( + self.dialect.quote_ident(prop.column_name_val) + if prop.column_name_val == prop.name else + f"{self.dialect.quote_ident(prop.column_name_val)} AS " + f"{self.dialect.quote_ident(prop.name)}" + for prop in blob_fields) + before_query = CompiledQuery( + f"SELECT {columns} FROM {self.dialect.quote_ident(entity_desc.table_name_val)} " + f"WHERE {self.dialect.quote_ident(id_prop.column_name_val)} = " + f"{self.dialect.placeholder(1)}", + [req_data.id], + parameter_log_policies=[self.dialect.field_log_policy(entity_desc, id_prop.name)], + sql_origin='generated') + try: + before_rows = await self.transport.fetch_all_sql(before_query) + except Exception: + raise RuntimeError('BLOB_CATALOG_FAILED') from None + if len(before_rows) > 1: + raise RuntimeError('BLOB_CATALOG_FAILED') + before_blob_record = before_rows[0] if before_rows else None + start = datetime.now() last_insert_id = None operation = {'insert': DataServiceOperation.Insert, 'update': DataServiceOperation.Update, @@ -1175,6 +1222,21 @@ async def _mutate(self, context: 'UserContext', request: MutationRequest, privac read_start, persisted_rows, None, privacy, read_scope.correlation) metadata = replace(metadata, statements=(metadata, read_metadata)) + if affected_rows > 0 and context is not None: + blob_fields = [prop for prop in entity_desc.properties + if getattr(prop, 'blob_spec', None) is not None] + if blob_fields and (isinstance(req_data, (InsertCommand, DeleteCommand, RecoverCommand)) + or isinstance(req_data, UpdateCommand) and any( + prop.name in req_data.values for prop in blob_fields)): + participant = context.get_resource('blob_mutation_participant') + if not callable(getattr(participant, 'apply', None)): + raise RuntimeError('BLOB_CAPABILITY_MISSING') + if callable(getattr(participant, 'apply_with_before', None)): + await participant.apply_with_before(context, entity_desc, req_data, + persisted_record, before_blob_record) + else: + await participant.apply(context, entity_desc, req_data, persisted_record) + if affected_rows > 0 and context is not None: from teaql.runtime.audit import AuditFieldChange, MutationAuditKind, RawAuditEvent if isinstance(req_data, InsertCommand): @@ -1390,6 +1452,32 @@ def __init__(self, dialect: SqlDialect, transport: SqlTransactionTransportTx, sc self._audit_journal = None self._audit_owner = None self._completed = False + self._rollback_only = False + self._physical_sql_active = False + + async def with_physical_sql(self, work): + """Lend this physical SQL owner to an explicit graph module callback.""" + if self._completed or self._rollback_only or self._physical_sql_active or not callable(work): + raise RuntimeError('BLOB_TRANSACTION_REQUIRED') + lease = _PhysicalSQLLease(self.transport) + self._physical_sql_active = True + try: + result = await work(lease) + if lease.failed: + self._rollback_only = True + raise RuntimeError('GRAPH_TRANSACTION_ROLLBACK_ONLY') + return result + except BaseException: + self._rollback_only = True + raise + finally: + lease.close() + self._physical_sql_active = False + + def is_current_physical_sql_lease(self, lease): + return (not self._completed and self._physical_sql_active and + isinstance(lease, _PhysicalSQLLease) and lease._active and + lease._transport is self.transport) def capabilities(self) -> DataServiceCapabilities: return DataServiceCapabilities( @@ -1410,6 +1498,8 @@ async def mutate(self, context: 'UserContext', request: MutationRequest) -> Muta request.validate() if self._completed: raise RuntimeError('SQL transaction is already completed') + if self._rollback_only: + raise RuntimeError('GRAPH_TRANSACTION_ROLLBACK_ONLY') if context is not None: context._require_mutation_invocation() if context._graph_session is None: @@ -1436,6 +1526,8 @@ async def ensure_id_floor(self, entity: str, floor: int) -> None: async def commit(self, context: 'UserContext') -> None: if self._completed: raise RuntimeError('SQL transaction is already completed') + if self._rollback_only: + raise RuntimeError('GRAPH_TRANSACTION_ROLLBACK_ONLY') await self.transport.commit_sql() self._completed = True if self._audit_journal: @@ -1446,3 +1538,31 @@ async def rollback(self, context: 'UserContext') -> None: self._audit_journal.discard() self._completed = True await self.transport.rollback_sql() + + +class _PhysicalSQLLease(SqlTransport): + def __init__(self, transport: SqlTransactionTransportTx): + self._transport = transport + self._active = True + self.failed = False + + def close(self): + self._active = False + + async def fetch_all_sql(self, query: CompiledQuery): + if not self._active: + raise RuntimeError('BLOB_TRANSACTION_REQUIRED') + try: + return await self._transport.fetch_all_sql(query) + except BaseException: + self.failed = True + raise + + async def execute_sql(self, query: CompiledQuery): + if not self._active: + raise RuntimeError('BLOB_TRANSACTION_REQUIRED') + try: + return await self._transport.execute_sql(query) + except BaseException: + self.failed = True + raise diff --git a/test-vectors/blob-value-cases.json b/test-vectors/blob-value-cases.json new file mode 100644 index 00000000..e5775309 --- /dev/null +++ b/test-vectors/blob-value-cases.json @@ -0,0 +1,45 @@ +{ + "schema": "teaql.blob-value-cases/v1", + "valid": [ + "blob_a", + "blob_AZaz09_-", + "blob_xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" + ], + "invalid": [ + "", + "blob_", + "BLOB_a", + "blob_xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", + " blob_a", + "blob_a ", + "blob_a\n", + "blob_汉", + "blob_a/b", + "blob_a.b", + "blob_a?sig=1", + "/tmp/file.pdf", + "https://example.invalid/file.pdf", + "application.docx" + ], + "invalidJson": [ + 0, + true, + {}, + [], + { + "key": "blob_a" + } + ], + "formats": { + "jpg": "image/jpeg", + "jpeg": "image/jpeg", + "png": "image/png", + "mp4": "video/mp4", + "mp3": "audio/mpeg", + "doc": "application/msword", + "docx": "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + "xls": "application/vnd.ms-excel", + "xlsx": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + "pdf": "application/pdf" + } +} diff --git a/tests/fixtures/blob-sample.jpg b/tests/fixtures/blob-sample.jpg new file mode 100644 index 00000000..fc5cfd5c Binary files /dev/null and b/tests/fixtures/blob-sample.jpg differ diff --git a/tests/fixtures/blob-sample.pdf b/tests/fixtures/blob-sample.pdf new file mode 100644 index 00000000..5fc6a302 Binary files /dev/null and b/tests/fixtures/blob-sample.pdf differ diff --git a/tests/fixtures/blob-sample.png b/tests/fixtures/blob-sample.png new file mode 100644 index 00000000..2194b1e1 Binary files /dev/null and b/tests/fixtures/blob-sample.png differ diff --git a/tests/fixtures/reference-modules/sample.doc b/tests/fixtures/reference-modules/sample.doc new file mode 100644 index 00000000..0979c3cd Binary files /dev/null and b/tests/fixtures/reference-modules/sample.doc differ diff --git a/tests/fixtures/reference-modules/sample.mp3 b/tests/fixtures/reference-modules/sample.mp3 new file mode 100644 index 00000000..99129a8c Binary files /dev/null and b/tests/fixtures/reference-modules/sample.mp3 differ diff --git a/tests/fixtures/reference-modules/sample.mp4 b/tests/fixtures/reference-modules/sample.mp4 new file mode 100644 index 00000000..40d9e28d Binary files /dev/null and b/tests/fixtures/reference-modules/sample.mp4 differ diff --git a/tests/fixtures/reference-modules/sample.xls b/tests/fixtures/reference-modules/sample.xls new file mode 100644 index 00000000..fec97cd9 Binary files /dev/null and b/tests/fixtures/reference-modules/sample.xls differ diff --git a/tests/provider/sqlite/test_blob_catalog_graph.py b/tests/provider/sqlite/test_blob_catalog_graph.py new file mode 100644 index 00000000..f5603fc3 --- /dev/null +++ b/tests/provider/sqlite/test_blob_catalog_graph.py @@ -0,0 +1,399 @@ +"""Real SQLite catalog binding belongs to the native business graph owner.""" + +import aiosqlite +import os +import pytest +from datetime import datetime, timedelta, timezone +from urllib.request import urlopen + +from teaql.core.blob_ref import BlobRef +from teaql.core.object_storage import ObjectKey, ObjectReadUrl, PutOptions, WriteMode +from teaql.core.entity import EntityKey +from teaql.core.meta import EntityDescriptor, PropertyDescriptor +from teaql.core.mutation import InsertCommand +from teaql.core.value import DataType +from teaql.provider.sqlite import SimpleSchemaProvider +from teaql.provider.sqlite.dialect import SqliteDialect +from teaql.provider.sqlite.transport import SqliteTransport +from teaql.runtime import RuntimeModule, UserContext +from teaql.runtime.view_preparation import ( + AuthorizedBlobSlot, AuthorizedResponseGraph, ViewOptions, + ViewPreparationError, ViewPreparationService, +) +from teaql.sql.executor import SqlDataServiceExecutor +from teaql_blob_reference import ( + BlobBindingChange, BlobBindingTarget, BlobRetentionPolicy, FileObjectStorage, SqliteBlobCatalog, + SqliteBlobResponseContributor, +) + + +class Authorization: + def __init__(self): + self.current_scope = "tenant-a" + self.allowed = True + + def scope(self, _context): + return self.current_scope + + async def authorize(self, _context, _target, _previous, _next, _action): + return self.allowed + + async def current_policy_on_lease(self, _context, lease, _target): + assert lease is not None + return BlobRetentionPolicy("default", "server_stream", 86400, 604800, 3600) + + +async def fixture(tmp_path): + path = str(tmp_path / "blob-catalog.db") + descriptor = EntityDescriptor("Asset").table_name("asset") + descriptor.property(PropertyDescriptor("id", DataType.I64).is_id()) + descriptor.property(PropertyDescriptor("version", DataType.I64).is_version()) + descriptor.property(PropertyDescriptor("name", DataType.Text)) + schema = SimpleSchemaProvider() + schema.register_entity(descriptor) + context = RuntimeModule.new().entity(descriptor).into_context() + context.insert_resource("dataService", SqlDataServiceExecutor( + SqliteDialect(), SqliteTransport(path), schema)) + async with aiosqlite.connect(path) as db: + await db.execute("CREATE TABLE asset (id INTEGER PRIMARY KEY, version INTEGER, name TEXT)") + await db.commit() + return context, path, SqliteBlobCatalog(SqliteTransport(path), + now_ms=lambda: 1_700_000_000_000) + + +async def all_rows(path, table): + async with aiosqlite.connect(path) as db: + async with db.execute(f"SELECT * FROM {table}") as cursor: + return await cursor.fetchall() + + +@pytest.mark.asyncio +async def test_catalog_binding_same_graph_rollback_authority_tombstone_and_rebind(tmp_path): + context, path, catalog = await fixture(tmp_path) + with pytest.raises(RuntimeError, match="BLOB_SCHEMA_MISSING"): + await catalog.validate_schema() + await catalog.ensure_schema() + await catalog.validate_schema() + reference = BlobRef("blob_python_catalog_1") + async with aiosqlite.connect(path) as db: + await db.execute("INSERT INTO teaql_blob_catalog " + "(key,scope,object_key,content_version,file_name,content_type,size_bytes,status," + "completed_at_ms,policy_profile,upload_mode,unbound_retention_seconds,effective_unref_seconds) " + "VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?)", + (reference.key, "tenant-a", "objects/1", "v1", "asset.pdf", + "application/pdf", 7, "uploaded_unbound", 1_700_000_000_000, + "default", "server_stream", 86400, 604800)) + await db.commit() + auth = Authorization() + + async def save(graph, asset_id, *, abort=False, catch_denial=False, foreign_actor=False): + command = (InsertCommand.new("Asset").value("id", asset_id) + .value("version", 1).value("name", "document")) + await graph.transaction.mutate(graph.context, + graph.request(command, graph.scope(EntityKey("Asset", asset_id)))) + change = BlobBindingChange(BlobBindingTarget("Asset", str(asset_id), "document"), + reference, 0, 1, "Create") + if catch_denial: + try: + await catalog.bind_in_graph(graph, UserContext.new() if foreign_actor else graph.context, + change, auth) + except RuntimeError: + return + else: + assert await catalog.bind_in_graph(graph, graph.context, change, auth) == 1 + if abort: + raise RuntimeError("abort graph") + + await context.execute_graph_save(lambda graph: save(graph, 1), comment="save asset") + assert len(await all_rows(path, "asset")) == len(await all_rows(path, "teaql_blob_binding")) == 1 + assert (await all_rows(path, "teaql_blob_catalog"))[0][-1] == "available" + with pytest.raises(RuntimeError, match="abort graph"): + await context.execute_graph_save(lambda graph: save(graph, 2, abort=True), comment="save asset") + assert len(await all_rows(path, "asset")) == len(await all_rows(path, "teaql_blob_binding")) == 1 + + auth.allowed = False + with pytest.raises(RuntimeError, match="GRAPH_TRANSACTION_ROLLBACK_ONLY"): + await context.execute_graph_save( + lambda graph: save(graph, 3, catch_denial=True), comment="save asset") + auth.allowed = True + with pytest.raises(RuntimeError, match="GRAPH_TRANSACTION_ROLLBACK_ONLY"): + await context.execute_graph_save( + lambda graph: save(graph, 4, catch_denial=True, foreign_actor=True), comment="save asset") + assert len(await all_rows(path, "asset")) == len(await all_rows(path, "teaql_blob_binding")) == 1 + + stale = BlobBindingChange(BlobBindingTarget("Asset", "1", "document"), reference, 0, 2, "Update") + with pytest.raises(RuntimeError, match="BLOB_BIND_DENIED"): + await context.execute_graph_save( + lambda graph: catalog.bind_in_graph(graph, graph.context, stale, auth), + comment="reject stale bind") + auth.current_scope = "tenant-b" + with pytest.raises(RuntimeError, match="BLOB_BIND_DENIED"): + await context.execute_graph_save( + lambda graph: catalog.bind_in_graph(graph, graph.context, + BlobBindingChange(BlobBindingTarget("Asset", "2", "document"), reference, 0, 1, "Create"), auth), + comment="reject wrong scope") + auth.current_scope = "tenant-a" + clear = BlobBindingChange(BlobBindingTarget("Asset", "1", "document"), None, 1, 2, "Update") + await context.execute_graph_save( + lambda graph: catalog.bind_in_graph(graph, graph.context, clear, auth), comment="clear binding") + assert (await all_rows(path, "teaql_blob_binding"))[0][-2:] == (None, 2) + assert (await all_rows(path, "teaql_blob_catalog"))[0][-1] == "available" + rebind = BlobBindingChange(BlobBindingTarget("Asset", "1", "document"), reference, 2, 3, "Update") + await context.execute_graph_save( + lambda graph: catalog.bind_in_graph(graph, graph.context, rebind, auth), comment="rebind") + assert (await all_rows(path, "teaql_blob_binding"))[0][-2:] == (reference.key, 3) + assert (await all_rows(path, "teaql_blob_catalog"))[0][-1] == "available" + await context.execute_graph_save(lambda graph: save(graph, 2), comment="bind second asset") + clear = BlobBindingChange(BlobBindingTarget("Asset", "1", "document"), None, 3, 4, "Update") + await context.execute_graph_save( + lambda graph: catalog.bind_in_graph(graph, graph.context, clear, auth), comment="clear first binding") + assert (await all_rows(path, "teaql_blob_catalog"))[0][-1] == "available" + clear = BlobBindingChange(BlobBindingTarget("Asset", "2", "document"), None, 1, 2, "Update") + await context.execute_graph_save( + lambda graph: catalog.bind_in_graph(graph, graph.context, clear, auth), comment="clear final binding") + assert (await all_rows(path, "teaql_blob_catalog"))[0][-1] == "available" + + +@pytest.mark.asyncio +async def test_catalog_rejects_old_not_null_binding_schema(tmp_path): + _, path, catalog = await fixture(tmp_path) + async with aiosqlite.connect(path) as db: + await db.execute("CREATE TABLE teaql_blob_binding (scope TEXT NOT NULL, entity TEXT NOT NULL, " + "entity_id TEXT NOT NULL, field TEXT NOT NULL, key TEXT NOT NULL, " + "version BIGINT NOT NULL, PRIMARY KEY (scope,entity,entity_id,field))") + await db.commit() + await catalog.ensure_schema() + with pytest.raises(RuntimeError, match="BLOB_SCHEMA_MISSING"): + await catalog.validate_schema() + + +@pytest.mark.asyncio +async def test_binding_enforces_retention_deadline_and_longest_accepted_floor(tmp_path): + context, path, catalog = await fixture(tmp_path) + await catalog.ensure_schema() + clock = [1_700_000_000_000] + catalog.now_ms = lambda: clock[0] + expired, live = BlobRef("blob_expired_policy"), BlobRef("blob_live_policy") + async with aiosqlite.connect(path) as db: + for reference, completed in ((expired, clock[0] - 86_400_000), + (live, clock[0] - 1000)): + await db.execute("INSERT INTO teaql_blob_catalog " + "(key,scope,object_key,content_version,file_name,content_type,size_bytes,status," + "completed_at_ms,policy_profile,upload_mode,unbound_retention_seconds,effective_unref_seconds) " + "VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?)", + (reference.key, "tenant-a", "objects/" + reference.key, "v1", "report.pdf", + "application/pdf", 7, "uploaded_unbound", completed, + "default", "server_stream", 86400, 3600)) + await db.commit() + + class ChangingAuthorization(Authorization): + def __init__(self): + super().__init__() + self.policy = BlobRetentionPolicy("default", "server_stream", 86400, 604800, 3600) + + async def current_policy_on_lease(self, _context, lease, _target): + assert lease is not None + return self.policy + + authority = ChangingAuthorization() + first = BlobBindingTarget("Asset", "1", "document") + second = BlobBindingTarget("Asset", "2", "document") + + async def change(target, next_ref, before, after, action): + return await context.execute_graph_save( + lambda graph: catalog.bind_in_graph(graph, graph.context, + BlobBindingChange(target, next_ref, before, after, action), authority), + comment="retention binding") + + with pytest.raises(RuntimeError, match="BLOB_REFERENCE_EXPIRED"): + await change(first, expired, 0, 1, "Create") + await change(first, live, 0, 1, "Create") + authority.policy = BlobRetentionPolicy("longer", "server_stream", 86400, 2592000, 2678400) + await change(second, live, 0, 1, "Create") + await change(first, None, 1, 2, "Update") + async with aiosqlite.connect(path) as db: + async with db.execute("SELECT effective_unref_seconds,retention_floor_until_ms," + "last_unbound_at_ms FROM teaql_blob_catalog WHERE key=?", + (live.key,)) as cursor: + seconds, floor, last_unbound = await cursor.fetchone() + assert seconds == 2592000 and floor == clock[0] + 2_678_400_000 + assert last_unbound is None + await change(second, None, 1, 2, "Update") + async with aiosqlite.connect(path) as db: + async with db.execute("SELECT status,last_unbound_at_ms FROM teaql_blob_catalog WHERE key=?", + (live.key,)) as cursor: + assert await cursor.fetchone() == ("available", clock[0]) + authority.policy = BlobRetentionPolicy("shorter", "server_stream", 86400, 3600, 0) + clock[0] = floor - 1 + await change(first, live, 2, 3, "Update") + await change(first, None, 3, 4, "Update") + clock[0] = floor + # The accepted maximum still protects this object beyond the old floor. + await change(first, live, 4, 5, "Update") + await change(first, None, 5, 6, "Update") + clock[0] += 2_592_000_000 + with pytest.raises(RuntimeError, match="BLOB_REFERENCE_EXPIRED"): + await change(first, live, 6, 7, "Update") + + +@pytest.mark.asyncio +async def test_new_binding_rejects_legacy_unknown_policy_row(tmp_path): + context, path, catalog = await fixture(tmp_path) + await catalog.ensure_schema() + reference = BlobRef("blob_legacy_unknown") + async with aiosqlite.connect(path) as db: + await db.execute("INSERT INTO teaql_blob_catalog " + "(key,scope,object_key,content_version,file_name,content_type,size_bytes,status) " + "VALUES (?,?,?,?,?,?,?,?)", + (reference.key, "tenant-a", "objects/legacy", "v1", "legacy.pdf", + "application/pdf", 5, "uploaded_unbound")) + await db.commit() + with pytest.raises(RuntimeError, match="^BLOB_BIND_DENIED$"): + await context.execute_graph_save( + lambda graph: catalog.bind_in_graph(graph, graph.context, + BlobBindingChange(BlobBindingTarget("Asset", "1", "document"), + reference, 0, 1, "Create"), Authorization()), + comment="deny legacy unknown policy") + assert await all_rows(path, "teaql_blob_binding") == [] + + +@pytest.mark.asyncio +async def test_current_sqlite_binding_is_reread_before_view_url_signing(tmp_path): + context, path, catalog = await fixture(tmp_path) + await catalog.ensure_schema() + reference = BlobRef("blob_python_read_1") + async with aiosqlite.connect(path) as db: + await db.execute("INSERT INTO teaql_blob_catalog " + "(key,scope,object_key,content_version,file_name,content_type,size_bytes,status," + "completed_at_ms,last_unbound_at_ms,policy_profile,upload_mode," + "unbound_retention_seconds,effective_unref_seconds) " + "VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?)", + (reference.key, "tenant-a", "objects/read-1", "v1", "report.pdf", + "application/pdf", 7, "available", 1_700_000_000_000, + 1_700_000_000_000, "default", "server_stream", 86400, 604800)) + await db.commit() + auth = Authorization() + change = BlobBindingChange(BlobBindingTarget("Asset", "1", "document"), reference, 0, 1, "Create") + await context.execute_graph_save( + lambda graph: catalog.bind_in_graph(graph, graph.context, change, auth), comment="bind read fixture") + slot = AuthorizedBlobSlot(("document",), "Asset", "1", "document", "tenant-a", 1, reference) + record = await catalog.current_read(slot) + assert (record.object_key, record.content_version, record.size_bytes) == ("objects/read-1", "v1", 7) + for invalid in ( + AuthorizedBlobSlot(slot.path, slot.entity, slot.entity_id, slot.field, + slot.scope, 2, slot.reference), + AuthorizedBlobSlot(slot.path, slot.entity, slot.entity_id, slot.field, + "tenant-b", 1, slot.reference), + AuthorizedBlobSlot(slot.path, slot.entity, slot.entity_id, slot.field, + slot.scope, 1, BlobRef("blob_old")), + ): + with pytest.raises(RuntimeError, match="BLOB_READ_DENIED"): + await catalog.current_read(invalid) + + class ReadAuthorization: + allowed = True + + async def authorize(self, _context, _slot): + return self.allowed + + class Authorizer: + async def authorize(self, _context, _result, _options): + return AuthorizedResponseGraph({"document": reference}, (slot,)) + + class SigningStore: + calls = 0 + key = None + header = None + + def get_read_url(self, _invocation, key, options): + self.calls += 1 + self.key = key.value + self.header = options.content_disposition() + return ObjectReadUrl("https://example.invalid/opaque-ticket", + datetime.now(timezone.utc) + timedelta(seconds=30)) + + read_authorization = ReadAuthorization() + store = SigningStore() + context.with_object_storage(store) + context.insert_resource("response_graph_authorizer", Authorizer()) + context.insert_resource("blob_response_contributor", + SqliteBlobResponseContributor(catalog, read_authorization)) + view = await ViewPreparationService().prepare(context, None, + ViewOptions("ordinary", "read_only", datetime.now(timezone.utc) + timedelta(seconds=60))) + assert view.presentation == {"document": "https://example.invalid/opaque-ticket"} + assert (store.calls, store.key, store.header) == \ + (1, "objects/read-1", 'inline; filename="report.pdf"') + + read_authorization.allowed = False + with pytest.raises(ViewPreparationError) as denied: + await ViewPreparationService().prepare(context, None, + ViewOptions("ordinary", "read_only", datetime.now(timezone.utc) + timedelta(seconds=60))) + assert denied.value.code == "BLOB_READ_DENIED" + assert store.calls == 1 + + async with aiosqlite.connect(path) as db: + await db.execute("UPDATE teaql_blob_catalog SET status = 'deleted' WHERE key = ?", (reference.key,)) + await db.commit() + with pytest.raises(RuntimeError, match="BLOB_READ_DENIED"): + await catalog.current_read(slot) + + +@pytest.mark.asyncio +async def test_authorized_sqlite_view_projects_real_file_http_bytes_and_denies_revoked_refresh(tmp_path): + context, path, catalog = await fixture(tmp_path) + await catalog.ensure_schema() + provider = FileObjectStorage(tmp_path / "private", os.urandom(32)) + try: + assert provider.start().startswith("http://127.0.0.1:") + context.with_object_storage(provider) + key = ObjectKey("objects/read-real") + data = b"%PDF-1x" + context.object_storage("publish current Blob").put_bytes( + key, data, PutOptions(len(data), WriteMode.CREATE_ONLY, "application/pdf")) + reference = BlobRef("blob_python_real_1") + async with aiosqlite.connect(path) as db: + await db.execute("INSERT INTO teaql_blob_catalog " + "(key,scope,object_key,content_version,file_name,content_type,size_bytes,status," + "completed_at_ms,policy_profile,upload_mode,unbound_retention_seconds,effective_unref_seconds) " + "VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?)", + (reference.key, "tenant-a", key.value, "v1", "report.pdf", + "application/pdf", len(data), "uploaded_unbound", 1_700_000_000_000, + "default", "server_stream", 86400, 604800)) + await db.commit() + change = BlobBindingChange(BlobBindingTarget("Asset", "1", "document"), + reference, 0, 1, "Create") + await context.execute_graph_save( + lambda graph: catalog.bind_in_graph(graph, graph.context, change, Authorization()), + comment="bind real file") + slot = AuthorizedBlobSlot(("document",), "Asset", "1", "document", "tenant-a", 1, reference) + + class ReadAuthorization: + allowed = True + + async def authorize(self, _context, _slot): + return self.allowed + + class Authorizer: + async def authorize(self, _context, _result, _options): + return AuthorizedResponseGraph({"document": reference}, (slot,)) + + read_permission = ReadAuthorization() + context.insert_resource("response_graph_authorizer", Authorizer()) + context.insert_resource("blob_response_contributor", + SqliteBlobResponseContributor(catalog, read_permission)) + options = ViewOptions("ordinary", "read_only", + datetime.now(timezone.utc) + timedelta(seconds=60)) + view = await ViewPreparationService().prepare(context, None, options) + link = view.presentation["document"] + assert link.startswith("http://127.0.0.1:") and key.value not in link + with urlopen(link, timeout=5) as response: + assert response.status == 200 + assert response.read() == data + assert response.headers["Content-Type"] == "application/pdf" + assert response.headers["Content-Disposition"] == 'inline; filename="report.pdf"' + read_permission.allowed = False + with pytest.raises(ViewPreparationError) as denied: + await ViewPreparationService().prepare(context, None, options) + assert denied.value.code == "BLOB_READ_DENIED" + finally: + provider.close() diff --git a/tests/provider/sqlite/test_blob_content_composite.py b/tests/provider/sqlite/test_blob_content_composite.py new file mode 100644 index 00000000..8b779ed3 --- /dev/null +++ b/tests/provider/sqlite/test_blob_content_composite.py @@ -0,0 +1,43 @@ +"""One upload verifier dispatches real optional image and media parsers.""" + +from hashlib import sha256 +from io import BytesIO +from pathlib import Path +from shutil import which + +import pytest + +from teaql_blob_reference import CompositeBlobContentVerifier +from teaql_blob_reference.content_ffmpeg import FfmpegBlobContentVerifier +from teaql_blob_reference.content_pillow import PillowBlobContentVerifier + + +class Unreadable: + def read(self, *_): + raise AssertionError("unregistered route read source") + + +def test_real_image_and_media_share_trusted_upload_verifier(): + ffprobe, ffmpeg = which("ffprobe"), which("ffmpeg") + if not ffprobe or not ffmpeg: + pytest.skip("host FFmpeg tools required") + image = PillowBlobContentVerifier() + media = FfmpegBlobContentVerifier(ffprobe, ffmpeg) + routes = {"jpg": image, "jpeg": image, "png": image, "mp3": media, "mp4": media} + verifier = CompositeBlobContentVerifier(routes) + routes.clear() + assert verifier.supported_extensions == {"jpg", "jpeg", "png", "mp3", "mp4"} + verifier.validate_ready() + fixture = Path(__file__).resolve().parents[2] / "fixtures" + for extension, path, mime in ( + ("jpg", fixture / "blob-sample.jpg", "image/jpeg"), + ("mp3", fixture / "reference-modules" / "sample.mp3", "audio/mpeg"), + ): + data = path.read_bytes() + result = verifier.verify(BytesIO(data), extension, len(data)) + assert (result.size_bytes, result.sha256, result.content_type) == ( + len(data), sha256(data).hexdigest(), mime) + with pytest.raises(RuntimeError, match="BLOB_FORMAT_NOT_ALLOWED"): + verifier.verify(Unreadable(), "docx", 1024) + with pytest.raises(RuntimeError, match="BLOB_POLICY_INVALID"): + CompositeBlobContentVerifier({"PDF": image}) diff --git a/tests/provider/sqlite/test_blob_content_ffmpeg.py b/tests/provider/sqlite/test_blob_content_ffmpeg.py new file mode 100644 index 00000000..7dfa5326 --- /dev/null +++ b/tests/provider/sqlite/test_blob_content_ffmpeg.py @@ -0,0 +1,156 @@ +"""Real MP3/MP4 decode and SQLite/File upload promotion boundary.""" + +from hashlib import sha256 +from io import BytesIO +from pathlib import Path +from shutil import which +from subprocess import DEVNULL, PIPE, run +from threading import Event, Thread +import os + +import aiosqlite +import pytest + +from teaql.core.blob_ref import BlobFieldSpec +from teaql.provider.sqlite.transport import SqliteTransport +from teaql.runtime import RuntimeModule +from teaql_blob_reference import ( + BlobUploadInput, BlobUploadTarget, FileObjectStorage, SqliteBlobCatalog, + SqliteBlobUploadService, +) +from teaql_blob_reference.content_ffmpeg import FfmpegBlobContentVerifier + + +_SHARED_FIXTURE_SHA256 = { + "mp3": "a4facfc9e36b342c3f7b2de8283ab9f4c9cf53af2dfb2f1959825aa930748d9b", + "mp4": "5a3665235df157cb9d1cffe53501b86889849a38a5a643f0a64dbc957f4a58af", +} + + +def media_fixture(extension): + # Copies of the shared reference files keep this repository's tests standalone. + root = Path(__file__).resolve().parents[2] / "fixtures" / "reference-modules" + data = (root / f"sample.{extension}").read_bytes() + assert sha256(data).hexdigest() == _SHARED_FIXTURE_SHA256[extension] + return data + + +@pytest.fixture +def verifier(): + ffprobe, ffmpeg = which("ffprobe"), which("ffmpeg") + if not ffprobe or not ffmpeg: + pytest.skip("host FFmpeg command-line tools required") + parser = FfmpegBlobContentVerifier(ffprobe, ffmpeg) + parser.validate_ready() + return parser + + +@pytest.mark.parametrize("extension,mime", [("mp3", "audio/mpeg"), ("mp4", "video/mp4")]) +def test_real_media_decode_container_and_limits(verifier, extension, mime): + data = media_fixture(extension) + checked = verifier.verify(BytesIO(data), extension, len(data)) + assert (checked.size_bytes, checked.sha256, checked.content_type) == ( + len(data), sha256(data).hexdigest(), mime) + for invalid in (data[:-1], data[:-12], data[:len(data) // 2], b"", b"ID3\x04\x00\x00\x00\x00\x00\x00", + b"\x00\x00\x00\x20ftypisom" + b"\x00" * 24): + with pytest.raises(RuntimeError, match="BLOB_FORMAT_NOT_ALLOWED"): + verifier.verify(BytesIO(invalid), extension, len(data)) + with pytest.raises(RuntimeError, match="BLOB_FORMAT_NOT_ALLOWED"): + verifier.verify(BytesIO(data), "pdf", len(data)) + with pytest.raises(RuntimeError, match="BLOB_SIZE_EXCEEDED"): + verifier.verify(BytesIO(data), extension, len(data) - 1) + if extension == "mp3": + with pytest.raises(RuntimeError, match="BLOB_FORMAT_NOT_ALLOWED"): + FfmpegBlobContentVerifier(verifier.ffprobe_path, verifier.ffmpeg_path, + maximum_frames=1).verify(BytesIO(data), extension, len(data)) + else: + with pytest.raises(RuntimeError, match="BLOB_FORMAT_NOT_ALLOWED"): + FfmpegBlobContentVerifier(verifier.ffprobe_path, verifier.ffmpeg_path, + maximum_video_pixels=1).verify(BytesIO(data), extension, len(data)) + + +def test_concurrent_admission_releases_slot(verifier): + entered, release = Event(), Event() + data = media_fixture("mp3") + + class PausedInput(BytesIO): + def read(self, size=-1): + entered.set() + assert release.wait(5) + return super().read(size) + + parser = FfmpegBlobContentVerifier(verifier.ffprobe_path, verifier.ffmpeg_path, + concurrent_verifications=1) + result = [] + first = Thread(target=lambda: result.append(parser.verify(PausedInput(data), "mp3", len(data)))) + first.start() + try: + assert entered.wait(5) + with pytest.raises(RuntimeError, match="BLOB_VERIFICATION_BUSY"): + parser.verify(BytesIO(data), "mp3", len(data)) + finally: + release.set() + first.join(5) + assert not first.is_alive() + assert result[0].content_type == "audio/mpeg" + + +def test_audio_only_mp4_is_not_a_video(verifier): + # Remux the shared MP3 into a structurally valid MP4 with no video track. + media = run((verifier.ffmpeg_path, "-nostdin", "-v", "error", "-f", "mp3", + "-i", "pipe:0", "-vn", "-c:a", "copy", "-movflags", + "frag_keyframe+empty_moov", "-f", "mp4", "pipe:1"), + input=media_fixture("mp3"), stdout=PIPE, stderr=DEVNULL, + timeout=10, check=True).stdout + with pytest.raises(RuntimeError, match="BLOB_FORMAT_NOT_ALLOWED"): + verifier.verify(BytesIO(media), "mp4", len(media)) + + +class MediaAuthority: + def scope(self, _context): + return "tenant-a" + + def field(self, _context, _target): + return BlobFieldSpec("Asset", "media", ["mp3", "mp4"], True, "default") + + async def authorize(self, _context, target, action, _input): + return target.entity == "Asset" and target.field == "media" and action in ( + "Begin", "Write", "Confirm") + + +@pytest.mark.asyncio +async def test_real_media_upload_promotes_and_truncated_copy_stays_initiated(tmp_path, verifier): + database = str(tmp_path / "media.db") + catalog = SqliteBlobCatalog(SqliteTransport(database)) + provider = FileObjectStorage(tmp_path / "objects", os.urandom(32)) + actor = RuntimeModule.new().into_context().with_object_storage(provider) + actor.with_object_storage_observer(lambda event: None) + service = SqliteBlobUploadService(catalog, MediaAuthority(), verifier, 100_000) + target = BlobUploadTarget("Asset", "1", "media") + try: + await catalog.ensure_schema() + for extension, mime in (("mp3", "audio/mpeg"), ("mp4", "video/mp4")): + good = media_fixture(extension) + upload = await service.begin(actor, target, + BlobUploadInput("valid." + extension, len(good)), + "media-good-" + extension) + assert await service.write(actor, upload.upload_id, BytesIO(good)) == len(good) + assert await service.complete(actor, upload.upload_id) == upload.reference + async with aiosqlite.connect(database) as db: + async with db.execute( + "SELECT status,content_type,size_bytes FROM teaql_blob_catalog WHERE key=?", + (upload.reference.key,)) as cursor: + assert await cursor.fetchone() == ("uploaded_unbound", mime, len(good)) + bad = good[:-12] + rejected = await service.begin(actor, target, + BlobUploadInput("broken." + extension, len(bad)), + "media-bad-" + extension) + await service.write(actor, rejected.upload_id, BytesIO(bad)) + with pytest.raises(RuntimeError, match="BLOB_FORMAT_NOT_ALLOWED"): + await service.complete(actor, rejected.upload_id) + async with aiosqlite.connect(database) as db: + async with db.execute("SELECT status FROM teaql_blob_catalog WHERE key=?", + (rejected.reference.key,)) as cursor: + assert (await cursor.fetchone())[0] == "initiated" + finally: + provider.close() diff --git a/tests/provider/sqlite/test_blob_content_legacy_office.py b/tests/provider/sqlite/test_blob_content_legacy_office.py new file mode 100644 index 00000000..05570262 --- /dev/null +++ b/tests/provider/sqlite/test_blob_content_legacy_office.py @@ -0,0 +1,161 @@ +"""Sandboxed binary DOC/XLS parsing and real SQLite/File promotion.""" + +from hashlib import sha256 +from io import BytesIO +from pathlib import Path +from shutil import which +from threading import Event, Thread +import os +import sys + +import aiosqlite +import pytest + +pytest.importorskip("docx") +pytest.importorskip("openpyxl") + +from teaql.core.blob_ref import BlobFieldSpec +from teaql.provider.sqlite.transport import SqliteTransport +from teaql.runtime import RuntimeModule +from teaql_blob_reference import (BlobUploadInput, BlobUploadTarget, FileObjectStorage, + SqliteBlobCatalog, SqliteBlobUploadService) +from teaql_blob_reference.content_composite import CompositeBlobContentVerifier +from teaql_blob_reference.content_legacy_office import LegacyOfficeBlobContentVerifier + + +TOOLS = tuple(which(name) for name in ("soffice", "bwrap", "prlimit")) +pytestmark = pytest.mark.skipif(sys.platform != "linux" or not all(TOOLS), + reason="Linux LibreOffice, bubblewrap and prlimit required") +FIXTURES = Path(__file__).parents[2] / "fixtures" / "reference-modules" + + +def verifier(tmp_path, **overrides): + scratch = tmp_path / "scratch" + scratch.mkdir(mode=0o700) + scratch.chmod(0o700) + return LegacyOfficeBlobContentVerifier(*TOOLS, scratch, + maximum_input_bytes=100_000, + maximum_converted_bytes=overrides.get("output", 100_000), + concurrent_verifications=1), scratch + + +@pytest.mark.parametrize("extension,mime,digest", [ + ("doc", "application/msword", "9fca8d262d2aa905d58b1cc1c8914d1a8069c16c340f24cd3c4a77daff9c8b73"), + ("xls", "application/vnd.ms-excel", "7f387858aaa3ed39d3c7c692aa865b36ac6620462714f3d0daa96388424cf75d"), +]) +def test_legacy_office_original_facts_and_structural_rejection(tmp_path, extension, mime, digest): + parser, scratch = verifier(tmp_path) + parser.validate_ready() + data = (FIXTURES / ("sample." + extension)).read_bytes() + assert sha256(data).hexdigest() == digest + result = parser.verify(BytesIO(data), extension, len(data)) + assert (result.size_bytes, result.sha256, result.content_type) == (len(data), digest, mime) + for damaged in (data[:-12], data[:512], bytes([data[0] ^ 1]) + data[1:]): + with pytest.raises(RuntimeError, match="BLOB_FORMAT_NOT_ALLOWED"): + parser.verify(BytesIO(damaged), extension, len(data)) + with pytest.raises(RuntimeError, match="BLOB_SIZE_EXCEEDED"): + parser.verify(BytesIO(data), extension, len(data) - 1) + assert list(scratch.iterdir()) == [] + + +def test_legacy_office_limits_busy_readiness_and_composite(tmp_path): + parser, scratch = verifier(tmp_path) + data = (FIXTURES / "sample.doc").read_bytes() + limited = LegacyOfficeBlobContentVerifier(*TOOLS, scratch, + maximum_input_bytes=100_000, + maximum_converted_bytes=1, + concurrent_verifications=1) + with pytest.raises(RuntimeError, match="BLOB_FORMAT_NOT_ALLOWED"): + limited.verify(BytesIO(data), "doc", len(data)) + entered, release = Event(), Event() + + class PausedInput(BytesIO): + def read(self, size=-1): + entered.set() + assert release.wait(10) + return super().read(size) + + outcome = [] + first = Thread(target=lambda: outcome.append(parser.verify(PausedInput(data), "doc", len(data)))) + first.start() + try: + assert entered.wait(10) + with pytest.raises(RuntimeError, match="BLOB_VERIFICATION_BUSY"): + parser.verify(BytesIO(data), "doc", len(data)) + finally: + release.set() + first.join(15) + assert not first.is_alive() and outcome[0].content_type == "application/msword" + missing = LegacyOfficeBlobContentVerifier(str(scratch / "missing"), TOOLS[1], TOOLS[2], scratch) + with pytest.raises(RuntimeError, match="BLOB_CAPABILITY_MISSING"): + missing.validate_ready() + scratch.chmod(0o755) + with pytest.raises(RuntimeError, match="BLOB_CAPABILITY_MISSING"): + parser.validate_ready() + scratch.chmod(0o700) + class ForbiddenInput(BytesIO): + def read(self, size=-1): + raise AssertionError("wrong extension consumed source") + with pytest.raises(RuntimeError, match="BLOB_FORMAT_NOT_ALLOWED"): + parser.verify(ForbiddenInput(data), "pdf", len(data)) + composite = CompositeBlobContentVerifier({"doc": parser, "xls": parser}) + composite.validate_ready() + assert composite.verify(BytesIO(data), "doc", len(data)).size_bytes == len(data) + + +class LegacyOfficeAuthority: + def scope(self, _context): + return "tenant-a" + + def field(self, _context, _target): + return BlobFieldSpec("Asset", "document", ["doc", "xls"], True, "default") + + async def authorize(self, _context, target, action, _input): + return target.entity == "Asset" and target.field == "document" and action in ( + "Begin", "Write", "Confirm") + + +@pytest.mark.asyncio +async def test_real_legacy_office_upload_promotes_and_rejects(tmp_path): + parser, _ = verifier(tmp_path) + path = str(tmp_path / "office.db") + catalog = SqliteBlobCatalog(SqliteTransport(path)) + provider = FileObjectStorage(tmp_path / "objects", os.urandom(32)) + actor = RuntimeModule.new().into_context().with_object_storage(provider) + actor.with_object_storage_observer(lambda event: None) + service = SqliteBlobUploadService(catalog, LegacyOfficeAuthority(), parser, 100_000) + target = BlobUploadTarget("Asset", "1", "document") + try: + await catalog.ensure_schema() + for extension, mime in (("doc", "application/msword"), + ("xls", "application/vnd.ms-excel")): + data = (FIXTURES / ("sample." + extension)).read_bytes() + good = await service.begin(actor, target, + BlobUploadInput("sample." + extension, len(data)), + "legacy-good-" + extension) + assert await service.write(actor, good.upload_id, BytesIO(data)) == len(data) + assert await service.complete(actor, good.upload_id) == good.reference + async with aiosqlite.connect(path) as db: + async with db.execute( + "SELECT status,content_type,size_bytes FROM teaql_blob_catalog WHERE key=?", + (good.reference.key,)) as cursor: + status, actual_mime, size = await cursor.fetchone() + async with db.execute( + "SELECT written_sha256 FROM teaql_blob_upload WHERE upload_id=?", + (good.upload_id,)) as cursor: + digest = (await cursor.fetchone())[0] + assert (status, actual_mime, size, digest) == ( + "uploaded_unbound", mime, len(data), sha256(data).hexdigest()) + broken = data[:-12] + bad = await service.begin(actor, target, + BlobUploadInput("broken." + extension, len(broken)), + "legacy-bad-" + extension) + await service.write(actor, bad.upload_id, BytesIO(broken)) + with pytest.raises(RuntimeError, match="BLOB_FORMAT_NOT_ALLOWED"): + await service.complete(actor, bad.upload_id) + async with aiosqlite.connect(path) as db: + async with db.execute("SELECT status FROM teaql_blob_catalog WHERE key=?", + (bad.reference.key,)) as cursor: + assert (await cursor.fetchone())[0] == "initiated" + finally: + provider.close() diff --git a/tests/provider/sqlite/test_blob_content_office.py b/tests/provider/sqlite/test_blob_content_office.py new file mode 100644 index 00000000..2322e888 --- /dev/null +++ b/tests/provider/sqlite/test_blob_content_office.py @@ -0,0 +1,159 @@ +"""Actual DOCX/XLSX parsers and SQLite/File upload promotion boundary.""" + +from hashlib import sha256 +from io import BytesIO +from threading import Event, Thread +from zipfile import ZIP_DEFLATED, ZipFile +import os + +import aiosqlite +import pytest + +docx = pytest.importorskip("docx") +openpyxl = pytest.importorskip("openpyxl") + +from teaql.core.blob_ref import BlobFieldSpec +from teaql.provider.sqlite.transport import SqliteTransport +from teaql.runtime import RuntimeModule +from teaql_blob_reference import ( + BlobUploadInput, BlobUploadTarget, FileObjectStorage, SqliteBlobCatalog, + SqliteBlobUploadService, +) +from teaql_blob_reference.content_office import OfficeBlobContentVerifier + + +def office_fixture(extension): + output = BytesIO() + if extension == "docx": + document = docx.Document() + document.add_paragraph("TeaQL Office parser") + document.save(output) + else: + workbook = openpyxl.Workbook() + workbook.active["A1"] = "TeaQL Office parser" + workbook.active["B1"] = 42 + workbook.save(output) + return output.getvalue() + + +def replace_part(data, name, replacement): + output = BytesIO() + with ZipFile(BytesIO(data)) as source, ZipFile(output, "w", ZIP_DEFLATED) as target: + for part in source.infolist(): + target.writestr(part.filename, replacement if part.filename == name + else source.read(part.filename)) + return output.getvalue() + + +@pytest.mark.parametrize("extension,mime", [ + ("docx", "application/vnd.openxmlformats-officedocument.wordprocessingml.document"), + ("xlsx", "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"), +]) +def test_real_office_structure_content_and_limits(extension, mime): + data = office_fixture(extension) + verifier = OfficeBlobContentVerifier() + verifier.validate_ready() + checked = verifier.verify(BytesIO(data), extension, len(data)) + assert (checked.size_bytes, checked.sha256, checked.content_type) == ( + len(data), sha256(data).hexdigest(), mime) + main = "word/document.xml" if extension == "docx" else "xl/workbook.xml" + wrong_type = replace_part(data, "[Content_Types].xml", + b"") + wrong_relationship = replace_part(data, "_rels/.rels", + b"") + entity_document = replace_part(data, main, + b"]>&x;") + for invalid in (data[:-12], b"PK\x03\x04not Office", b"", + replace_part(data, main, b""), wrong_type, + wrong_relationship, entity_document): + with pytest.raises(RuntimeError, match="BLOB_FORMAT_NOT_ALLOWED"): + verifier.verify(BytesIO(invalid), extension, len(data) + 10000) + with pytest.raises(RuntimeError, match="BLOB_FORMAT_NOT_ALLOWED"): + verifier.verify(BytesIO(data), "pdf", len(data)) + with pytest.raises(RuntimeError, match="BLOB_SIZE_EXCEEDED"): + verifier.verify(BytesIO(data), extension, len(data) - 1) + with pytest.raises(RuntimeError, match="BLOB_FORMAT_NOT_ALLOWED"): + OfficeBlobContentVerifier(maximum_decoded_bytes=1024).verify( + BytesIO(data), extension, len(data)) + with pytest.raises(RuntimeError, match="BLOB_FORMAT_NOT_ALLOWED"): + OfficeBlobContentVerifier(maximum_elements=1).verify( + BytesIO(data), extension, len(data)) + + +def test_concurrent_admission_releases_slot(): + entered, release = Event(), Event() + data = office_fixture("xlsx") + + class PausedInput(BytesIO): + def read(self, size=-1): + entered.set() + assert release.wait(5) + return super().read(size) + + verifier = OfficeBlobContentVerifier(concurrent_verifications=1) + outcome = [] + first = Thread(target=lambda: outcome.append(verifier.verify(PausedInput(data), "xlsx", 100_000))) + first.start() + try: + assert entered.wait(5) + with pytest.raises(RuntimeError, match="BLOB_VERIFICATION_BUSY"): + verifier.verify(BytesIO(data), "xlsx", 100_000) + finally: + release.set() + first.join(5) + assert not first.is_alive() + assert outcome[0].content_type.endswith("sheet") + assert verifier.verify(BytesIO(data), "xlsx", 100_000).size_bytes == len(data) + + +class OfficeAuthority: + def scope(self, _context): + return "tenant-a" + + def field(self, _context, _target): + return BlobFieldSpec("Asset", "document", ["docx", "xlsx"], True, "default") + + async def authorize(self, _context, target, action, _input): + return target.entity == "Asset" and target.field == "document" and action in ( + "Begin", "Write", "Confirm") + + +@pytest.mark.asyncio +async def test_real_office_upload_promotes_both_formats_and_rejects_broken_copy(tmp_path): + path = str(tmp_path / "office.db") + catalog = SqliteBlobCatalog(SqliteTransport(path)) + provider = FileObjectStorage(tmp_path / "objects", os.urandom(32)) + actor = RuntimeModule.new().into_context().with_object_storage(provider) + actor.with_object_storage_observer(lambda event: None) + service = SqliteBlobUploadService(catalog, OfficeAuthority(), + OfficeBlobContentVerifier(), 100_000) + target = BlobUploadTarget("Asset", "1", "document") + try: + await catalog.ensure_schema() + for extension in ("docx", "xlsx"): + good = office_fixture(extension) + upload = await service.begin(actor, target, + BlobUploadInput("valid." + extension, len(good)), + "office-good-" + extension) + assert await service.write(actor, upload.upload_id, BytesIO(good)) == len(good) + assert await service.complete(actor, upload.upload_id) == upload.reference + async with aiosqlite.connect(path) as db: + async with db.execute( + "SELECT status,content_type,size_bytes FROM teaql_blob_catalog WHERE key=?", + (upload.reference.key,)) as cursor: + status, content_type, size = await cursor.fetchone() + assert status == "uploaded_unbound" and size == len(good) + assert content_type == OfficeBlobContentVerifier().verify( + BytesIO(good), extension, len(good)).content_type + broken = office_fixture("xlsx")[:-12] + rejected = await service.begin(actor, target, BlobUploadInput("broken.xlsx", len(broken)), + "office-bad") + await service.write(actor, rejected.upload_id, BytesIO(broken)) + with pytest.raises(RuntimeError, match="BLOB_FORMAT_NOT_ALLOWED"): + await service.complete(actor, rejected.upload_id) + async with aiosqlite.connect(path) as db: + async with db.execute("SELECT status FROM teaql_blob_catalog WHERE key=?", + (rejected.reference.key,)) as cursor: + assert (await cursor.fetchone())[0] == "initiated" + finally: + provider.close() diff --git a/tests/provider/sqlite/test_blob_content_pillow.py b/tests/provider/sqlite/test_blob_content_pillow.py new file mode 100644 index 00000000..3c6dcc4b --- /dev/null +++ b/tests/provider/sqlite/test_blob_content_pillow.py @@ -0,0 +1,127 @@ +"""Real shared JPEG/PNG decoder and SQLite/File upload boundary checks.""" + +from hashlib import sha256 +from io import BytesIO +from pathlib import Path +from threading import Event, Thread +import os + +import aiosqlite +import pytest + +pytest.importorskip("PIL") + +from teaql.core.blob_ref import BlobFieldSpec +from teaql.runtime import RuntimeModule +from teaql.provider.sqlite.transport import SqliteTransport +from teaql_blob_reference import ( + BlobUploadInput, BlobUploadTarget, FileObjectStorage, SqliteBlobCatalog, + SqliteBlobUploadService, +) +from teaql_blob_reference.content_pillow import PillowBlobContentVerifier + + +def fixture(extension): + return (Path(__file__).parents[2] / "fixtures" / f"blob-sample.{extension}").read_bytes() + + +def test_real_images_and_malformed_or_bounded_input(): + verifier = PillowBlobContentVerifier() + verifier.validate_ready() + for extension in ("jpg", "png"): + data = fixture(extension) + checked = verifier.verify(BytesIO(data), extension, 10_000) + assert checked.size_bytes == len(data) + assert checked.sha256 == sha256(data).hexdigest() + assert checked.content_type == ("image/jpeg" if extension == "jpg" else "image/png") + wrong = "png" if extension == "jpg" else "jpg" + with pytest.raises(RuntimeError, match="BLOB_FORMAT_NOT_ALLOWED"): + verifier.verify(BytesIO(data), wrong, 10_000) + with pytest.raises(RuntimeError, match="BLOB_FORMAT_NOT_ALLOWED"): + verifier.verify(BytesIO(data[:-12]), extension, 10_000) + with pytest.raises(RuntimeError, match="BLOB_SIZE_EXCEEDED"): + verifier.verify(BytesIO(data), extension, len(data) - 1) + corrupt = bytearray(fixture("png")) + corrupt[-25] ^= 1 + with pytest.raises(RuntimeError, match="BLOB_FORMAT_NOT_ALLOWED"): + verifier.verify(BytesIO(corrupt), "png", 10_000) + with pytest.raises(RuntimeError, match="BLOB_FORMAT_NOT_ALLOWED"): + verifier.verify(BytesIO(b"not an image"), "jpg", 10_000) + with pytest.raises(RuntimeError, match="BLOB_FORMAT_NOT_ALLOWED"): + verifier.verify(BytesIO(b""), "png", 10_000) + limited = PillowBlobContentVerifier(maximum_input_bytes=10_000, + maximum_decoded_bytes=4) + with pytest.raises(RuntimeError, match="BLOB_FORMAT_NOT_ALLOWED"): + limited.verify(BytesIO(fixture("png")), "png", 10_000) + + +def test_concurrent_admission_releases_slot_after_decode(): + entered, release = Event(), Event() + data = fixture("jpg") + + class PausedInput(BytesIO): + def read(self, size=-1): + entered.set() + assert release.wait(5) + return super().read(size) + + verifier = PillowBlobContentVerifier(concurrent_verifications=1) + outcome = [] + first = Thread(target=lambda: outcome.append(verifier.verify(PausedInput(data), "jpg", 10_000))) + first.start() + try: + assert entered.wait(5) + with pytest.raises(RuntimeError, match="BLOB_VERIFICATION_BUSY"): + verifier.verify(BytesIO(data), "jpg", 10_000) + finally: + release.set() + first.join(5) + assert not first.is_alive() + assert outcome[0].content_type == "image/jpeg" + assert verifier.verify(BytesIO(data), "jpg", 10_000).size_bytes == len(data) + + +class ImageAuthority: + def scope(self, _context): + return "tenant-a" + + def field(self, _context, _target): + return BlobFieldSpec("Asset", "image", ["jpg"], True, "default") + + async def authorize(self, _context, target, action, _input): + return target.entity == "Asset" and target.field == "image" and action in ( + "Begin", "Write", "Confirm") + + +@pytest.mark.asyncio +async def test_actual_jpeg_upload_promotes_and_truncated_file_stays_initiated(tmp_path): + path = str(tmp_path / "image.db") + catalog = SqliteBlobCatalog(SqliteTransport(path)) + provider = FileObjectStorage(tmp_path / "objects", os.urandom(32)) + actor = RuntimeModule.new().into_context().with_object_storage(provider) + actor.with_object_storage_observer(lambda event: None) + service = SqliteBlobUploadService(catalog, ImageAuthority(), + PillowBlobContentVerifier(), 10_000) + target = BlobUploadTarget("Asset", "1", "image") + try: + await catalog.ensure_schema() + good = fixture("jpg") + upload = await service.begin(actor, target, BlobUploadInput("image.jpg", len(good)), "image-good") + assert await service.write(actor, upload.upload_id, BytesIO(good)) == len(good) + assert await service.complete(actor, upload.upload_id) == upload.reference + async with aiosqlite.connect(path) as db: + async with db.execute("SELECT status,content_type,size_bytes FROM teaql_blob_catalog WHERE key=?", + (upload.reference.key,)) as cursor: + assert await cursor.fetchone() == ("uploaded_unbound", "image/jpeg", len(good)) + bad = good[:-12] + rejected = await service.begin(actor, target, BlobUploadInput("broken.jpg", len(bad)), + "image-bad") + await service.write(actor, rejected.upload_id, BytesIO(bad)) + with pytest.raises(RuntimeError, match="BLOB_FORMAT_NOT_ALLOWED"): + await service.complete(actor, rejected.upload_id) + async with aiosqlite.connect(path) as db: + async with db.execute("SELECT status FROM teaql_blob_catalog WHERE key=?", + (rejected.reference.key,)) as cursor: + assert (await cursor.fetchone())[0] == "initiated" + finally: + provider.close() diff --git a/tests/provider/sqlite/test_blob_content_pypdf.py b/tests/provider/sqlite/test_blob_content_pypdf.py new file mode 100644 index 00000000..d783138c --- /dev/null +++ b/tests/provider/sqlite/test_blob_content_pypdf.py @@ -0,0 +1,128 @@ +"""Actual PDF parser and SQLite/File upload promotion boundary.""" + +from hashlib import sha256 +from io import BytesIO +from threading import Event, Thread +import os + +import aiosqlite +import pytest + +pypdf = pytest.importorskip("pypdf") + +from pypdf.generic import DecodedStreamObject, NameObject +from teaql.core.blob_ref import BlobFieldSpec +from teaql.provider.sqlite.transport import SqliteTransport +from teaql.runtime import RuntimeModule +from teaql_blob_reference import ( + BlobUploadInput, BlobUploadTarget, FileObjectStorage, SqliteBlobCatalog, + SqliteBlobUploadService, +) +from teaql_blob_reference.content_pypdf import PypdfBlobContentVerifier + + +def pdf_fixture(pages=1): + writer = pypdf.PdfWriter() + for _ in range(pages): + page = writer.add_blank_page(width=72, height=72) + content = DecodedStreamObject() + content.set_data(b"q 1 0 0 1 0 0 cm Q") + page[NameObject("/Contents")] = writer._add_object(content) + output = BytesIO() + writer.write(output) + return output.getvalue() + + +def test_real_pdf_structure_content_and_limits(): + data = pdf_fixture() + verifier = PypdfBlobContentVerifier() + verifier.validate_ready() + checked = verifier.verify(BytesIO(data), "pdf", 10_000) + assert (checked.size_bytes, checked.sha256, checked.content_type) == ( + len(data), sha256(data).hexdigest(), "application/pdf") + for invalid in (data[:-12], b"%PDF-1.7\nnot a real document\n%%EOF", + b"", b"\x89PNG\r\n\x1a\n%%EOF"): + with pytest.raises(RuntimeError, match="BLOB_FORMAT_NOT_ALLOWED"): + verifier.verify(BytesIO(invalid), "pdf", 10_000) + with pytest.raises(RuntimeError, match="BLOB_FORMAT_NOT_ALLOWED"): + verifier.verify(BytesIO(data), "png", 10_000) + with pytest.raises(RuntimeError, match="BLOB_SIZE_EXCEEDED"): + verifier.verify(BytesIO(data), "pdf", len(data) - 1) + with pytest.raises(RuntimeError, match="BLOB_FORMAT_NOT_ALLOWED"): + PypdfBlobContentVerifier(maximum_decoded_bytes=5).verify( + BytesIO(data), "pdf", 10_000) + with pytest.raises(RuntimeError, match="BLOB_FORMAT_NOT_ALLOWED"): + PypdfBlobContentVerifier(maximum_pages=1).verify( + BytesIO(pdf_fixture(2)), "pdf", 10_000) + + +def test_concurrent_admission_releases_slot(): + entered, release = Event(), Event() + data = pdf_fixture() + + class PausedInput(BytesIO): + def read(self, size=-1): + entered.set() + assert release.wait(5) + return super().read(size) + + verifier = PypdfBlobContentVerifier(concurrent_verifications=1) + outcome = [] + first = Thread(target=lambda: outcome.append(verifier.verify(PausedInput(data), "pdf", 10_000))) + first.start() + try: + assert entered.wait(5) + with pytest.raises(RuntimeError, match="BLOB_VERIFICATION_BUSY"): + verifier.verify(BytesIO(data), "pdf", 10_000) + finally: + release.set() + first.join(5) + assert not first.is_alive() + assert outcome[0].content_type == "application/pdf" + assert verifier.verify(BytesIO(data), "pdf", 10_000).size_bytes == len(data) + + +class PdfAuthority: + def scope(self, _context): + return "tenant-a" + + def field(self, _context, _target): + return BlobFieldSpec("Asset", "document", ["pdf"], True, "default") + + async def authorize(self, _context, target, action, _input): + return target.entity == "Asset" and target.field == "document" and action in ( + "Begin", "Write", "Confirm") + + +@pytest.mark.asyncio +async def test_actual_pdf_upload_promotes_and_truncated_file_stays_initiated(tmp_path): + path = str(tmp_path / "pdf.db") + catalog = SqliteBlobCatalog(SqliteTransport(path)) + provider = FileObjectStorage(tmp_path / "objects", os.urandom(32)) + actor = RuntimeModule.new().into_context().with_object_storage(provider) + actor.with_object_storage_observer(lambda event: None) + service = SqliteBlobUploadService(catalog, PdfAuthority(), + PypdfBlobContentVerifier(), 10_000) + target = BlobUploadTarget("Asset", "1", "document") + try: + await catalog.ensure_schema() + good = pdf_fixture() + upload = await service.begin(actor, target, BlobUploadInput("valid.pdf", len(good)), "pdf-good") + assert await service.write(actor, upload.upload_id, BytesIO(good)) == len(good) + assert await service.complete(actor, upload.upload_id) == upload.reference + async with aiosqlite.connect(path) as db: + async with db.execute("SELECT status,content_type,size_bytes FROM teaql_blob_catalog WHERE key=?", + (upload.reference.key,)) as cursor: + assert await cursor.fetchone() == ("uploaded_unbound", "application/pdf", len(good)) + bad = good[:-12] + rejected = await service.begin(actor, target, BlobUploadInput("broken.pdf", len(bad)), + "pdf-bad") + await service.write(actor, rejected.upload_id, BytesIO(bad)) + with pytest.raises(RuntimeError, match="BLOB_FORMAT_NOT_ALLOWED"): + await service.complete(actor, rejected.upload_id) + async with aiosqlite.connect(path) as db: + async with db.execute("SELECT status FROM teaql_blob_catalog WHERE key=?", + (rejected.reference.key,)) as cursor: + assert (await cursor.fetchone())[0] == "initiated" + finally: + provider.close() diff --git a/tests/provider/sqlite/test_blob_file_generation_maintenance.py b/tests/provider/sqlite/test_blob_file_generation_maintenance.py new file mode 100644 index 00000000..0a7a3918 --- /dev/null +++ b/tests/provider/sqlite/test_blob_file_generation_maintenance.py @@ -0,0 +1,125 @@ +"""Catalog-scoped scheduling for real fenced File old-generation cleanup.""" + +import os + +import aiosqlite +import pytest + +from teaql.core.object_storage import ObjectKey, PutOptions, WriteMode +from teaql.provider.sqlite.transport import SqliteTransport +from teaql.runtime import RuntimeModule +from teaql_blob_reference import ( + BlobMaintenanceCursor, FileObjectStorage, SqliteBlobCatalog, + SqliteBlobFileGenerationSweepService, +) + + +class FailOnceFileStorage(FileObjectStorage): + fail_next_sweep = False + + def sweep_old_generations(self, key): + if self.fail_next_sweep: + self.fail_next_sweep = False + raise OSError("transient File failure") + return super().sweep_old_generations(key) + + +class MaintenanceAuthority: + scope_value = "tenant-a" + allowed = True + + def scope(self, _actor): + return self.scope_value + + async def can_maintain(self, _actor, _scope): + return self.allowed + + +async def seed(path, reference, scope, object_key, version, status): + async with aiosqlite.connect(path) as db: + await db.execute("""INSERT INTO teaql_blob_catalog + (key,scope,object_key,content_version,file_name,content_type,size_bytes, + completed_at_ms,policy_profile,upload_mode,unbound_retention_seconds, + effective_unref_seconds,status) + VALUES (?,?,?,?,?,?,?,?,'default','server_stream',86400,86400,?)""", + (reference, scope, object_key.value, version, "report.pdf", "application/pdf", + 10, 1000, status)) + await db.execute("""INSERT INTO teaql_blob_upload + (upload_id,scope,replay_key,request_hash,key,entity,entity_id,field,file_name, + extension,declared_size,expires_ms,state,written_etag) + VALUES (?,?,?,?,?,'Asset','1','document','report.pdf','pdf',10,9000, + 'completed',?)""", + ("bu_" + reference, scope, "replay_" + reference, "hash_" + reference, + reference, version)) + await db.commit() + + +@pytest.mark.asyncio +async def test_catalog_file_sweep_covers_deleted_rows_preserves_current_and_retries(tmp_path): + path = str(tmp_path / "catalog.db") + catalog = SqliteBlobCatalog(SqliteTransport(path)) + await catalog.ensure_schema() + provider = FailOnceFileStorage(tmp_path / "objects", os.urandom(32)) + actor = RuntimeModule.new().into_context().with_object_storage(provider) + actor.with_object_storage_observer(lambda event: None) + store = actor.object_storage("catalog File sweep") + authority = MaintenanceAuthority() + service = SqliteBlobFileGenerationSweepService(catalog, authority, True, 1) + original = b"first data" + replacement = b"replacement data" + keys = [ObjectKey("objects/blob_a"), ObjectKey("objects/blob_b"), + ObjectKey("objects/blob_other")] + try: + assert provider is actor.get_resource("object_storage_provider") + for key, reference, scope, status in zip( + keys, ("blob_a", "blob_b", "blob_other"), + ("tenant-a", "tenant-a", "tenant-b"), + ("deleted", "uploaded_unbound", "uploaded_unbound")): + store.put_bytes(key, original, + PutOptions(len(original), WriteMode.CREATE_ONLY)) + current = store.put_bytes(key, replacement, + PutOptions(len(replacement), WriteMode.REPLACE)) + if status == "deleted": + assert store.delete_if_version(key, current.etag) == "deleted" + await seed(path, reference, scope, key, current.etag, status) + + with pytest.raises(RuntimeError, match="BLOB_POLICY_INVALID"): + SqliteBlobFileGenerationSweepService(catalog, authority, False) + authority.allowed = False + with pytest.raises(RuntimeError, match="BLOB_MAINTENANCE_DENIED"): + await service.run_batch(actor, provider) + authority.allowed = True + with pytest.raises(RuntimeError, match="BLOB_MAINTENANCE_DENIED"): + await service.run_batch(actor, provider, BlobMaintenanceCursor("tenant-b", "")) + token = actor._graph_save_guard.set({"active": True}) + try: + with pytest.raises(RuntimeError, match="BLOB_TRANSACTION_REQUIRED"): + await service.run_batch(actor, provider) + finally: + actor._graph_save_guard.reset(token) + with pytest.raises(RuntimeError, match="BLOB_CAPABILITY_MISSING"): + await service.run_batch(actor, FileObjectStorage(tmp_path / "objects", os.urandom(32))) + + provider.fail_next_sweep = True + failed = await service.run_batch(actor, provider) + assert [(entry.status, entry.error) for entry in failed.outcomes] == [ + ("failed", "BLOB_STORE_FAILED")] + outcomes = [] + cursor = None + while True: + batch = await service.run_batch(actor, provider, cursor) + outcomes.extend(batch.outcomes) + if batch.next is None: + break + cursor = batch.next + assert len(outcomes) == 2 + assert [entry.status for entry in outcomes] == ["swept", "swept"] + assert [entry.removed_generations for entry in outcomes] == [1, 1] + assert store.head(keys[0]) is None + assert store.get_bytes(keys[1], 100) == replacement + assert store.get_bytes(keys[2], 100) == replacement + assert provider.sweep_old_generations(keys[0]) == 0 + assert provider.sweep_old_generations(keys[1]) == 0 + assert provider.sweep_old_generations(keys[2]) == 1 # Other tenant was not scanned. + finally: + provider.close() diff --git a/tests/provider/sqlite/test_blob_maintenance.py b/tests/provider/sqlite/test_blob_maintenance.py new file mode 100644 index 00000000..a466b7c0 --- /dev/null +++ b/tests/provider/sqlite/test_blob_maintenance.py @@ -0,0 +1,463 @@ +"""Real SQLite state transitions for explicit, fenced Blob expiry cleanup.""" + +import aiosqlite +import pytest +from uuid import uuid4 +import os + +from teaql.core.object_storage import ObjectKey, ObjectMetadata, PutOptions, WriteMode +from teaql.provider.sqlite.transport import SqliteTransport +from teaql.runtime import RuntimeModule +from teaql_blob_reference import FileObjectStorage, SqliteBlobCatalog, SqliteBlobMaintenanceService + + +class _MaintenanceAuthority: + def __init__(self): + self.current_scope = "tenant-a" + self.allowed = True + + def scope(self, context): + return self.current_scope + + async def can_maintain(self, context, scope): + return self.allowed + + +class _ConditionalStore: + def __init__(self): + self.objects = {} + self.deletes = 0 + + def head(self, invocation, key): + etag = self.objects.get(key.value) + return ObjectMetadata(key, 1, "application/pdf", etag) if etag else None + + def delete_if_version(self, invocation, key, version): + self.deletes += 1 + actual = self.objects.get(key.value) + if actual is None: + return "already_missing" + if actual != version: + return "version_mismatch" + del self.objects[key.value] + return "deleted" + + def delete(self, invocation, key): + raise AssertionError("unconditional delete is forbidden") + + +async def _seed(path, key, *, scope="tenant-a", state="written", version="", + written_etag=None, expires=9000, bound=False): + async with aiosqlite.connect(path) as db: + await db.execute("""INSERT INTO teaql_blob_catalog + (key,scope,object_key,content_version,file_name,content_type,size_bytes,status) + VALUES (?,?,?,?,?,?,?,?)""", (key, scope, "objects/" + key, version, + "file.pdf", "", 0, + "uploaded_unbound" if state == "completed" else "initiated")) + await db.execute("""INSERT INTO teaql_blob_upload + (upload_id,scope,replay_key,request_hash,key,entity,entity_id,field,file_name, + extension,declared_size,expires_ms,state,written_etag) + VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?)""", ("bu_" + key, scope, "replay_" + key, + "hash_" + key, key, "Asset", "1", "document", "file.pdf", "pdf", + 1, expires, state, written_etag)) + if bound: + await db.execute("""INSERT INTO teaql_blob_binding + (scope,entity,entity_id,field,key,version) VALUES (?,?,?,?,?,?)""", + (scope, "Asset", key, "document", key, 1)) + await db.commit() + + +async def _status(path, key): + async with aiosqlite.connect(path) as db: + async with db.execute("SELECT status FROM teaql_blob_catalog WHERE key=?", (key,)) as cursor: + return (await cursor.fetchone())[0] + + +@pytest.mark.asyncio +async def test_expired_direct_uploads_are_not_collected_without_quiescence(tmp_path): + path = str(tmp_path / "direct-maintenance.db") + catalog = SqliteBlobCatalog(SqliteTransport(path)) + await catalog.ensure_schema() + provider = FileObjectStorage(tmp_path / "direct-objects", os.urandom(32)) + actor = RuntimeModule.new().into_context() + actor.with_object_storage(provider).with_object_storage_observer(lambda event: None) + store = actor.object_storage("direct upload maintenance guard") + service = SqliteBlobMaintenanceService(catalog, _MaintenanceAuthority(), 10, lambda: 10000) + for name, state, status in ( + ("blob_direct_initiated", "initiated", "initiated"), + ("blob_direct_written", "written", "initiated"), + ("blob_direct_deleting", "initiated", "deleting"), + ): + key = ObjectKey("objects/" + name) + created = store.put_bytes(key, b"%PDF-1.7\n", PutOptions(9, WriteMode.CREATE_ONLY)) + await _seed(path, name, state=state, + written_etag=created.etag if state == "written" else None) + async with aiosqlite.connect(path) as db: + await db.execute("UPDATE teaql_blob_catalog SET upload_mode=?,status=? WHERE key=?", + ("direct", status, name)) + await db.commit() + legacy = ObjectKey("objects/blob_legacy_stream") + store.put_bytes(legacy, b"%PDF-1.7\n", PutOptions(9, WriteMode.CREATE_ONLY)) + await _seed(path, "blob_legacy_stream", state="initiated") + + batch = await service.run_batch(actor) + assert [(item.reference.key, item.status) for item in batch.outcomes] == [ + ("blob_legacy_stream", "deleted")] + assert store.head(legacy) is None + for name, _, status in ( + ("blob_direct_initiated", "initiated", "initiated"), + ("blob_direct_written", "written", "initiated"), + ("blob_direct_deleting", "initiated", "deleting"), + ): + assert await _status(path, name) == status + assert store.head(ObjectKey("objects/" + name)) is not None + + +@pytest.mark.asyncio +async def test_real_sqlite_file_maintenance_recovers_receipt_and_partial_data(tmp_path): + path = str(tmp_path / "file-maintenance.db") + catalog = SqliteBlobCatalog(SqliteTransport(path)) + await catalog.ensure_schema() + provider = FileObjectStorage(tmp_path / "file-objects", os.urandom(32)) + actor = RuntimeModule.new().into_context() + actor.with_object_storage(provider).with_object_storage_observer(lambda event: None) + store = actor.object_storage("real File expired Blob maintenance") + service = SqliteBlobMaintenanceService(catalog, _MaintenanceAuthority(), 10, lambda: 10000) + content = b"%PDF-" + key = ObjectKey("objects/blob_file_receipt") + store.put_bytes(key, content, PutOptions(len(content), WriteMode.CREATE_ONLY)) + await _seed(path, "blob_file_receipt", state="initiated") + async with aiosqlite.connect(path) as db: + await db.execute("""CREATE TRIGGER fail_file_receipt BEFORE UPDATE OF status ON teaql_blob_catalog + WHEN NEW.status='deleted' BEGIN SELECT RAISE(ABORT,'receipt failure'); END""") + await db.commit() + failed = await service.run_batch(actor) + assert [(o.reference.key, o.status) for o in failed.outcomes] == [("blob_file_receipt", "failed")] + assert await _status(path, "blob_file_receipt") == "deleting" + assert store.head(key) is None + async with aiosqlite.connect(path) as db: + await db.execute("DROP TRIGGER fail_file_receipt") + await db.commit() + retry = await service.run_batch(actor) + assert [(o.reference.key, o.status) for o in retry.outcomes] == [ + ("blob_file_receipt", "already_missing")] + assert await _status(path, "blob_file_receipt") == "deleted" + + stale_key = ObjectKey("objects/blob_file_stale") + old = store.put_bytes(stale_key, content, PutOptions(len(content), WriteMode.CREATE_ONLY)) + await _seed(path, "blob_file_stale", state="initiated", version=old.etag) + newer = store.put_bytes(stale_key, content, PutOptions(len(content), WriteMode.REPLACE)) + stale = await service.run_batch(actor) + assert [(o.reference.key, o.status) for o in stale.outcomes] == [ + ("blob_file_stale", "version_conflict")] + assert store.head(stale_key).etag == newer.etag + + partial_key = ObjectKey("objects/blob_file_partial") + partial = store.put_bytes(partial_key, content, PutOptions(len(content), WriteMode.CREATE_ONLY)) + await _seed(path, "blob_file_partial", state="initiated", version=partial.etag) + manifest = provider._manifest(provider._hash(partial_key)) + provider._data_path(provider._hash(partial_key), manifest).unlink() + recovered = await service.run_batch(actor) + assert ("blob_file_partial", "deleted") in [(o.reference.key, o.status) for o in recovered.outcomes] + assert await _status(path, "blob_file_partial") == "deleted" + + +@pytest.mark.asyncio +async def test_real_file_completed_retention_claim_and_generation_fences(tmp_path): + path = str(tmp_path / "completed-retention.db") + catalog = SqliteBlobCatalog(SqliteTransport(path)) + await catalog.ensure_schema() + provider = FileObjectStorage(tmp_path / "completed-objects", os.urandom(32)) + actor = RuntimeModule.new().into_context() + actor.with_object_storage(provider).with_object_storage_observer(lambda event: None) + store = actor.object_storage("real File completed retention") + service = SqliteBlobMaintenanceService(catalog, _MaintenanceAuthority(), 100, lambda: 10000) + content = b"%PDF-1.7\nretained\n%%EOF" + + async def seed(name, *, status="uploaded_unbound", completed=9000, + last_unbound=None, floor=None, profile="default", mode="server_stream", + bound=False): + key = ObjectKey("objects/" + name) + created = store.put_bytes(key, content, PutOptions(len(content), WriteMode.CREATE_ONLY)) + await _seed(path, name, state="completed", version=created.etag, + written_etag=created.etag, bound=bound) + async with aiosqlite.connect(path) as db: + await db.execute("UPDATE teaql_blob_catalog SET status=?,completed_at_ms=?," + "last_unbound_at_ms=?,policy_profile=?,upload_mode=?," + "unbound_retention_seconds=1,effective_unref_seconds=2," + "retention_floor_until_ms=?,content_type='application/pdf',size_bytes=? " + "WHERE key=?", (status, completed, last_unbound, profile, mode, + floor, len(content), name)) + await db.commit() + return key, created.etag + + expired, _ = await seed("blob_completed_expired") + formerly_bound, _ = await seed("blob_completed_formerly_bound", status="available", + last_unbound=7000, floor=10000) + active, _ = await seed("blob_completed_active", status="available", bound=True) + floored, _ = await seed("blob_completed_floor", status="available", + last_unbound=7000, floor=20000) + legacy, _ = await seed("blob_completed_legacy", profile=None) + direct, _ = await seed("blob_completed_direct", mode="direct") + batch = await service.run_batch(actor) + assert [(result.reference.key, result.status) for result in batch.outcomes] == [ + ("blob_completed_expired", "deleted"), + ("blob_completed_formerly_bound", "deleted")] + assert store.head(expired) is None and store.head(formerly_bound) is None + for name, key in (("blob_completed_active", active), ("blob_completed_floor", floored), + ("blob_completed_legacy", legacy), ("blob_completed_direct", direct)): + assert await _status(path, name) != "deleted" and store.head(key) is not None + + stale_key, stale_version = await seed("blob_completed_stale") + replacement = store.put_bytes(stale_key, content, PutOptions(len(content), WriteMode.REPLACE)) + assert replacement.etag != stale_version + stale = await service.run_batch(actor) + assert [(result.reference.key, result.status) for result in stale.outcomes] == [ + ("blob_completed_stale", "version_conflict")] + assert store.head(stale_key).etag == replacement.etag + assert await _status(path, "blob_completed_stale") == "deleting" + + retry_key, _ = await seed("blob_completed_retry") + async with aiosqlite.connect(path) as db: + await db.execute("""CREATE TRIGGER fail_completed_receipt BEFORE UPDATE OF status + ON teaql_blob_catalog WHEN NEW.status='deleted' AND OLD.key='blob_completed_retry' + BEGIN SELECT RAISE(ABORT,'receipt failure'); END""") + await db.commit() + failed = await service.run_batch(actor) + assert [(result.reference.key, result.status) for result in failed.outcomes] == [ + ("blob_completed_retry", "failed"), + ("blob_completed_stale", "version_conflict")] + assert await _status(path, "blob_completed_retry") == "deleting" + assert store.head(retry_key) is None + async with aiosqlite.connect(path) as db: + await db.execute("DROP TRIGGER fail_completed_receipt") + await db.commit() + recovered = await service.run_batch(actor) + assert [(result.reference.key, result.status) for result in recovered.outcomes] == [ + ("blob_completed_retry", "already_missing"), + ("blob_completed_stale", "version_conflict")] + assert await _status(path, "blob_completed_retry") == "deleted" + + +@pytest.mark.asyncio +async def test_expired_upload_maintenance_claim_scope_version_and_recovery(tmp_path): + path = str(tmp_path / "maintenance.db") + catalog = SqliteBlobCatalog(SqliteTransport(path)) + await catalog.ensure_schema() + actor = RuntimeModule.new().into_context() + store = _ConditionalStore() + actor.with_object_storage(store).with_object_storage_observer(lambda event: None) + auth = _MaintenanceAuthority() + service = SqliteBlobMaintenanceService(catalog, auth, 2, lambda: 10000) + await _seed(path, "blob_absent", state="initiated") + await _seed(path, "blob_bound", state="initiated", bound=True) + await _seed(path, "blob_completed", state="completed", written_etag="etag-completed") + await _seed(path, "blob_future", expires=20000, written_etag="etag-future") + await _seed(path, "blob_other", scope="tenant-b", written_etag="etag-other") + await _seed(path, "blob_race", version="etag-old", written_etag="etag-old") + await _seed(path, "blob_written", written_etag="etag-written") + store.objects.update({"objects/blob_race": "etag-new", + "objects/blob_written": "etag-written", + "objects/blob_completed": "etag-completed", + "objects/blob_future": "etag-future", + "objects/blob_other": "etag-other"}) + await catalog.validate_schema() + auth.allowed = False + with pytest.raises(RuntimeError, match="^BLOB_MAINTENANCE_DENIED$"): + await service.run_batch(actor) + auth.allowed = True + first = await service.run_batch(actor) + assert [(o.reference.key, o.status) for o in first.outcomes] == [ + ("blob_absent", "already_missing"), ("blob_race", "version_conflict")] + assert first.next is not None + second = await service.run_batch(actor, first.next) + assert [(o.reference.key, o.status) for o in second.outcomes] == [ + ("blob_written", "deleted")] + assert store.deletes == 2 # the stale pinned version is refused by conditional deletion + assert store.objects["objects/blob_race"] == "etag-new" + for key, expected in {"blob_absent": "deleted", "blob_race": "deleting", + "blob_written": "deleted", "blob_completed": "uploaded_unbound", + "blob_future": "initiated", "blob_other": "initiated", + "blob_bound": "initiated"}.items(): + assert await _status(path, key) == expected + auth.current_scope = "tenant-b" + with pytest.raises(RuntimeError, match="^BLOB_MAINTENANCE_DENIED$"): + await service.run_batch(actor, first.next) + auth.current_scope = "tenant-a" + await _seed(path, "blob_retry", written_etag="etag-retry") + store.objects["objects/blob_retry"] = "etag-retry" + async with aiosqlite.connect(path) as db: + await db.execute("""CREATE TRIGGER fail_final_ack BEFORE UPDATE OF status ON teaql_blob_catalog + WHEN NEW.status='deleted' AND OLD.key='blob_retry' + BEGIN SELECT RAISE(ABORT,'private trigger detail'); END""") + await db.commit() + failed = await service.run_batch(actor) + assert failed.outcomes[1].status == "failed" + assert failed.outcomes[1].error == "BLOB_CATALOG_FAILED" + assert "objects/blob_retry" not in store.objects + assert await _status(path, "blob_retry") == "deleting" + async with aiosqlite.connect(path) as db: + await db.execute("DROP TRIGGER fail_final_ack") + await db.commit() + recovered = await service.run_batch(actor) + assert recovered.outcomes[1].status == "already_missing" + assert await _status(path, "blob_retry") == "deleted" + + +@pytest.mark.asyncio +async def test_expired_upload_maintenance_rejects_unfenced_store_graph_and_schema(tmp_path): + path = str(tmp_path / "maintenance-gates.db") + catalog = SqliteBlobCatalog(SqliteTransport(path)) + await catalog.ensure_schema() + actor = RuntimeModule.new().into_context() + auth = _MaintenanceAuthority() + service = SqliteBlobMaintenanceService(catalog, auth, 2, lambda: 10000) + actor.with_object_storage(object()) + with pytest.raises(RuntimeError, match="^BLOB_CAPABILITY_MISSING$"): + await service.run_batch(actor) + actor.with_object_storage(_ConditionalStore()) + token = actor._graph_save_guard.set({"active": True}) + try: + with pytest.raises(RuntimeError, match="^BLOB_TRANSACTION_REQUIRED$"): + await service.run_batch(actor) + finally: + actor._graph_save_guard.reset(token) + async with aiosqlite.connect(path) as db: + await db.execute("DROP INDEX teaql_blob_catalog_object_key") + await db.commit() + with pytest.raises(RuntimeError, match="^BLOB_SCHEMA_MISSING$"): + await service.run_batch(actor) + + +@pytest.mark.asyncio +async def test_real_s3_expired_upload_maintenance_delete_recovery_and_stale_version(tmp_path): + if os.environ.get("TEAQL_S3_TEST_BLOB_MAINTENANCE") != "1": + pytest.skip("explicit disposable S3 maintenance gate required") + endpoint = os.environ.get("TEAQL_S3_TEST_ENDPOINT") + bucket = os.environ.get("TEAQL_S3_TEST_BUCKET") + access = os.environ.get("TEAQL_S3_TEST_ACCESS_KEY") + secret = os.environ.get("TEAQL_S3_TEST_SECRET_KEY") + assert all((endpoint, bucket, access, secret)), "disposable S3 configuration required" + from teaql.provider.s3 import S3ObjectStorage + provider = S3ObjectStorage.connect(bucket, endpoint_url=endpoint, + aws_access_key_id=access, aws_secret_access_key=secret) + actor = RuntimeModule.new().into_context() + actor.with_object_storage(provider).with_object_storage_observer(lambda event: None) + store = actor.object_storage("real S3 expired Blob maintenance") + path = str(tmp_path / "s3-maintenance.db") + catalog = SqliteBlobCatalog(SqliteTransport(path)) + await catalog.ensure_schema() + service = SqliteBlobMaintenanceService(catalog, _MaintenanceAuthority(), 10, lambda: 10000) + prefix = uuid4().hex + keys = [] + + async def seed(suffix, content): + reference = "blob_" + prefix + "_" + suffix + key = ObjectKey("objects/" + reference) + keys.append(key) + created = store.put_bytes(key, content, PutOptions(len(content), WriteMode.CREATE_ONLY)) + await _seed(path, reference, written_etag=created.etag) + return reference, key, created.etag + + try: + first_ref, first_key, _ = await seed("first", b"first version") + first = await service.run_batch(actor) + assert [(o.reference.key, o.status) for o in first.outcomes] == [(first_ref, "deleted")] + assert store.head(first_key) is None + assert await _status(path, first_ref) == "deleted" + + recovery_ref, recovery_key, _ = await seed("recovery", b"recovery version") + async with aiosqlite.connect(path) as db: + await db.execute("""CREATE TRIGGER fail_final_ack BEFORE UPDATE OF status ON teaql_blob_catalog + WHEN NEW.status='deleted' AND OLD.key='""" + recovery_ref + """' + BEGIN SELECT RAISE(ABORT,'private trigger detail'); END""") + await db.commit() + failed = await service.run_batch(actor) + assert [(o.reference.key, o.status, o.error) for o in failed.outcomes] == [ + (recovery_ref, "failed", "BLOB_CATALOG_FAILED")] + assert store.head(recovery_key) is None + assert await _status(path, recovery_ref) == "deleting" + async with aiosqlite.connect(path) as db: + await db.execute("DROP TRIGGER fail_final_ack") + await db.commit() + recovered = await service.run_batch(actor) + assert [(o.reference.key, o.status) for o in recovered.outcomes] == [ + (recovery_ref, "already_missing")] + assert await _status(path, recovery_ref) == "deleted" + + race_ref, race_key, _ = await seed("stale", b"old version") + replacement_bytes = b"replacement version" + replacement = store.put_bytes(race_key, replacement_bytes, + PutOptions(len(replacement_bytes), WriteMode.REPLACE)) + conflict = await service.run_batch(actor) + assert [(o.reference.key, o.status) for o in conflict.outcomes] == [ + (race_ref, "version_conflict")] + assert store.head(race_key).etag == replacement.etag + assert await _status(path, race_ref) == "deleting" + finally: + for key in keys: + store.delete(key) + + +@pytest.mark.asyncio +async def test_real_s3_completed_retention_selects_only_eligible_server_stream_objects(tmp_path): + if os.environ.get("TEAQL_S3_TEST_BLOB_MAINTENANCE") != "1": + pytest.skip("explicit disposable S3 maintenance gate required") + endpoint = os.environ.get("TEAQL_S3_TEST_ENDPOINT") + bucket = os.environ.get("TEAQL_S3_TEST_BUCKET") + access = os.environ.get("TEAQL_S3_TEST_ACCESS_KEY") + secret = os.environ.get("TEAQL_S3_TEST_SECRET_KEY") + assert all((endpoint, bucket, access, secret)), "disposable S3 configuration required" + from teaql.provider.s3 import S3ObjectStorage + provider = S3ObjectStorage.connect(bucket, endpoint_url=endpoint, + aws_access_key_id=access, aws_secret_access_key=secret) + actor = RuntimeModule.new().into_context() + actor.with_object_storage(provider).with_object_storage_observer(lambda event: None) + store = actor.object_storage("real S3 completed Blob retention") + path = str(tmp_path / "s3-completed-retention.db") + catalog = SqliteBlobCatalog(SqliteTransport(path)) + await catalog.ensure_schema() + service = SqliteBlobMaintenanceService(catalog, _MaintenanceAuthority(), 10, lambda: 10000) + prefix = uuid4().hex + keys = [] + + async def seed(suffix, status="uploaded_unbound", last_unbound=None, + floor=None, profile="default", mode="server_stream", bound=False): + reference = "blob_" + prefix + "_" + suffix + key = ObjectKey("objects/" + reference) + keys.append(key) + content = ("%PDF-1.7\n" + suffix + "\n%%EOF").encode() + created = store.put_bytes(key, content, PutOptions(len(content), WriteMode.CREATE_ONLY)) + await _seed(path, reference, state="completed", version=created.etag, + written_etag=created.etag, bound=bound) + async with aiosqlite.connect(path) as db: + await db.execute("UPDATE teaql_blob_catalog SET status=?,completed_at_ms=9000," + "last_unbound_at_ms=?,policy_profile=?,upload_mode=?," + "unbound_retention_seconds=1,effective_unref_seconds=2," + "retention_floor_until_ms=?,content_type='application/pdf',size_bytes=? " + "WHERE key=?", (status, last_unbound, profile, mode, floor, + len(content), reference)) + await db.execute("UPDATE teaql_blob_upload SET declared_size=? WHERE key=?", + (len(content), reference)) + await db.commit() + return reference, key + + try: + expired = await seed("expired") + former = await seed("former", "available", 7000, 10000) + active = await seed("active", "available", bound=True) + floored = await seed("floored", "available", 7000, 20000) + legacy = await seed("legacy", profile=None) + direct = await seed("direct", mode="direct") + batch = await service.run_batch(actor) + assert [(item.reference.key, item.status) for item in batch.outcomes] == [ + (expired[0], "deleted"), (former[0], "deleted")] + for reference, key in (expired, former): + assert store.head(key) is None + assert await _status(path, reference) == "deleted" + for reference, key in (active, floored, legacy, direct): + assert store.head(key) is not None + assert await _status(path, reference) != "deleted" + finally: + for key in keys: + store.delete(key) diff --git a/tests/provider/sqlite/test_blob_module.py b/tests/provider/sqlite/test_blob_module.py new file mode 100644 index 00000000..7713dea9 --- /dev/null +++ b/tests/provider/sqlite/test_blob_module.py @@ -0,0 +1,835 @@ +"""The optional Blob assembly fails closed until the owner and catalog are ready.""" + +import os +import time +from datetime import datetime, timedelta, timezone +from hashlib import sha256 +from io import BytesIO +from types import SimpleNamespace +from urllib.request import Request, urlopen + +import aiosqlite +import pytest + +from teaql.core.blob_ref import BlobFieldSpec, BlobRef +from teaql.core.entity import EntityKey, EntityRoot +from teaql.core.meta import EntityDescriptor, PropertyDescriptor +from teaql.core.mutation import DeleteCommand, InsertCommand, UpdateCommand +from teaql.core.object_storage import ObjectKey +from teaql.core.value import DataType +from teaql.provider.sqlite import SimpleSchemaProvider +from teaql.provider.sqlite.dialect import SqliteDialect +from teaql.provider.sqlite.transport import SqliteTransport +from teaql.runtime import RuntimeModule, validate_capabilities +from teaql.runtime.view_preparation import ( + AuthorizedBlobSlot, AuthorizedResponseGraph, ViewOptions, + ViewPreparationError, ViewPreparationService, +) +from teaql.sql.executor import SqlDataServiceExecutor +from teaql_blob_reference import ( + BlobReferenceModule, BlobRetentionPolicy, FileObjectStorage, SqliteBlobCatalog, + BlobUploadInput, BlobUploadTarget, BlobVerifiedContent, + SqliteBlobMaintenanceService, SqliteBlobUploadService, +) + + +class _ReadAuthority: + allowed = True + + async def authorize(self, context, slot): + return self.allowed + + +class _UploadAuthority: + def scope(self, context): + return "tenant-a" + + def field(self, context, target): + raise AssertionError("readiness must not begin an upload") + + async def authorize(self, context, target, action, input): + return True + + +class _Verifier: + def __init__(self): + self.ready_calls = 0 + + def verify(self, stream, extension, maximum_bytes): + raise AssertionError("readiness must not parse content") + + def validate_ready(self): + self.ready_calls += 1 + + +class _BindingAuthority: + allowed = True + + def scope(self, context): + return "tenant-a" + + async def authorize(self, context, target, previous, next_ref, action): + return self.allowed and target.entity == "Asset" and target.field == "document" + + async def current_policy_on_lease(self, context, lease, target): + assert lease is not None + return BlobRetentionPolicy("default", "server_stream", 86400, 604800, 3600) + + +class _MaintenanceAuthority: + def scope(self, context): + return "tenant-a" + + async def can_maintain(self, context, scope): + return scope == "tenant-a" + + +class _SafeAuditSink: + def __init__(self): + self.events = [] + + async def on_safe_event(self, context, event): + self.events.append(event) + + +@pytest.mark.asyncio +async def test_blob_module_requires_provider_owner_and_catalog(tmp_path): + path = str(tmp_path / "module.db") + transport = SqliteTransport(path) + catalog = SqliteBlobCatalog(transport) + verifier = _Verifier() + upload = SqliteBlobUploadService(catalog, _UploadAuthority(), verifier, 10_000) + maintenance = SqliteBlobMaintenanceService(catalog, _MaintenanceAuthority()) + module = BlobReferenceModule(catalog, upload, _ReadAuthority(), _BindingAuthority(), + maintenance=maintenance) + context = RuntimeModule.new().require_capability("blob.v1").into_context() + with pytest.raises(RuntimeError, match="BLOB_CAPABILITY_MISSING"): + await validate_capabilities(context) + module.install(context) + assert context.get_resource("blob_response_contributor") is module.read + assert context.get_resource("blob_maintenance_service") is maintenance + with pytest.raises(RuntimeError, match="BLOB_CAPABILITY_MISSING"): + await module.ready(context) + with pytest.raises(RuntimeError, match="BLOB_CAPABILITY_MISSING"): + module.install(context) + + context.insert_resource("dataService", SqlDataServiceExecutor( + SqliteDialect(), SqliteTransport(path), SimpleSchemaProvider())) + provider = FileObjectStorage(tmp_path / "private", os.urandom(32)) + context.with_object_storage(provider) + with pytest.raises(RuntimeError, match="BLOB_SCHEMA_MISSING"): + await module.ready(context) + await catalog.ensure_schema() + await module.ready(context) + assert verifier.ready_calls == 1 + await validate_capabilities(context) + provider.close() + + +def test_blob_module_rejects_an_upload_from_another_catalog(tmp_path): + first = SqliteBlobCatalog(SqliteTransport(str(tmp_path / "first.db"))) + second = SqliteBlobCatalog(SqliteTransport(str(tmp_path / "second.db"))) + upload = SqliteBlobUploadService(second, _UploadAuthority(), _Verifier(), 10_000) + with pytest.raises(RuntimeError, match="BLOB_CAPABILITY_MISSING"): + BlobReferenceModule(first, upload, _ReadAuthority(), _BindingAuthority()) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("scenario", ["other-file", "separate-memory", "non-native", "wrong-dialect", "upload-catalog-drift"]) +async def test_blob_module_rejects_incompatible_business_database(tmp_path, scenario): + path = str(tmp_path / "catalog.db") + catalog = SqliteBlobCatalog(SqliteTransport(path)) + await catalog.ensure_schema() + verifier = _Verifier() + upload = SqliteBlobUploadService(catalog, _UploadAuthority(), verifier, 10_000) + module = BlobReferenceModule(catalog, upload, _ReadAuthority(), _BindingAuthority()) + owner = SqlDataServiceExecutor(SqliteDialect(), SqliteTransport(path), SimpleSchemaProvider()) + if scenario == "other-file": + owner.transport = SqliteTransport(str(tmp_path / "other.db")) + elif scenario == "separate-memory": + owner.transport = SqliteTransport(":memory:") + elif scenario == "non-native": + async def begin(_context): + raise AssertionError("readiness must not open this non-native owner") + owner = SimpleNamespace(begin=begin) + elif scenario == "wrong-dialect": + from teaql.provider.mysql.dialect import MysqlDialect + owner.dialect = MysqlDialect() + context = RuntimeModule.new().require_capability("blob.v1").into_context() + context.insert_resource("dataService", owner) + provider = FileObjectStorage(tmp_path / "objects", os.urandom(32)) + context.with_object_storage(provider) + module.install(context) + assert verifier.ready_calls == 0 + if scenario == "upload-catalog-drift": + upload.catalog = SqliteBlobCatalog(SqliteTransport(str(tmp_path / "other.db"))) + try: + with pytest.raises(RuntimeError, match="^BLOB_CAPABILITY_MISSING$"): + await module.ready(context) + assert verifier.ready_calls == 0 + finally: + provider.close() + + +@pytest.mark.asyncio +async def test_blob_module_accepts_same_database_through_native_file_uri(tmp_path): + path = tmp_path / "module.db" + catalog = SqliteBlobCatalog(SqliteTransport(str(path))) + await catalog.ensure_schema() + verifier = _Verifier() + upload = SqliteBlobUploadService(catalog, _UploadAuthority(), verifier, 10_000) + module = BlobReferenceModule(catalog, upload, _ReadAuthority(), _BindingAuthority()) + owner = SqlDataServiceExecutor(SqliteDialect(), SqliteTransport(path.as_uri() + "?mode=rwc"), SimpleSchemaProvider()) + context = RuntimeModule.new().require_capability("blob.v1").into_context() + context.insert_resource("dataService", owner) + provider = FileObjectStorage(tmp_path / "objects", os.urandom(32)) + context.with_object_storage(provider) + try: + module.install(context) + await module.ready(context) + assert verifier.ready_calls == 1 + finally: + provider.close() + + +@pytest.mark.asyncio +async def test_native_mutation_binds_blob_in_same_sqlite_graph(tmp_path): + path = str(tmp_path / "native.db") + descriptor = EntityDescriptor("Asset").table_name("asset") + descriptor.property(PropertyDescriptor("id", DataType.I64).is_id()) + descriptor.property(PropertyDescriptor("version", DataType.I64).is_version()) + descriptor.property(PropertyDescriptor("caption", DataType.Text)) + descriptor.property(PropertyDescriptor("document", DataType.Text).blob( + BlobFieldSpec("Asset", "document", ["pdf"], True, "default"))) + schema = SimpleSchemaProvider() + schema.register_entity(descriptor) + context = RuntimeModule.new().entity(descriptor).require_capability("blob.v1").into_context() + context.insert_resource("dataService", SqlDataServiceExecutor( + SqliteDialect(), SqliteTransport(path), schema)) + audit = _SafeAuditSink() + context.with_app_audit_event_sink(audit) + store = FileObjectStorage(tmp_path / "native-private", os.urandom(32)) + context.with_object_storage(store) + catalog = SqliteBlobCatalog(SqliteTransport(path)) + await catalog.ensure_schema() + async with aiosqlite.connect(path) as db: + await db.execute("CREATE TABLE asset (id INTEGER PRIMARY KEY, version INTEGER, document TEXT, caption TEXT)") + for key in ("blob_native_one", "blob_native_two"): + await db.execute("INSERT INTO teaql_blob_catalog " + "(key,scope,object_key,content_version,file_name,content_type,size_bytes,status," + "completed_at_ms,policy_profile,upload_mode,unbound_retention_seconds,effective_unref_seconds) " + "VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?)", + (key, "tenant-a", f"objects/{key}", "v1", "report.pdf", + "application/pdf", 7, "uploaded_unbound", time.time_ns() // 1_000_000, + "default", "server_stream", 86400, 604800)) + await db.commit() + authority = _BindingAuthority() + upload = SqliteBlobUploadService(catalog, _UploadAuthority(), _Verifier(), 10_000) + module = BlobReferenceModule(catalog, upload, _ReadAuthority(), authority) + module.install(context) + await module.ready(context) + + async def save(command, comment, asset_id=1): + async def work(graph): + return await graph.transaction.mutate(graph.context, + graph.request(command, graph.scope(EntityKey("Asset", asset_id)))) + return await context.execute_graph_save(work, comment=comment) + + first, second = BlobRef("blob_native_one"), BlobRef("blob_native_two") + await save(InsertCommand.new("Asset").value("id", 1).value("document", first), "create") + assert audit.events and any(field.field == "document" and field.value == "[REDACTED]" + for field in audit.events[-1].fields) + async with aiosqlite.connect(path) as db: + async with db.execute("SELECT document,version FROM asset WHERE id=1") as cursor: + assert await cursor.fetchone() == (first.key, 1) + async with db.execute("SELECT key,version FROM teaql_blob_binding") as cursor: + assert await cursor.fetchone() == (first.key, 1) + + await save(UpdateCommand.new("Asset", 1).expected_version(1) + .value("document", second), "replace") + async with aiosqlite.connect(path) as db: + async with db.execute("SELECT document,version FROM asset WHERE id=1") as cursor: + assert await cursor.fetchone() == (second.key, 2) + async with db.execute("SELECT key,version FROM teaql_blob_binding") as cursor: + assert await cursor.fetchone() == (second.key, 2) + async with db.execute("SELECT key,status FROM teaql_blob_catalog ORDER BY key") as cursor: + assert await cursor.fetchall() == [(first.key, "available"), + (second.key, "available")] + + authority.allowed = False + with pytest.raises(RuntimeError, match="BLOB_BIND_DENIED"): + await save(UpdateCommand.new("Asset", 1).expected_version(2) + .value("document", first), "deny replacement") + authority.allowed = True + with pytest.raises(RuntimeError, match="BLOB_REFERENCE_INVALID"): + await save(UpdateCommand.new("Asset", 1).expected_version(2) + .value("document", second.key), "reject raw Key") + async with aiosqlite.connect(path) as db: + async with db.execute("SELECT document,version FROM asset WHERE id=1") as cursor: + assert await cursor.fetchone() == (second.key, 2) + async with db.execute("SELECT key,version FROM teaql_blob_binding") as cursor: + assert await cursor.fetchone() == (second.key, 2) + await save(UpdateCommand.new("Asset", 1).expected_version(2) + .value("document", None), "clear") + async with aiosqlite.connect(path) as db: + async with db.execute("SELECT key,version FROM teaql_blob_binding") as cursor: + assert await cursor.fetchone() == (None, 3) + await save(UpdateCommand.new("Asset", 1).expected_version(3) + .value("document", second), "rebind") + await save(DeleteCommand.new("Asset", 1).expected_version(4).hard_delete(), "delete") + async with aiosqlite.connect(path) as db: + async with db.execute("SELECT id FROM asset") as cursor: + assert await cursor.fetchone() is None + async with db.execute("SELECT key,version FROM teaql_blob_binding") as cursor: + assert await cursor.fetchone() == (None, 5) + async with db.execute("SELECT status FROM teaql_blob_catalog WHERE key=?", + (second.key,)) as cursor: + assert await cursor.fetchone() == ("available",) + await save(InsertCommand.new("Asset").value("id", 2), "create empty", asset_id=2) + async with aiosqlite.connect(path) as db: + async with db.execute("SELECT key,version FROM teaql_blob_binding WHERE entity_id='2'") as cursor: + assert await cursor.fetchone() == (None, 1) + await save(DeleteCommand.new("Asset", 2).expected_version(1).hard_delete(), + "delete empty", asset_id=2) + + ledger_ref = BlobRef("blob_native_ledger") + async with aiosqlite.connect(path) as db: + await db.execute("INSERT INTO teaql_blob_catalog " + "(key,scope,object_key,content_version,file_name,content_type,size_bytes,status," + "completed_at_ms,policy_profile,upload_mode,unbound_retention_seconds,effective_unref_seconds) " + "VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?)", + (ledger_ref.key, "tenant-a", "objects/ledger", "v1", "ledger.pdf", + "application/pdf", 7, "uploaded_unbound", time.time_ns() // 1_000_000, + "default", "server_stream", 86400, 604800)) + await db.commit() + + async def save_allocated_ledger(abort): + ledger = EntityRoot() + provisional = EntityKey("Asset", -1) + ledger.mark_as_new(provisional) + ledger.set(provisional, "document", ledger_ref) + + async def work(graph): + allocated = await graph.transaction.next_id("Asset") + assert type(allocated) is int and allocated > 0 + actual = EntityKey("Asset", allocated) + ledger.rekey(provisional, actual) + command = (InsertCommand.new("Asset").value("id", allocated) + .value("document", ledger_ref)) + result = await graph.transaction.mutate(graph.context, + graph.request(command, graph.scope(actual), ledger, actual)) + assert getattr(result.generated_values["id"], "val") == allocated + assert ledger.get(actual, "document") == ledger_ref + if abort: + raise RuntimeError("outer owner rejected") + return allocated + + return await context.execute_graph_save(work, comment="save allocated ledger Blob") + + with pytest.raises(RuntimeError, match="outer owner rejected"): + await save_allocated_ledger(True) + async with aiosqlite.connect(path) as db: + async with db.execute("SELECT COUNT(*) FROM asset WHERE document=?", + (ledger_ref.key,)) as cursor: + assert await cursor.fetchone() == (0,) + async with db.execute("SELECT COUNT(*) FROM teaql_blob_binding WHERE key=?", + (ledger_ref.key,)) as cursor: + assert await cursor.fetchone() == (0,) + async with db.execute("SELECT status FROM teaql_blob_catalog WHERE key=?", + (ledger_ref.key,)) as cursor: + assert await cursor.fetchone() == ("uploaded_unbound",) + allocated = await save_allocated_ledger(False) + async with aiosqlite.connect(path) as db: + async with db.execute("SELECT document FROM asset WHERE id=?", (allocated,)) as cursor: + assert await cursor.fetchone() == (ledger_ref.key,) + async with db.execute("SELECT key FROM teaql_blob_binding WHERE entity_id=?", + (str(allocated),)) as cursor: + assert await cursor.fetchone() == (ledger_ref.key,) + async with db.execute("SELECT status FROM teaql_blob_catalog WHERE key=?", + (ledger_ref.key,)) as cursor: + assert await cursor.fetchone() == ("available",) + async def snapshot(query, parameters): + async with aiosqlite.connect(path) as db: + async with db.execute(query, parameters) as cursor: + return await cursor.fetchall() + + binding_query = "SELECT * FROM teaql_blob_binding WHERE entity_id=?" + catalog_query = "SELECT * FROM teaql_blob_catalog WHERE key=?" + binding_before = await snapshot(binding_query, (str(allocated),)) + catalog_before = await snapshot(catalog_query, (ledger_ref.key,)) + ledger = EntityRoot() + actual = EntityKey("Asset", allocated) + ledger.set(actual, "caption", "after") + assigned = dict(ledger.current_change_set().changes())[actual] + assert "document" not in assigned + authority.allowed = False + + async def update_ordinary(graph): + command = (UpdateCommand.new("Asset", allocated).expected_version(1) + .value("caption", assigned["caption"])) + return await graph.transaction.mutate(graph.context, + graph.request(command, graph.scope(actual), ledger, actual)) + + await context.execute_graph_save(update_ordinary, comment="update caption with unloaded Blob") + assert await snapshot("SELECT document,caption,version FROM asset WHERE id=?", (allocated,)) == [ + (ledger_ref.key, "after", 2)] + assert await snapshot(binding_query, (str(allocated),)) == binding_before + assert await snapshot(catalog_query, (ledger_ref.key,)) == catalog_before + authority.allowed = True + await save(UpdateCommand.new("Asset", allocated).expected_version(2) + .value("document", None), "explicitly clear Blob", asset_id=allocated) + assert await snapshot("SELECT key,version FROM teaql_blob_binding WHERE entity_id=?", + (str(allocated),)) == [(None, 3)] + store.close() + + +@pytest.mark.asyncio +async def test_native_legacy_nullable_blob_slot_checks_authoritative_old_value(tmp_path): + path = str(tmp_path / "legacy-blob-slot.db") + descriptor = EntityDescriptor("Asset").table_name("asset") + descriptor.property(PropertyDescriptor("id", DataType.I64).is_id()) + descriptor.property(PropertyDescriptor("version", DataType.I64).is_version()) + descriptor.property(PropertyDescriptor("document", DataType.Text).blob( + BlobFieldSpec("Asset", "document", ["pdf"], True, "default"))) + schema = SimpleSchemaProvider() + schema.register_entity(descriptor) + context = RuntimeModule.new().entity(descriptor).require_capability("blob.v1").into_context() + context.insert_resource("dataService", SqlDataServiceExecutor( + SqliteDialect(), SqliteTransport(path), schema)) + store = FileObjectStorage(tmp_path / "legacy-private", os.urandom(32)) + context.with_object_storage(store) + catalog = SqliteBlobCatalog(SqliteTransport(path)) + await catalog.ensure_schema() + async with aiosqlite.connect(path) as db: + await db.execute("CREATE TABLE asset (id INTEGER PRIMARY KEY, version INTEGER, document TEXT)") + await db.executemany("INSERT INTO asset(id,version,document) VALUES (?,?,?)", + [(10, 1, None), (11, 1, "blob_old_orphan"), + (12, 1, None), (13, 1, "blob_business_old")]) + await db.execute("INSERT INTO teaql_blob_catalog " + "(key,scope,object_key,content_version,file_name,content_type,size_bytes,status) " + "VALUES (?,?,?,?,?,?,?,?)", + ("blob_catalog_other", "tenant-a", "objects/catalog-other", "v1", + "other.pdf", "application/pdf", 7, "available")) + await db.execute("INSERT INTO teaql_blob_binding " + "(scope,entity,entity_id,field,key,version) VALUES (?,?,?,?,?,?)", + ("tenant-a", "Asset", "13", "document", "blob_catalog_other", 1)) + await db.commit() + module = BlobReferenceModule(catalog, + SqliteBlobUploadService(catalog, _UploadAuthority(), _Verifier(), 10_000), + _ReadAuthority(), _BindingAuthority()) + module.install(context) + await module.ready(context) + + async def save(command, asset_id): + async def work(graph): + return await graph.transaction.mutate(graph.context, + graph.request(command, graph.scope(EntityKey("Asset", asset_id)))) + return await context.execute_graph_save(work, comment="legacy nullable Blob mutation") + + with pytest.raises(RuntimeError, match="BLOB_BIND_DENIED"): + await save(UpdateCommand.new("Asset", 11).expected_version(1) + .value("document", None), 11) + with pytest.raises(RuntimeError, match="BLOB_BIND_DENIED"): + await save(DeleteCommand.new("Asset", 11).expected_version(1).hard_delete(), 11) + with pytest.raises(RuntimeError, match="BLOB_BIND_DENIED"): + await save(UpdateCommand.new("Asset", 13).expected_version(1) + .value("document", None), 13) + async with aiosqlite.connect(path) as db: + async with db.execute("SELECT id,version,document FROM asset WHERE id=11") as cursor: + assert await cursor.fetchone() == (11, 1, "blob_old_orphan") + async with db.execute("SELECT count(*) FROM teaql_blob_binding") as cursor: + assert await cursor.fetchone() == (1,) + async with db.execute("SELECT key,version FROM teaql_blob_binding WHERE entity_id='13'") as cursor: + assert await cursor.fetchone() == ("blob_catalog_other", 1) + + await save(UpdateCommand.new("Asset", 12).expected_version(1) + .value("document", None), 12) + async with aiosqlite.connect(path) as db: + async with db.execute("SELECT key,version FROM teaql_blob_binding WHERE entity_id='12'") as cursor: + assert await cursor.fetchone() == (None, 2) + await save(DeleteCommand.new("Asset", 10).expected_version(1).hard_delete(), 10) + async with aiosqlite.connect(path) as db: + async with db.execute("SELECT id FROM asset WHERE id=10") as cursor: + assert await cursor.fetchone() is None + async with db.execute("SELECT count(*) FROM teaql_blob_binding WHERE entity_id='10'") as cursor: + assert await cursor.fetchone() == (0,) + store.close() + + +@pytest.mark.asyncio +async def test_native_executor_keeps_four_argument_blob_participant_contract(tmp_path): + path = str(tmp_path / "custom-participant.db") + descriptor = EntityDescriptor("Asset").table_name("asset") + descriptor.property(PropertyDescriptor("id", DataType.I64).is_id()) + descriptor.property(PropertyDescriptor("version", DataType.I64).is_version()) + descriptor.property(PropertyDescriptor("document", DataType.Text).blob( + BlobFieldSpec("Asset", "document", ["pdf"], True, "default"))) + schema = SimpleSchemaProvider() + schema.register_entity(descriptor) + context = RuntimeModule.new().entity(descriptor).into_context() + context.insert_resource("dataService", SqlDataServiceExecutor( + SqliteDialect(), SqliteTransport(path), schema)) + async with aiosqlite.connect(path) as db: + await db.execute("CREATE TABLE asset (id INTEGER PRIMARY KEY, version INTEGER, document TEXT)") + await db.commit() + + class ExistingParticipant: + def __init__(self): + self.calls = 0 + + async def apply(self, _context, _descriptor, _mutation, persisted): + assert persisted["document"] is None + self.calls += 1 + + participant = ExistingParticipant() + context.insert_resource("blob_mutation_participant", participant) + + async def save(graph): + command = InsertCommand.new("Asset").value("id", 1).value("document", None) + return await graph.transaction.mutate(graph.context, + graph.request(command, graph.scope(EntityKey("Asset", 1)))) + + await context.execute_graph_save(save, comment="four argument Blob participant") + assert participant.calls == 1 + async with aiosqlite.connect(path) as db: + async with db.execute("SELECT id,document FROM asset") as cursor: + assert await cursor.fetchone() == (1, None) + + +@pytest.mark.asyncio +async def test_native_blob_multi_entity_graph_rolls_back_together(tmp_path): + path = str(tmp_path / "multi-native.db") + descriptor = EntityDescriptor("Asset").table_name("asset") + descriptor.property(PropertyDescriptor("id", DataType.I64).is_id()) + descriptor.property(PropertyDescriptor("version", DataType.I64).is_version()) + for field in ("document", "legacy_document"): + descriptor.property(PropertyDescriptor(field, DataType.Text).blob( + BlobFieldSpec("Asset", field, ["pdf"], True, "default"))) + schema = SimpleSchemaProvider() + schema.register_entity(descriptor) + context = RuntimeModule.new().entity(descriptor).require_capability("blob.v1").into_context() + context.insert_resource("dataService", SqlDataServiceExecutor( + SqliteDialect(), SqliteTransport(path), schema)) + context.disable_sql_log() + store = FileObjectStorage(tmp_path / "multi-private", os.urandom(32)) + context.with_object_storage(store) + catalog = SqliteBlobCatalog(SqliteTransport(path)) + await catalog.ensure_schema() + refs = [BlobRef(f"blob_multi_{suffix}") for suffix in ("first", "second", "third")] + async with aiosqlite.connect(path) as db: + await db.execute("CREATE TABLE asset (id INTEGER PRIMARY KEY, version INTEGER, " + "document TEXT, legacy_document TEXT)") + for ref in refs: + await db.execute("INSERT INTO teaql_blob_catalog " + "(key,scope,object_key,content_version,file_name,content_type,size_bytes,status," + "completed_at_ms,policy_profile,upload_mode,unbound_retention_seconds,effective_unref_seconds) " + "VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?)", + (ref.key, "tenant-a", f"objects/{ref.key}", "v1", f"{ref.key}.pdf", + "application/pdf", 7, "uploaded_unbound", time.time_ns() // 1_000_000, + "default", "server_stream", 86400, 604800)) + await db.commit() + + class MultiBindingAuthority(_BindingAuthority): + def __init__(self): + self.denied_id = None + self.denied = False + self.visited = [] + + async def authorize(self, context, target, previous, next_ref, action): + self.visited.append((target.entity_id, target.field)) + if target.entity_id == self.denied_id and target.field == "legacy_document": + self.denied = True + return False + return target.entity == "Asset" + + authority = MultiBindingAuthority() + upload = SqliteBlobUploadService(catalog, _UploadAuthority(), _Verifier(), 10_000) + module = BlobReferenceModule(catalog, upload, _ReadAuthority(), authority) + module.install(context) + await module.ready(context) + allocated = [] + + async def save(*, catch_denial=False, outer_reject=False): + allocated.clear() + authority.visited.clear() + authority.denied_id = None + authority.denied = False + ledger = EntityRoot() + provisional = [EntityKey("Asset", -1), EntityKey("Asset", -2)] + for key in provisional: + ledger.mark_as_new(key) + ledger.set(provisional[0], "document", refs[0]) + ledger.set(provisional[1], "document", refs[1]) + ledger.set(provisional[1], "legacy_document", refs[2]) + + async def work(graph): + for _ in provisional: + allocated.append(await graph.transaction.next_id("Asset")) + if catch_denial: + authority.denied_id = str(allocated[1]) + for index, old in enumerate(provisional): + actual = EntityKey("Asset", allocated[index]) + ledger.rekey(old, actual) + assert ledger.get(actual, "document") == refs[index] + command = (InsertCommand.new("Asset").value("id", allocated[index]) + .value("document", ledger.get(actual, "document"))) + if index == 1: + command.value("legacy_document", ledger.get(actual, "legacy_document")) + try: + await graph.transaction.mutate(graph.context, + graph.request(command, graph.scope(actual), ledger, actual)) + except RuntimeError: + if catch_denial: + return + raise + if outer_reject: + raise RuntimeError("outer owner rejected") + + await context.execute_graph_save(work, comment="save multi-entity Blob graph") + return tuple(allocated) + + async def assert_rollback(ids): + async with aiosqlite.connect(path) as db: + async with db.execute("SELECT COUNT(*) FROM asset WHERE id IN (?,?)", ids) as cursor: + assert await cursor.fetchone() == (0,) + async with db.execute("SELECT COUNT(*) FROM teaql_blob_binding") as cursor: + assert await cursor.fetchone() == (0,) + async with db.execute("SELECT key,status FROM teaql_blob_catalog ORDER BY key") as cursor: + assert await cursor.fetchall() == [(ref.key, "uploaded_unbound") for ref in refs] + + try: + with pytest.raises(RuntimeError, match="GRAPH_TRANSACTION_ROLLBACK_ONLY"): + await save(catch_denial=True) + assert authority.denied + assert authority.visited == [ + (str(allocated[0]), "document"), (str(allocated[0]), "legacy_document"), + (str(allocated[1]), "document"), (str(allocated[1]), "legacy_document")] + await assert_rollback(tuple(allocated)) + + with pytest.raises(RuntimeError, match="outer owner rejected"): + await save(outer_reject=True) + await assert_rollback(tuple(allocated)) + + committed = await save() + assert authority.visited == [ + (str(committed[0]), "document"), (str(committed[0]), "legacy_document"), + (str(committed[1]), "document"), (str(committed[1]), "legacy_document")] + async with aiosqlite.connect(path) as db: + async with db.execute("SELECT id,document,legacy_document FROM asset ORDER BY id") as cursor: + assert await cursor.fetchall() == [ + (committed[0], refs[0].key, None), + (committed[1], refs[1].key, refs[2].key)] + async with db.execute("SELECT entity_id,field,key FROM teaql_blob_binding " + "ORDER BY entity_id,field") as cursor: + assert await cursor.fetchall() == [ + (str(committed[0]), "document", refs[0].key), + (str(committed[0]), "legacy_document", None), + (str(committed[1]), "document", refs[1].key), + (str(committed[1]), "legacy_document", refs[2].key)] + async with db.execute("SELECT key,status FROM teaql_blob_catalog ORDER BY key") as cursor: + assert await cursor.fetchall() == [(ref.key, "available") for ref in refs] + finally: + store.close() + + +@pytest.mark.asyncio +async def test_native_shared_blob_final_unbind_controls_file_retention(tmp_path): + store = FileObjectStorage(tmp_path / "shared-private", os.urandom(32)) + await _run_native_shared_blob_final_unbind(tmp_path, store, real_s3=False) + + +@pytest.mark.asyncio +async def test_native_shared_blob_final_unbind_controls_real_s3_retention(tmp_path): + if os.environ.get("TEAQL_S3_TEST_SHARED_BLOB") != "1": + pytest.skip("explicit shared-reference S3 gate required") + endpoint = os.environ.get("TEAQL_S3_TEST_ENDPOINT") + bucket = os.environ.get("TEAQL_S3_TEST_BUCKET") + access = os.environ.get("TEAQL_S3_TEST_ACCESS_KEY") + secret = os.environ.get("TEAQL_S3_TEST_SECRET_KEY") + assert all((endpoint, bucket, access, secret)), "explicit S3 test configuration required" + from teaql.provider.s3 import S3ObjectStorage + store = S3ObjectStorage.connect(bucket, endpoint_url=endpoint, + aws_access_key_id=access, aws_secret_access_key=secret) + await _run_native_shared_blob_final_unbind(tmp_path, store, real_s3=True) + + +async def _run_native_shared_blob_final_unbind(tmp_path, store, real_s3): + path = str(tmp_path / "shared-native.db") + descriptor = EntityDescriptor("Asset").table_name("asset") + descriptor.property(PropertyDescriptor("id", DataType.I64).is_id()) + descriptor.property(PropertyDescriptor("version", DataType.I64).is_version()) + descriptor.property(PropertyDescriptor("document", DataType.Text).blob( + BlobFieldSpec("Asset", "document", ["pdf"], True, "default"))) + schema = SimpleSchemaProvider() + schema.register_entity(descriptor) + context = RuntimeModule.new().entity(descriptor).require_capability("blob.v1").into_context() + context.insert_resource("dataService", SqlDataServiceExecutor( + SqliteDialect(), SqliteTransport(path), schema)) + context.disable_sql_log() + context.with_object_storage(store).with_object_storage_observer(lambda event: None) + clock = [1_700_000_000_000] + catalog = SqliteBlobCatalog(SqliteTransport(path), now_ms=lambda: clock[0]) + policy = BlobRetentionPolicy("default", "server_stream", 10, 20, 0) + + class UploadAuthority: + def scope(self, _context): + return "tenant-a" + + def field(self, _context, _target): + return BlobFieldSpec("Asset", "document", ["pdf"], True, "default") + + async def authorize(self, _context, target, action, _input): + return target.entity == "Asset" and target.field == "document" and action in ( + "Begin", "Write", "Confirm") + + async def current_policy_on_lease(self, _context, lease, _target): + assert lease is not None + return policy + + class BindingAuthority(_BindingAuthority): + async def current_policy_on_lease(self, _context, lease, _target): + assert lease is not None + return policy + + class FixtureVerifier: + def verify(self, stream, extension, maximum_bytes): + data = stream.read(maximum_bytes + 1) + assert extension == "pdf" and len(data) <= maximum_bytes + assert data.startswith(b"%PDF-") and b"%%EOF" in data + assert stream.read(1) == b"" + return BlobVerifiedContent(len(data), sha256(data).hexdigest(), "application/pdf") + + upload = SqliteBlobUploadService(catalog, UploadAuthority(), FixtureVerifier(), 10_000, + now_ms=lambda: clock[0]) + maintenance = SqliteBlobMaintenanceService(catalog, _MaintenanceAuthority(), + now_ms=lambda: clock[0]) + read_permission = _ReadAuthority() + module = BlobReferenceModule(catalog, upload, read_permission, BindingAuthority(), + maintenance=maintenance) + async with aiosqlite.connect(path) as db: + await db.execute("CREATE TABLE asset (id INTEGER PRIMARY KEY, version INTEGER, document TEXT)") + await db.commit() + await catalog.ensure_schema() + module.install(context) + await module.ready(context) + + async def row(sql, params=()): + async with aiosqlite.connect(path) as db: + async with db.execute(sql, params) as cursor: + return await cursor.fetchone() + + object_key = None + try: + data = b"%PDF-1.7\nshared native Blob\n%%EOF" + session = await upload.begin(context, BlobUploadTarget("Asset", "1", "document"), + BlobUploadInput("shared.pdf", len(data), "application/pdf"), + "shared-native-blob") + assert await upload.write(context, session.upload_id, BytesIO(data)) == len(data) + assert await upload.complete(context, session.upload_id) == session.reference + object_key = ObjectKey((await row("SELECT object_key FROM teaql_blob_catalog WHERE key=?", + (session.reference.key,)))[0]) + object_store = context.object_storage("shared native Blob graph") + assert object_store.head(object_key) is not None + + ledger = EntityRoot() + provisional = [EntityKey("Asset", -1), EntityKey("Asset", -2)] + for key in provisional: + ledger.mark_as_new(key) + ledger.set(key, "document", session.reference) + + async def bind(graph): + ids = [] + for key in provisional: + allocated = await graph.transaction.next_id("Asset") + actual = EntityKey("Asset", allocated) + ledger.rekey(key, actual) + command = (InsertCommand.new("Asset").value("id", allocated) + .value("document", ledger.get(actual, "document"))) + await graph.transaction.mutate(graph.context, + graph.request(command, graph.scope(actual), ledger, actual)) + ids.append(allocated) + return ids + + ids = await context.execute_graph_save(bind, comment="bind shared Blob in two native rows") + assert len(ids) == 2 + assert await row("SELECT COUNT(*) FROM teaql_blob_binding WHERE key=?", + (session.reference.key,)) == (2,) + assert await row("SELECT status,last_unbound_at_ms FROM teaql_blob_catalog WHERE key=?", + (session.reference.key,)) == ("available", None) + + clock[0] += 40_000 + + async def unbind(id, outer_reject=False): + async def work(graph): + command = UpdateCommand.new("Asset", id).value("document", None) + await graph.transaction.mutate(graph.context, + graph.request(command, graph.scope(EntityKey("Asset", id)))) + if outer_reject: + raise RuntimeError("outer owner rejected final unbind") + await context.execute_graph_save(work, comment="unbind shared Blob row") + + await unbind(ids[0]) + assert await row("SELECT COUNT(*) FROM teaql_blob_binding WHERE key=?", + (session.reference.key,)) == (1,) + assert await row("SELECT last_unbound_at_ms FROM teaql_blob_catalog WHERE key=?", + (session.reference.key,)) == (None,) + assert (await maintenance.run_batch(context)).outcomes == () + assert object_store.head(object_key) is not None + + if real_s3: + class Authorizer: + async def authorize(self, _context, _result, _options): + slot = AuthorizedBlobSlot(("document",), "Asset", str(ids[1]), "document", + "tenant-a", 1, session.reference) + return AuthorizedResponseGraph({"document": session.reference}, (slot,)) + + context.insert_resource("response_graph_authorizer", Authorizer()) + options = ViewOptions("ordinary", "read_only", + datetime.now(timezone.utc) + timedelta(seconds=60)) + view = await ViewPreparationService().prepare(context, None, options) + link = view.presentation["document"] + assert link.startswith("http") and view.cache_control == "private, no-store" + with urlopen(Request(link, headers={"Range": "bytes=0-7"}), timeout=5) as response: + assert response.status == 206 and response.read() == data[:8] + read_permission.allowed = False + with pytest.raises(ViewPreparationError) as denied: + await ViewPreparationService().prepare(context, None, options) + assert denied.value.code == "BLOB_READ_DENIED" + read_permission.allowed = True + + with pytest.raises(RuntimeError, match="outer owner rejected final unbind"): + await unbind(ids[1], outer_reject=True) + assert await row("SELECT document FROM asset WHERE id=?", (ids[1],)) == (session.reference.key,) + assert await row("SELECT COUNT(*) FROM teaql_blob_binding WHERE key=?", + (session.reference.key,)) == (1,) + assert await row("SELECT last_unbound_at_ms FROM teaql_blob_catalog WHERE key=?", + (session.reference.key,)) == (None,) + assert (await maintenance.run_batch(context)).outcomes == () + + await unbind(ids[1]) + assert await row("SELECT COUNT(*) FROM teaql_blob_binding WHERE key=?", + (session.reference.key,)) == (0,) + assert await row("SELECT last_unbound_at_ms FROM teaql_blob_catalog WHERE key=?", + (session.reference.key,)) == (clock[0],) + if real_s3: + with pytest.raises(ViewPreparationError) as stale_view: + await ViewPreparationService().prepare(context, None, options) + assert stale_view.value.code == "BLOB_READ_DENIED" + clock[0] += 19_999 + assert (await maintenance.run_batch(context)).outcomes == () + assert object_store.head(object_key) is not None + clock[0] += 1 + batch = await maintenance.run_batch(context) + assert [(item.reference.key, item.status) for item in batch.outcomes] == [ + (session.reference.key, "deleted")] + assert object_store.head(object_key) is None + assert await row("SELECT status FROM teaql_blob_catalog WHERE key=?", + (session.reference.key,)) == ("deleted",) + if real_s3: + print("PASS Python shared S3 native graph/final unbind/authorized Range/retention GC") + finally: + try: + if real_s3 and object_key is not None: + context.object_storage("clean shared S3 Blob").delete(object_key) + finally: + store.close() diff --git a/tests/provider/sqlite/test_blob_retention_snapshot.py b/tests/provider/sqlite/test_blob_retention_snapshot.py new file mode 100644 index 00000000..db348818 --- /dev/null +++ b/tests/provider/sqlite/test_blob_retention_snapshot.py @@ -0,0 +1,146 @@ +"""Trusted Blob retention facts on a real SQLite lease and File provider.""" + +from hashlib import sha256 +from io import BytesIO +from pathlib import Path +import json +import os + +import aiosqlite +import pytest + +from teaql.core.blob_ref import BlobFieldSpec +from teaql.provider.sqlite.transport import SqliteTransport +from teaql.runtime import RuntimeModule +from teaql.sql.types import CompiledQuery +from teaql_blob_reference import ( + BlobRetentionPolicy, BlobUploadInput, BlobUploadTarget, BlobVerifiedContent, + FileObjectStorage, SqliteBlobCatalog, SqliteBlobUploadService, +) +from teaql_blob_reference.retention import ( + effective_retention, require_bindable, retention_deadline, + unreferenced_eligible_at, +) + + +class Authority: + def __init__(self): + self.policy = BlobRetentionPolicy("default", "server_stream", 86400, 604800, 3600) + self.policy_calls = 0 + + def scope(self, _actor): + return "tenant-a" + + def field(self, _actor, _target): + return BlobFieldSpec("Asset", "document", ("pdf",), True, "default") + + async def authorize(self, _actor, _target, _action, _input): + return True + + async def current_policy_on_lease(self, _actor, lease, _target): + rows = await lease.fetch_all_sql(CompiledQuery("SELECT 1 AS alive", [])) + assert rows == [{"alive": 1}] + self.policy_calls += 1 + return self.policy + + +class Verifier: + def verify(self, stream, extension, maximum_bytes): + data = stream.read(maximum_bytes + 1) + assert extension == "pdf" and data.startswith(b"%PDF-") + assert stream.read(1) == b"" + return BlobVerifiedContent(len(data), sha256(data).hexdigest(), "application/pdf") + + +@pytest.mark.asyncio +async def test_policy_snapshotted_on_lease_completion_time_and_replay(tmp_path): + path = str(tmp_path / "catalog.db") + catalog = SqliteBlobCatalog(SqliteTransport(path)) + await catalog.ensure_schema() + await catalog.validate_schema() + authority = Authority() + clock = [1_700_000_000_000] + actor = RuntimeModule.new().into_context().with_object_storage( + FileObjectStorage(tmp_path / "private", os.urandom(32))) + actor.with_object_storage_observer(lambda _event: None) + service = SqliteBlobUploadService(catalog, authority, Verifier(), 10_000, + now_ms=lambda: clock[0]) + data = b"%PDF-1.7\nretention snapshot\n%%EOF" + target = BlobUploadTarget("Asset", "1", "document") + request = BlobUploadInput("report.pdf", len(data)) + session = await service.begin(actor, target, request, "snapshot-1") + assert authority.policy_calls == 1 + async with aiosqlite.connect(path) as db: + async with db.execute("SELECT policy_profile,upload_mode,unbound_retention_seconds," + "effective_unref_seconds,completed_at_ms,last_unbound_at_ms " + "FROM teaql_blob_catalog WHERE key=?", (session.reference.key,)) as cursor: + assert await cursor.fetchone() == ("default", "server_stream", 86400, 604800, None, None) + authority.policy = BlobRetentionPolicy("new-profile", "server_stream", 1, 1, 1) + assert (await service.begin(actor, target, request, "snapshot-1")).upload_id == session.upload_id + assert authority.policy_calls == 1 + await service.write(actor, session.upload_id, BytesIO(data)) + clock[0] += 30_000 + assert await service.complete(actor, session.upload_id) == session.reference + assert await service.complete(actor, session.upload_id) == session.reference + async with aiosqlite.connect(path) as db: + async with db.execute("SELECT policy_profile,unbound_retention_seconds," + "effective_unref_seconds,completed_at_ms,status " + "FROM teaql_blob_catalog WHERE key=?", (session.reference.key,)) as cursor: + assert await cursor.fetchone() == ("default", 86400, 604800, clock[0], "uploaded_unbound") + + +@pytest.mark.asyncio +async def test_migration_keeps_legacy_policy_unknown_and_direct_mode_rejected(tmp_path): + path = str(tmp_path / "old.db") + async with aiosqlite.connect(path) as db: + await db.execute("CREATE TABLE teaql_blob_catalog (key TEXT PRIMARY KEY, scope TEXT NOT NULL, " + "object_key TEXT NOT NULL, content_version TEXT NOT NULL, file_name TEXT NOT NULL, " + "content_type TEXT NOT NULL, size_bytes BIGINT NOT NULL, status TEXT NOT NULL)") + await db.execute("INSERT INTO teaql_blob_catalog VALUES (?,?,?,?,?,?,?,?)", + ("blob_legacy", "tenant-a", "blob/legacy", "v1", "old.pdf", "application/pdf", 10, "available")) + await db.commit() + catalog = SqliteBlobCatalog(SqliteTransport(path)) + await catalog.ensure_schema() + await catalog.ensure_schema() + await catalog.validate_schema() + async with aiosqlite.connect(path) as db: + async with db.execute("SELECT completed_at_ms,last_unbound_at_ms,policy_profile,upload_mode," + "unbound_retention_seconds,effective_unref_seconds,retention_floor_until_ms " + "FROM teaql_blob_catalog WHERE key='blob_legacy'") as cursor: + assert await cursor.fetchone() == (None,) * 7 + authority = Authority() + authority.policy = BlobRetentionPolicy("default", "direct", 1, 1, 1) + service = SqliteBlobUploadService(catalog, authority, Verifier(), 10_000) + with pytest.raises(RuntimeError, match="^BLOB_CAPABILITY_MISSING$"): + await service.begin(RuntimeModule.new().into_context(), BlobUploadTarget("Asset", "1", "document"), + BlobUploadInput("direct.pdf", 4), "direct-mode") + assert authority.policy_calls == 1 + async with aiosqlite.connect(path) as db: + async with db.execute("SELECT count(*) FROM teaql_blob_upload") as cursor: + assert await cursor.fetchone() == (0,) + + +def test_shared_retention_vectors(): + fixture = Path(__file__).with_name("testdata") / "blob-policy-cases.json" + assert sha256(fixture.read_bytes()).hexdigest() == ( + "b93cff3f8dabd5690aac5f8f2096633dcd4e158d3aa0424b9a902a16f6d3c2b6") + cases = {case["id"]: case for case in json.loads(fixture.read_text())["cases"]} + upload = cases["upload-clock-independent"] + assert upload["input"]["nowSeconds"] >= 3600 + assert retention_deadline(upload["input"]["completedAtSeconds"] * 1000, 86400) == ( + upload["expected"]["unboundEligibleAtSeconds"] * 1000) + expired = cases["unbound-deadline-rejects-bind-before-gc"] + with pytest.raises(RuntimeError, match="^BLOB_REFERENCE_EXPIRED$"): + require_bindable("uploaded_unbound", expired["input"]["nowSeconds"] * 1000, + expired["input"]["completedAtSeconds"] * 1000, + expired["input"]["unboundRetentionSeconds"], None) + maximum = cases["maximum-retention-across-fields"] + seconds = effective_retention(tuple(maximum["input"]["acceptedRetentionSeconds"]), 0) + assert seconds == maximum["expected"]["effectiveUnreferencedRetentionSeconds"] + assert unreferenced_eligible_at(maximum["input"]["lastUnboundAtSeconds"] * 1000, + seconds, maximum["input"]["retentionFloorUntilSeconds"] * 1000) == ( + maximum["expected"]["unreferencedEligibleAtSeconds"] * 1000) + shorter = cases["profile-update-does-not-shorten"] + assert effective_retention((shorter["input"]["acceptedUnreferencedRetentionSeconds"],), + shorter["input"]["currentProfileRetentionSeconds"]) == ( + shorter["expected"]["effectiveUnreferencedRetentionSeconds"]) diff --git a/tests/provider/sqlite/test_blob_upload.py b/tests/provider/sqlite/test_blob_upload.py new file mode 100644 index 00000000..58c81ff2 --- /dev/null +++ b/tests/provider/sqlite/test_blob_upload.py @@ -0,0 +1,1027 @@ +"""Real SQLite/File upload confirmation; parser remains test-only.""" + +from datetime import datetime, timedelta, timezone +from hashlib import sha256 +from io import BytesIO +from pathlib import Path +from shutil import which +from threading import Event +from urllib.request import Request, urlopen +from uuid import uuid4 +import asyncio +import os +import time + +import aiosqlite +import pytest + +from teaql.core.blob_ref import BlobFieldSpec +from teaql.core.entity import EntityKey +from teaql.core.meta import EntityDescriptor, PropertyDescriptor +from teaql.core.mutation import InsertCommand, UpdateCommand +from teaql.core.object_storage import ObjectContent, ObjectKey, PutOptions, WriteMode +from teaql.core.value import DataType +from teaql.provider.sqlite import SimpleSchemaProvider +from teaql.provider.sqlite.dialect import SqliteDialect +from teaql.provider.sqlite.transport import SqliteTransport +from teaql.runtime import RuntimeModule +from teaql.runtime.view_preparation import ( + AuthorizedBlobSlot, AuthorizedResponseGraph, ViewOptions, + ViewPreparationError, ViewPreparationService, +) +from teaql.sql.executor import SqlDataServiceExecutor +from teaql.sql.types import CompiledQuery +from teaql_blob_reference import ( + BlobBindingChange, BlobBindingTarget, BlobReferenceModule, BlobRetentionPolicy, + BlobUploadAuthorizationState, + BlobUploadInput, BlobUploadTarget, + BlobVerifiedContent, FileObjectStorage, SqliteBlobCatalog, + SqliteBlobMaintenanceService, SqliteBlobUploadService, +) + + +class UploadAuthority: + def __init__(self, extensions=("pdf",)): + self.current_scope = "tenant-a" + self.allowed = True + self.extensions = extensions + + def scope(self, _context): + return self.current_scope + + def field(self, _context, _target): + return BlobFieldSpec("Asset", "document", self.extensions, True, "default") + + async def authorize(self, _context, target, action, _input): + return (self.allowed and target.entity == "Asset" and target.field == "document" + and action in ("Begin", "Write", "Confirm")) + + async def current_policy_on_lease(self, _context, lease, _target): + assert lease is not None + return BlobRetentionPolicy("default", "server_stream", 86400, 604800, 3600) + + +class BindingAuthority: + def scope(self, _context): + return "tenant-a" + + async def authorize(self, _context, target, _previous, _next, action): + return target.entity == "Asset" and target.field == "document" and action == "Create" + + async def current_policy_on_lease(self, _context, lease, _target): + assert lease is not None + return BlobRetentionPolicy("default", "server_stream", 86400, 604800, 3600) + + +class FixturePDFVerifier: + def __init__(self, early=False): + self.early = early + self.calls = 0 + + def verify(self, stream, extension, maximum_bytes): + self.calls += 1 + if self.early: + return BlobVerifiedContent(1, "0" * 64, "application/pdf") + data = stream.read(maximum_bytes + 1) + assert len(data) <= maximum_bytes and extension == "pdf" + assert data.startswith(b"%PDF-") and b"%%EOF" in data + assert stream.read(1) == b"" + return BlobVerifiedContent(len(data), sha256(data).hexdigest(), "application/pdf") + + +@pytest.mark.asyncio +async def test_upload_begin_commit_unknown_replays_original_live_session(tmp_path): + path = str(tmp_path / "begin-reply-lost.db") + catalog = SqliteBlobCatalog(SqliteTransport(path)) + await catalog.ensure_schema() + actor = RuntimeModule.new().into_context() + authority = UploadAuthority() + now = [10000] + service = SqliteBlobUploadService(catalog, authority, FixturePDFVerifier(), 1000, + now_ms=lambda: now[0]) + target = BlobUploadTarget("Asset", "1", "document") + upload = BlobUploadInput("report.pdf", 25) + begin_sql = catalog.transport.begin_sql + + class CommitThenFail: + def __init__(self, tx): + self.tx = tx + + def __getattr__(self, name): + return getattr(self.tx, name) + + async def commit_sql(self): + await self.tx.commit_sql() + raise OSError("PRIVATE-BEGIN-REPLY-CANARY") + + async def uncertain_begin(): + return CommitThenFail(await begin_sql()) + + catalog.transport.begin_sql = uncertain_begin + try: + with pytest.raises(RuntimeError, match="^BLOB_CATALOG_FAILED$") as failure: + await service.begin(actor, target, upload, "lost-begin") + assert "CANARY" not in str(failure.value) + finally: + catalog.transport.begin_sql = begin_sql + + async with aiosqlite.connect(path) as db: + async with db.execute("""SELECT u.upload_id,u.key,u.expires_ms,c.status,c.upload_mode + FROM teaql_blob_upload u JOIN teaql_blob_catalog c ON c.key=u.key + WHERE u.scope=? AND u.replay_key=?""", ("tenant-a", "lost-begin")) as cursor: + rows = await cursor.fetchall() + assert len(rows) == 1 + upload_id, key, expires_ms, status, mode = rows[0] + assert status == "initiated" and mode == "server_stream" + + authority.allowed = False + with pytest.raises(RuntimeError, match="^BLOB_UPLOAD_DENIED$"): + await service.begin(actor, target, upload, "lost-begin") + authority.allowed = True + with pytest.raises(RuntimeError, match="^BLOB_IDEMPOTENCY_CONFLICT$"): + await service.begin(actor, target, BlobUploadInput("changed.pdf", 25), "lost-begin") + replay = await service.begin(actor, target, upload, "lost-begin") + assert (replay.upload_id, replay.reference.key, + int(replay.expires_at.timestamp() * 1000)) == (upload_id, key, expires_ms) + now[0] = expires_ms + with pytest.raises(RuntimeError, match="^BLOB_UPLOAD_EXPIRED$"): + await service.begin(actor, target, upload, "lost-begin") + async with aiosqlite.connect(path) as db: + async with db.execute("SELECT COUNT(*) FROM teaql_blob_upload") as cursor: + assert (await cursor.fetchone())[0] == 1 + + +@pytest.mark.asyncio +async def test_upload_write_closes_source_once_before_promotion_and_replay(tmp_path): + class CountedSource(BytesIO): + def __init__(self, data, *, fail_close=False): + super().__init__(data) + self.close_count = 0 + self.fail_close = fail_close + + def close(self): + self.close_count += 1 + super().close() + if self.fail_close: + raise OSError("PRIVATE-UPLOAD-SOURCE-CLOSE-CANARY") + if self.close_count > 1: + raise OSError("PRIVATE-DOUBLE-CLOSE-CANARY") + + path = str(tmp_path / "write-source-close.db") + catalog = SqliteBlobCatalog(SqliteTransport(path)) + await catalog.ensure_schema() + actor = RuntimeModule.new().into_context().with_object_storage( + FileObjectStorage(tmp_path / "private", os.urandom(32))) + actor.with_object_storage_observer(lambda event: None) + service = SqliteBlobUploadService(catalog, UploadAuthority(), FixturePDFVerifier(), 10_000) + data = b"%PDF-1.7\nsource close before promotion\n%%EOF" + session = await service.begin(actor, BlobUploadTarget("Asset", "1", "document"), + BlobUploadInput("report.pdf", len(data)), "source-close") + failed = CountedSource(data, fail_close=True) + with pytest.raises(RuntimeError, match="^BLOB_STORE_FAILED$"): + await service.write(actor, session.upload_id, failed) + assert failed.close_count == 1 + async with aiosqlite.connect(path) as db: + async with db.execute("""SELECT u.state,c.status FROM teaql_blob_upload u + JOIN teaql_blob_catalog c ON c.key=u.key WHERE u.upload_id=?""", + (session.upload_id,)) as cursor: + assert await cursor.fetchone() == ("initiated", "initiated") + retry = CountedSource(data) + assert await service.write(actor, session.upload_id, retry) == len(data) + assert retry.close_count == 1 + assert await service.complete(actor, session.upload_id) == session.reference + replay = CountedSource(data) + assert await service.write(actor, session.upload_id, replay) == len(data) + assert replay.close_count == 1 + + +@pytest.mark.asyncio +async def test_upload_confirm_close_failure_rolls_back_and_retries(tmp_path): + class CloseFailStream: + def __init__(self, source): + self.source = source + + def read(self, size=-1): + return self.source.read(size) + + def close(self): + self.source.close() + raise OSError("PRIVATE-OBJECT-CLOSE-CANARY") + + class CloseFailStore(FileObjectStorage): + fail_close = True + + def get(self, invocation, key): + content = super().get(invocation, key) + if self.fail_close: + return ObjectContent(content.metadata, CloseFailStream(content.stream)) + return content + + path = str(tmp_path / "confirm-close-failure.db") + catalog = SqliteBlobCatalog(SqliteTransport(path)) + await catalog.ensure_schema() + store = CloseFailStore(tmp_path / "private", os.urandom(32)) + actor = RuntimeModule.new().into_context().with_object_storage(store) + actor.with_object_storage_observer(lambda event: None) + service = SqliteBlobUploadService(catalog, UploadAuthority(), FixturePDFVerifier(), 10_000) + target = BlobUploadTarget("Asset", "1", "document") + data = b"%PDF-1.7\nclose failure must not promote\n%%EOF" + session = await service.begin(actor, target, BlobUploadInput("report.pdf", len(data)), "close-failure") + assert await service.write(actor, session.upload_id, BytesIO(data)) == len(data) + with pytest.raises(RuntimeError, match="^BLOB_UPLOAD_INCOMPLETE$") as failure: + await service.complete(actor, session.upload_id) + assert "CANARY" not in str(failure.value) + async with aiosqlite.connect(path) as db: + async with db.execute("""SELECT u.state,c.status FROM teaql_blob_upload u + JOIN teaql_blob_catalog c ON c.key=u.key WHERE u.upload_id=?""", + (session.upload_id,)) as cursor: + assert await cursor.fetchone() == ("written", "initiated") + store.fail_close = False + assert await service.complete(actor, session.upload_id) == session.reference + + +@pytest.mark.asyncio +@pytest.mark.parametrize("case", ["fixture-pdf", "real-jpg", "real-jpeg", "real-png", "real-pdf", + "real-docx", "real-xlsx", "real-doc", "real-xls", + "real-mp3", "real-mp4"]) +async def test_upload_real_sqlite_s3_create_only_confirm_and_replay(tmp_path, case): + if os.environ.get("TEAQL_S3_TEST_BLOB_UPLOAD") != "1": + pytest.skip("explicit S3-compatible Blob upload gate required") + if case in ("real-jpg", "real-jpeg", "real-png") and os.environ.get("TEAQL_S3_TEST_REAL_IMAGE") != "1": + pytest.skip("explicit production image S3 gate required") + if case == "real-pdf" and os.environ.get("TEAQL_S3_TEST_REAL_PDF") != "1": + pytest.skip("explicit production PDF S3 gate required") + if case in ("real-docx", "real-xlsx") and os.environ.get("TEAQL_S3_TEST_REAL_OFFICE") != "1": + pytest.skip("explicit production Office S3 gate required") + if case in ("real-doc", "real-xls") and os.environ.get("TEAQL_S3_TEST_REAL_LEGACY_OFFICE") != "1": + pytest.skip("explicit production legacy Office S3 gate required") + if case in ("real-mp3", "real-mp4") and os.environ.get("TEAQL_S3_TEST_REAL_MEDIA") != "1": + pytest.skip("explicit production media S3 gate required") + endpoint = os.environ.get("TEAQL_S3_TEST_ENDPOINT") + bucket = os.environ.get("TEAQL_S3_TEST_BUCKET") + access = os.environ.get("TEAQL_S3_TEST_ACCESS_KEY") + secret = os.environ.get("TEAQL_S3_TEST_SECRET_KEY") + assert all((endpoint, bucket, access, secret)), "explicit S3 test configuration required" + from teaql.provider.s3 import S3ObjectStorage + + provider = S3ObjectStorage.connect(bucket, endpoint_url=endpoint, + aws_access_key_id=access, aws_secret_access_key=secret) + path = str(tmp_path / "s3-upload.db") + descriptor = EntityDescriptor("Asset").table_name("asset") + descriptor.property(PropertyDescriptor("id", DataType.I64).is_id()) + descriptor.property(PropertyDescriptor("version", DataType.I64).is_version()) + descriptor.property(PropertyDescriptor("name", DataType.Text)) + descriptor.property(PropertyDescriptor("document", DataType.Text).blob( + BlobFieldSpec("Asset", "document", + ["doc", "docx", "jpeg", "jpg", "mp3", "mp4", "pdf", "png", "xls", "xlsx"], + True, "default"))) + schema = SimpleSchemaProvider() + schema.register_entity(descriptor) + actor = RuntimeModule.new().entity(descriptor).into_context() + actor.insert_resource("dataService", SqlDataServiceExecutor( + SqliteDialect(), SqliteTransport(path), schema)) + actor.with_object_storage(provider).with_object_storage_observer(lambda event: None) + clock = [time.time_ns() // 1_000_000] + catalog = SqliteBlobCatalog(SqliteTransport(path), now_ms=lambda: clock[0]) + if case in ("real-jpg", "real-jpeg", "real-png"): + from teaql_blob_reference.content_pillow import PillowBlobContentVerifier + extension = case.removeprefix("real-") + authority, parser = UploadAuthority((extension,)), PillowBlobContentVerifier() + fixture = "png" if extension == "png" else "jpg" + data = (Path(__file__).parents[2] / "fixtures" / ("blob-sample." + fixture)).read_bytes() + file_name = "sample." + extension + content_type = "image/png" if extension == "png" else "image/jpeg" + elif case == "real-pdf": + from teaql_blob_reference.content_pypdf import PypdfBlobContentVerifier + authority, parser = UploadAuthority(("pdf",)), PypdfBlobContentVerifier() + data = (Path(__file__).parents[2] / "fixtures" / "blob-sample.pdf").read_bytes() + file_name, content_type = "sample.pdf", "application/pdf" + elif case in ("real-docx", "real-xlsx"): + from teaql_blob_reference.content_office import OfficeBlobContentVerifier + extension = case.removeprefix("real-") + authority, parser = UploadAuthority((extension,)), OfficeBlobContentVerifier() + output = BytesIO() + if extension == "docx": + from docx import Document + document = Document() + document.add_paragraph("TeaQL S3 Office upload") + document.save(output) + content_type = "application/vnd.openxmlformats-officedocument.wordprocessingml.document" + else: + from openpyxl import Workbook + workbook = Workbook() + workbook.active["A1"] = "TeaQL S3 Office upload" + workbook.save(output) + content_type = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" + data, file_name = output.getvalue(), "sample." + extension + elif case in ("real-doc", "real-xls"): + from teaql_blob_reference.content_legacy_office import LegacyOfficeBlobContentVerifier + soffice, bwrap, prlimit = (which(name) for name in ("soffice", "bwrap", "prlimit")) + if not all((soffice, bwrap, prlimit)): + pytest.skip("host LibreOffice, bubblewrap and prlimit required") + extension = case.removeprefix("real-") + scratch = tmp_path / "legacy-scratch" + scratch.mkdir(mode=0o700) + scratch.chmod(0o700) + authority = UploadAuthority((extension,)) + parser = LegacyOfficeBlobContentVerifier(soffice, bwrap, prlimit, scratch, + maximum_input_bytes=100_000, + maximum_converted_bytes=100_000) + data = (Path(__file__).parents[2] / "fixtures" / "reference-modules" / + ("sample." + extension)).read_bytes() + file_name = "sample." + extension + content_type = "application/msword" if extension == "doc" else "application/vnd.ms-excel" + elif case in ("real-mp3", "real-mp4"): + from teaql_blob_reference.content_ffmpeg import FfmpegBlobContentVerifier + ffprobe, ffmpeg = which("ffprobe"), which("ffmpeg") + if not ffprobe or not ffmpeg: + pytest.skip("host FFmpeg command-line tools required") + extension = case.removeprefix("real-") + authority = UploadAuthority((extension,)) + parser = FfmpegBlobContentVerifier(ffprobe, ffmpeg) + data = (Path(__file__).parents[2] / "fixtures" / "reference-modules" / + ("sample." + extension)).read_bytes() + file_name = "sample." + extension + content_type = "audio/mpeg" if extension == "mp3" else "video/mp4" + else: + authority, parser = UploadAuthority(), FixturePDFVerifier() + data = b"%PDF-1.7\nreal S3 Blob upload\n%%EOF" + file_name, content_type = "report.pdf", "application/pdf" + + class CountingVerifier: + def __init__(self, inner): + self.inner = inner + self.calls = 0 + + def verify(self, stream, extension, maximum_bytes): + self.calls += 1 + return self.inner.verify(stream, extension, maximum_bytes) + + verifier = CountingVerifier(parser) + service = SqliteBlobUploadService(catalog, authority, verifier, 100_000, + now_ms=lambda: clock[0]) + object_key = None + try: + async with aiosqlite.connect(path) as db: + await db.execute("CREATE TABLE asset (id INTEGER PRIMARY KEY, version INTEGER, name TEXT, document TEXT)") + await db.commit() + await catalog.ensure_schema() + session = await service.begin(actor, BlobUploadTarget("Asset", "1", "document"), + BlobUploadInput(file_name, len(data), content_type), + uuid4().hex) + async with aiosqlite.connect(path) as db: + async with db.execute("SELECT object_key FROM teaql_blob_catalog WHERE key=?", + (session.reference.key,)) as cursor: + object_key = ObjectKey((await cursor.fetchone())[0]) + assert await service.write(actor, session.upload_id, BytesIO(data)) == len(data) + authority.allowed = False + with pytest.raises(RuntimeError, match="BLOB_UPLOAD_DENIED"): + await service.complete(actor, session.upload_id) + authority.allowed = True + assert await service.complete(actor, session.upload_id) == session.reference + assert await service.complete(actor, session.upload_id) == session.reference + assert verifier.calls == 1 + assert await service.write(actor, session.upload_id, BytesIO(data)) == len(data) + store = actor.object_storage("verify completed S3 Blob upload") + assert store.get_bytes(object_key, 100_000) == data + head = store.head(object_key) + async with aiosqlite.connect(path) as db: + async with db.execute( + "SELECT status, content_type, size_bytes, content_version " + "FROM teaql_blob_catalog WHERE key=?", (session.reference.key,) + ) as cursor: + status, mime, size, version = await cursor.fetchone() + assert (status, mime, size, version) == ( + "uploaded_unbound", content_type, len(data), head.etag) + + class LifecycleBindingAuthority(BindingAuthority): + async def authorize(self, _context, target, previous, next_ref, action): + return (target == BlobBindingTarget("Asset", "1", "document") + and ((action == "Create" and previous is None + and next_ref == session.reference) + or (action == "Update" and previous == session.reference + and next_ref is None))) + + binding_authority = LifecycleBindingAuthority() + + class ReadAuthorization: + allowed = True + async def authorize(self, _context, _slot): + return self.allowed + + read_permission = ReadAuthorization() + module = BlobReferenceModule(catalog, service, read_permission, binding_authority, + maximum_url_seconds=3) + module.install(actor) + await module.ready(actor) + + async def rejected_bind(graph): + command = (InsertCommand.new("Asset").value("id", 1) + .value("version", 1).value("name", "rolled back") + .value("document", session.reference)) + await graph.transaction.mutate(graph.context, + graph.request(command, graph.scope(EntityKey("Asset", 1)))) + raise RuntimeError("S3 outer owner rejected") + + with pytest.raises(RuntimeError, match="S3 outer owner rejected"): + await actor.execute_graph_save(rejected_bind, comment="reject S3 Blob outer owner") + async with aiosqlite.connect(path) as db: + async with db.execute("SELECT COUNT(*) FROM asset WHERE id=1") as cursor: + assert await cursor.fetchone() == (0,) + async with db.execute("SELECT COUNT(*) FROM teaql_blob_binding WHERE " + "scope='tenant-a' AND entity='Asset' AND entity_id='1' " + "AND field='document'") as cursor: + assert await cursor.fetchone() == (0,) + async with db.execute("SELECT status FROM teaql_blob_catalog WHERE key=?", + (session.reference.key,)) as cursor: + assert await cursor.fetchone() == ("uploaded_unbound",) + assert store.head(object_key) is not None + print("PASS Python " + case + " native module outer owner rollback") + + async def bind(graph): + command = (InsertCommand.new("Asset").value("id", 1) + .value("version", 1).value("name", "document") + .value("document", session.reference)) + await graph.transaction.mutate(graph.context, + graph.request(command, graph.scope(EntityKey("Asset", 1)))) + + await actor.execute_graph_save(bind, comment="bind S3 Blob in business graph") + async with aiosqlite.connect(path) as db: + async with db.execute("SELECT document,version FROM asset WHERE id=1") as cursor: + assert await cursor.fetchone() == (session.reference.key, 1) + async with db.execute("SELECT status FROM teaql_blob_catalog WHERE key=?", + (session.reference.key,)) as cursor: + assert (await cursor.fetchone())[0] == "available" + + class Authorizer: + version = 1 + async def authorize(self, _context, _result, _options): + slot = AuthorizedBlobSlot(("document",), "Asset", "1", "document", + "tenant-a", self.version, session.reference) + return AuthorizedResponseGraph({"document": session.reference}, (slot,)) + + authorizer = Authorizer() + actor.insert_resource("response_graph_authorizer", authorizer) + options = ViewOptions("ordinary", "read_only", + datetime.now(timezone.utc) + timedelta(seconds=60)) + view = await ViewPreparationService().prepare(actor, None, options) + link = view.presentation["document"] + assert link.startswith("http") and view.cache_control == "private, no-store" + with urlopen(Request(link, headers={"Range": "bytes=0-7"}), timeout=5) as response: + assert response.status == 206 and response.read() == data[:8] + assert response.headers["Content-Type"] == content_type + assert file_name in response.headers["Content-Disposition"] + read_permission.allowed = False + with pytest.raises(ViewPreparationError) as denied: + await ViewPreparationService().prepare(actor, None, options) + assert denied.value.code == "BLOB_READ_DENIED" + read_permission.allowed = True + authorizer.version = 0 + with pytest.raises(ViewPreparationError) as stale: + await ViewPreparationService().prepare(actor, None, + ViewOptions("ordinary", "committed", + datetime.now(timezone.utc) + timedelta(seconds=60))) + assert (stale.value.code, stale.value.business_state) == ("BLOB_READ_DENIED", "committed") + if os.environ.get("TEAQL_S3_TEST_END_TO_END_GC") == "1": + async def unbind(graph): + command = (UpdateCommand.new("Asset", 1).expected_version(1) + .value("document", None)) + await graph.transaction.mutate(graph.context, + graph.request(command, graph.scope(EntityKey("Asset", 1)))) + + await actor.execute_graph_save(unbind, comment="unbind S3 Blob in business graph") + async with aiosqlite.connect(path) as db: + async with db.execute("SELECT document,version FROM asset WHERE id=1") as cursor: + assert await cursor.fetchone() == (None, 2) + async with db.execute("SELECT key,version FROM teaql_blob_binding WHERE " + "scope='tenant-a' AND entity='Asset' AND entity_id='1' " + "AND field='document'") as cursor: + assert await cursor.fetchone() == (None, 2) + async with db.execute( + "SELECT status,last_unbound_at_ms,effective_unref_seconds," + "retention_floor_until_ms FROM teaql_blob_catalog WHERE key=?", + (session.reference.key,) + ) as cursor: + status, last_unbound, effective, floor = await cursor.fetchone() + assert status == "available" and last_unbound == clock[0] + assert effective == 604800 and floor >= clock[0] + 3600 * 1000 + authorizer.version = 1 + with pytest.raises(ViewPreparationError) as unbound_view: + await ViewPreparationService().prepare(actor, None, options) + assert unbound_view.value.code == "BLOB_READ_DENIED" + + class MaintenanceAuthority: + def scope(self, _context): + return "tenant-a" + + async def can_maintain(self, _context, scope): + return scope == "tenant-a" + + maintenance = SqliteBlobMaintenanceService( + catalog, MaintenanceAuthority(), 10, now_ms=lambda: clock[0]) + deadline = max(last_unbound + effective * 1000, floor) + clock[0] = deadline - 1 + assert (await maintenance.run_batch(actor)).outcomes == () + assert store.head(object_key) is not None + clock[0] = deadline + collected = await maintenance.run_batch(actor) + assert [(item.reference, item.status) for item in collected.outcomes] == [ + (session.reference, "deleted")] + assert store.head(object_key) is None + async with aiosqlite.connect(path) as db: + async with db.execute("SELECT status FROM teaql_blob_catalog WHERE key=?", + (session.reference.key,)) as cursor: + assert (await cursor.fetchone())[0] == "deleted" + print("PASS Python " + case + + " S3 upload/confirm/graph bind/authorized view/unbind/retention GC") + finally: + try: + if object_key is not None: + actor.object_storage("clean S3 Blob upload fixture").delete(object_key) + finally: + provider.close() + + +@pytest.mark.asyncio +async def test_upload_rechecks_current_permission_on_physical_sql_lease(tmp_path): + path = str(tmp_path / "lease-authority.db") + catalog = SqliteBlobCatalog(SqliteTransport(path)) + await catalog.ensure_schema() + async with aiosqlite.connect(path) as db: + await db.execute("CREATE TABLE upload_permission (id INTEGER PRIMARY KEY, allowed INTEGER)") + await db.execute("INSERT INTO upload_permission VALUES (1,0)") + await db.commit() + + class LeaseAuthority(UploadAuthority): + def __init__(self): + super().__init__() + self.seen = [] + + async def current_on_lease(self, actor, lease, target, action, _input): + self.seen.append(action) + rows = await lease.fetch_all_sql(CompiledQuery( + "SELECT allowed FROM upload_permission WHERE id=1", [])) + assert len(rows) == 1 + return BlobUploadAuthorizationState(self.scope(actor), self.field(actor, target), + rows[0]["allowed"] == 1) + + async def set_allowed(value): + async with aiosqlite.connect(path) as db: + await db.execute("UPDATE upload_permission SET allowed=? WHERE id=1", (value,)) + await db.commit() + + authority = LeaseAuthority() + verifier = FixturePDFVerifier() + service = SqliteBlobUploadService(catalog, authority, verifier, 10_000) + actor = RuntimeModule.new().into_context().with_object_storage( + FileObjectStorage(tmp_path / "private", os.urandom(32))) + actor.with_object_storage_observer(lambda event: None) + target = BlobUploadTarget("Asset", "1", "document") + data = b"%PDF-1.7\nlease permission\n%%EOF" + input = BlobUploadInput("report.pdf", len(data), "application/pdf") + + with pytest.raises(RuntimeError, match="BLOB_UPLOAD_DENIED"): + await service.begin(actor, target, input, "lease-denied") + async with aiosqlite.connect(path) as db: + async with db.execute("SELECT COUNT(*) FROM teaql_blob_upload") as cursor: + assert (await cursor.fetchone())[0] == 0 + + await set_allowed(1) + session = await service.begin(actor, target, input, "lease-allowed") + await set_allowed(0) + source = BytesIO(data) + with pytest.raises(RuntimeError, match="BLOB_UPLOAD_DENIED"): + await service.write(actor, session.upload_id, source) + async with aiosqlite.connect(path) as db: + async with db.execute("SELECT state FROM teaql_blob_upload WHERE upload_id=?", + (session.upload_id,)) as cursor: + assert (await cursor.fetchone())[0] == "initiated" + await set_allowed(1) + assert await service.write(actor, session.upload_id, BytesIO(data)) == len(data) + await set_allowed(0) + with pytest.raises(RuntimeError, match="BLOB_UPLOAD_DENIED"): + await service.complete(actor, session.upload_id) + assert verifier.calls == 0 + await set_allowed(1) + assert await service.complete(actor, session.upload_id) == session.reference + assert verifier.calls == 1 + await set_allowed(0) + with pytest.raises(RuntimeError, match="BLOB_UPLOAD_DENIED"): + await service.complete(actor, session.upload_id) + assert verifier.calls == 1 + await set_allowed(1) + assert await service.complete(actor, session.upload_id) == session.reference + assert verifier.calls == 1 + assert {"Begin", "Write", "Confirm"}.issubset(authority.seen) + + +@pytest.mark.asyncio +async def test_expired_replay_key_cannot_recover_upload_session(tmp_path): + catalog = SqliteBlobCatalog(SqliteTransport(str(tmp_path / "expired.db"))) + await catalog.ensure_schema() + clock = [1_700_000_000_000] + service = SqliteBlobUploadService(catalog, UploadAuthority(), FixturePDFVerifier(), + 10_000, lease_seconds=1, now_ms=lambda: clock[0]) + context = RuntimeModule.new().into_context() + target = BlobUploadTarget("Asset", "1", "document") + input = BlobUploadInput("report.pdf", 42) + first = await service.begin(context, target, input, "expired-request") + assert (await service.begin(context, target, input, "expired-request")).upload_id == first.upload_id + clock[0] = int(first.expires_at.timestamp() * 1000) + with pytest.raises(RuntimeError, match="BLOB_UPLOAD_EXPIRED"): + await service.begin(context, target, input, "expired-request") + second = await service.begin(context, target, input, "fresh-request") + assert second.upload_id != first.upload_id + with pytest.raises(RuntimeError, match="BLOB_UPLOAD_EXPIRED"): + await service.write(context, first.upload_id, BytesIO(b"x" * 42)) + + +@pytest.mark.asyncio +async def test_upload_write_holds_sql_owner_across_file_io(tmp_path): + path = str(tmp_path / "locked.db") + catalog = SqliteBlobCatalog(SqliteTransport(path)) + await catalog.ensure_schema() + entered, release = Event(), Event() + file_store = FileObjectStorage(tmp_path / "private", os.urandom(32)) + + class BlockingStore: + puts = 0 + + def __getattr__(self, name): + return getattr(file_store, name) + + def put(self, invocation, key, content, options): + self.puts += 1 + result = file_store.put(invocation, key, content, options) + entered.set() + if not release.wait(5): + raise AssertionError("test did not release File write") + return result + + store = BlockingStore() + actor = RuntimeModule.new().into_context().with_object_storage(store) + actor.with_object_storage_observer(lambda event: None) + service = SqliteBlobUploadService(catalog, UploadAuthority(), FixturePDFVerifier(), 10_000) + data = b"%PDF-1.7\nlocked upload\n%%EOF" + session = await service.begin(actor, BlobUploadTarget("Asset", "1", "document"), + BlobUploadInput("report.pdf", len(data)), "lock-request") + first = asyncio.create_task(service.write(actor, session.upload_id, BytesIO(data))) + second = None + try: + assert await asyncio.to_thread(entered.wait, 5) + second = asyncio.create_task(service.write(actor, session.upload_id, BytesIO(data))) + await asyncio.sleep(0.05) + assert not second.done(), "second writer bypassed the physical SQL owner" + finally: + release.set() + assert await first == len(data) + assert await second == len(data) + assert store.puts == 1 + async with aiosqlite.connect(path) as db: + async with db.execute("SELECT state FROM teaql_blob_upload WHERE upload_id=?", + (session.upload_id,)) as cursor: + assert (await cursor.fetchone())[0] == "written" + + +@pytest.mark.asyncio +async def test_upload_confirmation_holds_sql_owner_through_parser_io(tmp_path): + path = str(tmp_path / "confirm-locked.db") + catalog = SqliteBlobCatalog(SqliteTransport(path)) + await catalog.ensure_schema() + entered, release = Event(), Event() + + class BlockingVerifier(FixturePDFVerifier): + def verify(self, stream, extension, maximum_bytes): + result = super().verify(stream, extension, maximum_bytes) + entered.set() + if not release.wait(5): + raise AssertionError("test did not release parser") + return result + + verifier = BlockingVerifier() + actor = RuntimeModule.new().into_context().with_object_storage( + FileObjectStorage(tmp_path / "private", os.urandom(32))) + actor.with_object_storage_observer(lambda event: None) + service = SqliteBlobUploadService(catalog, UploadAuthority(), verifier, 10_000) + data = b"%PDF-1.7\nlocked confirmation\n%%EOF" + session = await service.begin(actor, BlobUploadTarget("Asset", "1", "document"), + BlobUploadInput("report.pdf", len(data)), "confirm-request") + await service.write(actor, session.upload_id, BytesIO(data)) + first = asyncio.create_task(service.complete(actor, session.upload_id)) + second = None + try: + assert await asyncio.to_thread(entered.wait, 5) + second = asyncio.create_task(service.complete(actor, session.upload_id)) + await asyncio.sleep(0.05) + assert not second.done(), "second confirmation bypassed the physical SQL owner" + finally: + release.set() + assert await first == session.reference + assert await second == session.reference + assert await service.complete(actor, session.upload_id) == session.reference + assert verifier.calls == 1 + + +@pytest.mark.asyncio +async def test_confirm_serializes_permission_revocation_during_parser_io(tmp_path): + path = str(tmp_path / "confirm-revocation.db") + catalog = SqliteBlobCatalog(SqliteTransport(path)) + await catalog.ensure_schema() + async with aiosqlite.connect(path) as db: + await db.execute("CREATE TABLE upload_permission (id INTEGER PRIMARY KEY, allowed INTEGER)") + await db.execute("INSERT INTO upload_permission VALUES (1,1)") + await db.commit() + entered, release = Event(), Event() + + class LeaseAuthority(UploadAuthority): + async def current_on_lease(self, actor, lease, target, action, _input): + rows = await lease.fetch_all_sql(CompiledQuery( + "SELECT allowed FROM upload_permission WHERE id=1", [])) + return BlobUploadAuthorizationState(self.scope(actor), self.field(actor, target), + len(rows) == 1 and rows[0]["allowed"] == 1) + + class BlockingVerifier(FixturePDFVerifier): + def verify(self, stream, extension, maximum_bytes): + result = super().verify(stream, extension, maximum_bytes) + entered.set() + if not release.wait(5): + raise AssertionError("test did not release parser") + return result + + actor = RuntimeModule.new().into_context().with_object_storage( + FileObjectStorage(tmp_path / "private", os.urandom(32))) + actor.with_object_storage_observer(lambda event: None) + verifier = BlockingVerifier() + service = SqliteBlobUploadService(catalog, LeaseAuthority(), verifier, 10_000) + data = b"%PDF-1.7\nconcurrent revocation\n%%EOF" + session = await service.begin(actor, BlobUploadTarget("Asset", "1", "document"), + BlobUploadInput("report.pdf", len(data)), "revoke-request") + assert await service.write(actor, session.upload_id, BytesIO(data)) == len(data) + + async def revoke(): + async with aiosqlite.connect(path) as db: + await db.execute("UPDATE upload_permission SET allowed=0 WHERE id=1") + await db.commit() + + confirmation = asyncio.create_task(service.complete(actor, session.upload_id)) + revocation = None + try: + assert await asyncio.to_thread(entered.wait, 5) + revocation = asyncio.create_task(revoke()) + await asyncio.sleep(0.05) + assert not revocation.done(), "revocation committed during the parser's SQL owner" + finally: + release.set() + assert await confirmation == session.reference + assert revocation is not None + await revocation + with pytest.raises(RuntimeError, match="BLOB_UPLOAD_DENIED"): + await service.complete(actor, session.upload_id) + assert verifier.calls == 1 + async with aiosqlite.connect(path) as db: + async with db.execute("SELECT state FROM teaql_blob_upload WHERE upload_id=?", + (session.upload_id,)) as cursor: + assert (await cursor.fetchone())[0] == "completed" + + +@pytest.mark.asyncio +async def test_upload_write_commit_unknown_replays_only_same_bytes_and_generation(tmp_path): + path = str(tmp_path / "commit-unknown.db") + catalog = SqliteBlobCatalog(SqliteTransport(path)) + await catalog.ensure_schema() + file_store = FileObjectStorage(tmp_path / "private", os.urandom(32)) + + class CountingStore: + puts = 0 + + def __getattr__(self, name): + return getattr(file_store, name) + + def put(self, invocation, key, content, options): + self.puts += 1 + return file_store.put(invocation, key, content, options) + + store = CountingStore() + actor = RuntimeModule.new().into_context().with_object_storage(store) + actor.with_object_storage_observer(lambda event: None) + authority, verifier = UploadAuthority(), FixturePDFVerifier() + service = SqliteBlobUploadService(catalog, authority, verifier, 10_000) + data = b"%PDF-1.7\nSQL commit succeeded, reply lost\n%%EOF" + session = await service.begin(actor, BlobUploadTarget("Asset", "1", "document"), + BlobUploadInput("report.pdf", len(data)), "unknown-write") + begin_sql = catalog.transport.begin_sql + + class CommitThenFail: + def __init__(self, tx): + self.tx = tx + + def __getattr__(self, name): + return getattr(self.tx, name) + + async def commit_sql(self): + await self.tx.commit_sql() + raise OSError("PRIVATE-COMMIT-CANARY") + + async def uncertain_begin(): + return CommitThenFail(await begin_sql()) + + catalog.transport.begin_sql = uncertain_begin + try: + with pytest.raises(RuntimeError, match="^BLOB_CATALOG_FAILED$"): + await service.write(actor, session.upload_id, BytesIO(data)) + finally: + catalog.transport.begin_sql = begin_sql + async with aiosqlite.connect(path) as db: + async with db.execute("SELECT u.state,c.status,c.object_key,u.written_etag " + "FROM teaql_blob_upload u JOIN teaql_blob_catalog c ON c.key=u.key " + "WHERE u.upload_id=?", (session.upload_id,)) as cursor: + state, catalog_state, object_key, etag = await cursor.fetchone() + assert state == "written" and catalog_state == "initiated" and etag + assert store.puts == 1 + authority.allowed = False + with pytest.raises(RuntimeError, match="^BLOB_UPLOAD_DENIED$"): + await service.write(actor, session.upload_id, BytesIO(data)) + authority.allowed = True + with pytest.raises(RuntimeError, match="^BLOB_IDEMPOTENCY_CONFLICT$"): + await service.write(actor, session.upload_id, BytesIO(b"x" * len(data))) + class FailingSource(BytesIO): + def read(self, *_): + raise OSError("PRIVATE-SOURCE-CANARY") + + with pytest.raises(RuntimeError, match="^BLOB_STORE_FAILED$") as failed: + await service.write(actor, session.upload_id, FailingSource(data)) + assert "CANARY" not in str(failed.value) + assert await service.write(actor, session.upload_id, BytesIO(data)) == len(data) + assert store.puts == 1 + assert actor.object_storage("check unchanged version").head(ObjectKey(object_key)).etag == etag + catalog.transport.begin_sql = uncertain_begin + try: + with pytest.raises(RuntimeError, match="^BLOB_CATALOG_FAILED$"): + await service.complete(actor, session.upload_id) + finally: + catalog.transport.begin_sql = begin_sql + async with aiosqlite.connect(path) as db: + async with db.execute("SELECT state FROM teaql_blob_upload WHERE upload_id=?", + (session.upload_id,)) as cursor: + assert (await cursor.fetchone())[0] == "completed" + authority.allowed = False + with pytest.raises(RuntimeError, match="^BLOB_UPLOAD_DENIED$"): + await service.complete(actor, session.upload_id) + authority.allowed = True + assert await service.complete(actor, session.upload_id) == session.reference + assert verifier.calls == 1 + assert await service.write(actor, session.upload_id, BytesIO(data)) == len(data) + assert verifier.calls == 1 and store.puts == 1 + actor.object_storage("replace generation").put_bytes(ObjectKey(object_key), data, + PutOptions(len(data), WriteMode.REPLACE, "application/pdf")) + with pytest.raises(RuntimeError, match="^BLOB_IDEMPOTENCY_CONFLICT$"): + await service.write(actor, session.upload_id, BytesIO(data)) + + +@pytest.mark.asyncio +async def test_upload_retry_recovers_only_identical_file_after_journal_rollback(tmp_path): + path = str(tmp_path / "recover.db") + catalog = SqliteBlobCatalog(SqliteTransport(path)) + await catalog.ensure_schema() + actor = RuntimeModule.new().into_context().with_object_storage( + FileObjectStorage(tmp_path / "private", os.urandom(32))) + actor.with_object_storage_observer(lambda event: None) + verifier = FixturePDFVerifier() + service = SqliteBlobUploadService(catalog, UploadAuthority(), verifier, 10_000) + data = b"%PDF-1.7\nrecover committed File content\n%%EOF" + session = await service.begin(actor, BlobUploadTarget("Asset", "1", "document"), + BlobUploadInput("report.pdf", len(data)), "recovery-request") + async with aiosqlite.connect(path) as db: + await db.execute("""CREATE TRIGGER reject_upload_write BEFORE UPDATE ON teaql_blob_upload + WHEN NEW.state='written' BEGIN SELECT RAISE(ABORT, 'injected'); END""") + await db.commit() + with pytest.raises(RuntimeError, match="BLOB_CATALOG_FAILED"): + await service.write(actor, session.upload_id, BytesIO(data)) + async with aiosqlite.connect(path) as db: + async with db.execute("SELECT c.object_key,u.state FROM teaql_blob_upload u " + "JOIN teaql_blob_catalog c ON c.key=u.key WHERE u.upload_id=?", + (session.upload_id,)) as cursor: + object_key, state = await cursor.fetchone() + assert state == "initiated" + await db.execute("DROP TRIGGER reject_upload_write") + await db.commit() + stored = actor.object_storage("check recovery candidate").head(ObjectKey(object_key)) + assert stored is not None and stored.size == len(data) + with pytest.raises(RuntimeError, match="BLOB_STORE_FAILED"): + await service.write(actor, session.upload_id, BytesIO(b"x" * len(data))) + assert await service.write(actor, session.upload_id, BytesIO(data)) == len(data) + assert actor.object_storage("check recovered generation").head( + ObjectKey(object_key)).etag == stored.etag + assert await service.complete(actor, session.upload_id) == session.reference + assert verifier.calls == 1 + + +@pytest.mark.asyncio +async def test_upload_real_sqlite_file_verifier_and_native_bind(tmp_path): + path = str(tmp_path / "upload.db") + descriptor = EntityDescriptor("Asset").table_name("asset") + descriptor.property(PropertyDescriptor("id", DataType.I64).is_id()) + descriptor.property(PropertyDescriptor("version", DataType.I64).is_version()) + descriptor.property(PropertyDescriptor("name", DataType.Text)) + schema = SimpleSchemaProvider() + schema.register_entity(descriptor) + context = RuntimeModule.new().entity(descriptor).into_context() + context.insert_resource("dataService", SqlDataServiceExecutor( + SqliteDialect(), SqliteTransport(path), schema)) + provider = FileObjectStorage(tmp_path / "private", os.urandom(32)) + context.with_object_storage(provider).with_object_storage_observer(lambda event: None) + catalog = SqliteBlobCatalog(SqliteTransport(path)) + async with aiosqlite.connect(path) as db: + await db.execute("CREATE TABLE asset (id INTEGER PRIMARY KEY, version INTEGER, name TEXT)") + await db.commit() + await catalog.ensure_schema() + await catalog.validate_schema() + authority = UploadAuthority() + verifier = FixturePDFVerifier() + service = SqliteBlobUploadService(catalog, authority, verifier, 10_000) + target = BlobUploadTarget("Asset", "1", "document") + data = b"%PDF-1.7\nreal file bytes\n%%EOF" + input = BlobUploadInput("path/report.pdf", len(data), "application/pdf") + + session = await service.begin(context, target, input, "request-1") + replay = await service.begin(context, target, input, "request-1") + assert replay.upload_id == session.upload_id + with pytest.raises(RuntimeError, match="BLOB_IDEMPOTENCY_CONFLICT"): + await service.begin(context, target, BlobUploadInput("other.pdf", len(data)), "request-1") + with pytest.raises(RuntimeError, match="BLOB_UPLOAD_INCOMPLETE"): + await service.complete(context, session.upload_id) + authority.current_scope = "tenant-b" + with pytest.raises(RuntimeError, match="BLOB_UPLOAD_DENIED"): + await service.write(context, session.upload_id, BytesIO(data)) + authority.current_scope = "tenant-a" + assert await service.write(context, session.upload_id, BytesIO(data)) == len(data) + async with aiosqlite.connect(path) as db: + async with db.execute("SELECT status FROM teaql_blob_catalog WHERE key=?", + (session.reference.key,)) as cursor: + assert (await cursor.fetchone())[0] == "initiated" + authority.allowed = False + with pytest.raises(RuntimeError, match="BLOB_UPLOAD_DENIED"): + await service.complete(context, session.upload_id) + authority.allowed = True + reference = await service.complete(context, session.upload_id) + assert reference == session.reference and verifier.calls == 1 + assert await service.complete(context, session.upload_id) == reference + assert verifier.calls == 1 + authority.allowed = False + with pytest.raises(RuntimeError, match="BLOB_UPLOAD_DENIED"): + await service.complete(context, session.upload_id) + authority.allowed = True + + async def bind(graph): + command = (InsertCommand.new("Asset").value("id", 1) + .value("version", 1).value("name", "document")) + await graph.transaction.mutate(graph.context, + graph.request(command, graph.scope(EntityKey("Asset", 1)))) + await catalog.bind_in_graph(graph, graph.context, + BlobBindingChange(BlobBindingTarget("Asset", "1", "document"), + reference, 0, 1, "Create"), BindingAuthority()) + + await context.execute_graph_save(bind, comment="bind verified upload") + async with aiosqlite.connect(path) as db: + async with db.execute("SELECT status FROM teaql_blob_catalog WHERE key=?", + (reference.key,)) as cursor: + assert (await cursor.fetchone())[0] == "available" + assert await service.complete(context, session.upload_id) == reference + assert verifier.calls == 1 + async with aiosqlite.connect(path) as db: + async with db.execute("SELECT object_key FROM teaql_blob_catalog WHERE key=?", + (reference.key,)) as cursor: + completed_key = (await cursor.fetchone())[0] + context.object_storage("replace completed generation").put_bytes( + ObjectKey(completed_key), data, PutOptions(len(data), WriteMode.REPLACE, "application/pdf")) + with pytest.raises(RuntimeError, match="BLOB_UPLOAD_INCOMPLETE"): + await service.complete(context, session.upload_id) + assert verifier.calls == 1 + + second = await service.begin(context, target, input, "request-2") + await service.write(context, second.upload_id, BytesIO(data)) + async with aiosqlite.connect(path) as db: + async with db.execute("SELECT object_key FROM teaql_blob_catalog WHERE key=?", + (second.reference.key,)) as cursor: + object_key = (await cursor.fetchone())[0] + context.object_storage("replace generation").put_bytes( + ObjectKey(object_key), data, PutOptions(len(data), WriteMode.REPLACE, "application/pdf")) + with pytest.raises(RuntimeError, match="BLOB_UPLOAD_INCOMPLETE"): + await service.complete(context, second.upload_id) + + third = await service.begin(context, target, input, "request-3") + await service.write(context, third.upload_id, BytesIO(data)) + early = SqliteBlobUploadService(catalog, authority, FixturePDFVerifier(early=True), 10_000) + with pytest.raises(RuntimeError, match="BLOB_UPLOAD_INCOMPLETE"): + await early.complete(context, third.upload_id) + + fourth = await service.begin(context, target, input, "request-4") + with pytest.raises(RuntimeError, match="BLOB_STORE_FAILED"): + await service.write(context, fourth.upload_id, BytesIO(data + b"x")) + with pytest.raises(RuntimeError, match="BLOB_UPLOAD_INCOMPLETE"): + await service.complete(context, fourth.upload_id) + fifth = await service.begin(context, target, input, "request-5") + with pytest.raises(RuntimeError, match="BLOB_STORE_FAILED"): + await service.write(context, fifth.upload_id, BytesIO(data[:-1])) + with pytest.raises(RuntimeError, match="BLOB_UPLOAD_INCOMPLETE"): + await service.complete(context, fifth.upload_id) diff --git a/tests/provider/sqlite/test_file_blob_storage.py b/tests/provider/sqlite/test_file_blob_storage.py new file mode 100644 index 00000000..6ef9f26e --- /dev/null +++ b/tests/provider/sqlite/test_file_blob_storage.py @@ -0,0 +1,186 @@ +"""Real local File/HTTP behavior for the independent optional wheel.""" +from concurrent.futures import ThreadPoolExecutor +from datetime import timedelta +from io import BytesIO +from threading import Event +from urllib.error import HTTPError +from urllib.request import Request, urlopen +import hashlib +import os +import subprocess +import sys +import time + +import pytest + +from teaql_blob_reference import FileObjectStorage +from teaql.core.object_storage import ( + ObjectKey, PutOptions, ReadUrlOptions, StorageError, StorageErrorCode, WriteMode, +) +from teaql.runtime import UserContext + + +def test_file_conditional_delete_fences_same_bytes_and_external_process(tmp_path): + root = tmp_path / "objects" + first = FileObjectStorage(root, os.urandom(32)) + second = FileObjectStorage(root, os.urandom(32)) + actor = UserContext().with_object_storage(first).with_object_storage_observer(lambda event: None) + writer = actor.object_storage("write File generation") + actor.with_object_storage(second) + collector = actor.object_storage("delete File generation") + key = ObjectKey("blob/same-bytes") + assert collector.head(key) is None + assert collector.delete_if_version(key, "absent") == "already_missing" + data = b"same bytes" + old = writer.put_bytes(key, data, PutOptions(len(data), WriteMode.CREATE_ONLY)) + current = writer.put_bytes(key, data, PutOptions(len(data), WriteMode.REPLACE)) + assert old.etag != current.etag + assert collector.delete_if_version(key, old.etag) == "version_mismatch" + assert collector.get_bytes(key, len(data)) == data + assert collector.delete_if_version(key, current.etag) == "deleted" + assert collector.head(key) is None + assert collector.delete_if_version(key, current.etag) == "already_missing" + key_hash = second._hash(key) + assert not (second._directory(key_hash) / ("data-" + current.etag.split(".")[-1])).exists() + + if os.name != "posix": + return + locked_key = ObjectKey("blob/external-lock") + written = writer.put_bytes(locked_key, data, PutOptions(len(data), WriteMode.CREATE_ONLY)) + lock_path = second._directory(second._hash(locked_key)) / "manifest.lock" + marker = tmp_path / "locked" + script = ("import fcntl, os, sys, time; " + "fd=os.open(sys.argv[1], os.O_RDWR); fcntl.flock(fd, fcntl.LOCK_EX); " + "open(sys.argv[2], 'w').close(); time.sleep(2)") + process = subprocess.Popen([sys.executable, "-c", script, str(lock_path), str(marker)]) + try: + for _ in range(100): + if marker.exists(): + break + time.sleep(0.01) + assert marker.exists() and process.poll() is None + with ThreadPoolExecutor(max_workers=1) as pool: + pending = pool.submit(collector.delete_if_version, locked_key, written.etag) + time.sleep(0.1) + assert process.poll() is None and not pending.done() + assert process.wait(timeout=5) == 0 + assert pending.result(timeout=5) == "deleted" + finally: + if process.poll() is None: + process.terminate() + process.wait(timeout=5) + + +def test_file_generation_sweep_waits_for_writer_and_preserves_current(tmp_path): + root = tmp_path / "objects" + writer_provider = FileObjectStorage(root, os.urandom(32)) + sweep_provider = FileObjectStorage(root, os.urandom(32)) + actor = UserContext().with_object_storage(writer_provider) + writer = actor.object_storage("write File generations") + key = ObjectKey("blob/generation-sweep") + data = b"same bytes" + first = writer.put_bytes(key, data, PutOptions(len(data), WriteMode.CREATE_ONLY)) + second = writer.put_bytes(key, data, PutOptions(len(data), WriteMode.REPLACE)) + assert first.etag != second.etag + key_hash = sweep_provider._hash(key) + old_path = sweep_provider._directory(key_hash) / ("data-" + first.etag.split(".")[-1]) + assert old_path.is_file() + + reading = Event() + proceed = Event() + + class PausedSource(BytesIO): + def read(self, size=-1): + reading.set() + assert proceed.wait(5) + return super().read(size) + + with ThreadPoolExecutor(max_workers=2) as pool: + replacing = pool.submit(writer_provider.put, None, key, PausedSource(data), + PutOptions(len(data), WriteMode.REPLACE)) + assert reading.wait(5) + sweeping = pool.submit(sweep_provider.sweep_old_generations, key) + time.sleep(0.1) + assert not sweeping.done() + assert old_path.is_file() + proceed.set() + third = replacing.result(timeout=5) + assert sweeping.result(timeout=5) == 2 + + assert sweep_provider.head(None, key).etag == third.etag + assert sweep_provider.sweep_old_generations(key) == 0 + assert not old_path.exists() + current_path = sweep_provider._directory(key_hash) / ("data-" + third.etag.split(".")[-1]) + assert current_path.is_file() + assert sweep_provider.delete_if_version(None, key, third.etag) == "deleted" + assert sweep_provider.sweep_old_generations(key) == 0 + + +def _status(url, *, method="GET", headers=None): + try: + with urlopen(Request(url, method=method, headers=headers or {}), timeout=5) as response: + return response.status, dict(response.headers), response.read() + except HTTPError as error: + return error.code, dict(error.headers), error.read() + + +def test_file_storage_exact_bytes_and_authenticated_http(tmp_path): + provider = FileObjectStorage(tmp_path / "objects", os.urandom(32)) + context = UserContext().with_object_storage(provider).with_object_storage_observer(lambda event: None) + store = context.object_storage("verify real optional file store") + key = ObjectKey("objects/PRIVATE-PYTHON-CANARY") + content = b"%PDF-1.7\nTeaQL document\n%%EOF" + try: + with pytest.raises(StorageError) as unstarted: + store.get_read_url(key) + assert unstarted.value.code == StorageErrorCode.NOT_CONFIGURED + assert provider.start().startswith("http://127.0.0.1:") + options = PutOptions(len(content), WriteMode.CREATE_ONLY, "application/pdf") + metadata = store.put_bytes(key, content, options) + assert metadata.etag.startswith(hashlib.sha256(content).hexdigest() + ".") + same = store.put_bytes(key, content, PutOptions(len(content), WriteMode.REPLACE, "application/pdf")) + assert same.etag != metadata.etag + assert same.etag.startswith(hashlib.sha256(content).hexdigest() + ".") + assert store.head(key).size == len(content) + assert store.get_bytes(key, len(content)) == content + with pytest.raises(StorageError) as conflict: + store.put_bytes(key, content, options) + assert conflict.value.code == StorageErrorCode.CONFLICT + with pytest.raises(StorageError): + store.put_bytes(key, content, PutOptions(len(content) + 1, WriteMode.REPLACE)) + assert store.get_bytes(key, len(content)) == content + + signed = store.get_read_url(key, ReadUrlOptions(timedelta(seconds=2), "attachment", "asset.pdf")) + assert key.value not in signed.url and "PRIVATE-PYTHON-CANARY" not in repr(signed) + code, headers, body = _status(signed.url) + assert code == 200 and body == content + assert headers["Content-Type"] == "application/pdf" + assert headers["Content-Disposition"] == 'attachment; filename="asset.pdf"' + assert headers["Cache-Control"] == "private, no-store" + code, headers, body = _status(signed.url, method="HEAD") + assert code == 200 and body == b"" and headers["Content-Length"] == str(len(content)) + code, headers, body = _status(signed.url, headers={"Range": "bytes=0-3"}) + assert code == 206 and body == content[:4] + assert headers["Content-Range"] == f"bytes 0-3/{len(content)}" + code, _, body = _status(signed.url, headers={"Range": "bytes=-3"}) + assert code == 206 and body == content[-3:] + assert _status(signed.url, headers={"Range": "bytes=999-"})[0] == 416 + + tampered = signed.url.replace("/objects/", "/objects/A", 1) + assert _status(tampered)[0] == 404 + replacement = b"next generation" + store.put_bytes(key, replacement, PutOptions(len(replacement), WriteMode.REPLACE)) + assert _status(signed.url)[0] == 404 + assert store.get_bytes(key, len(replacement)) == replacement + expiring = store.get_read_url(key, ReadUrlOptions(timedelta(seconds=1))) + time.sleep(1.1) + assert _status(expiring.url)[0] == 404 + current = provider._manifest(provider._hash(key)) + provider._data_path(provider._hash(key), current).unlink() + with pytest.raises(StorageError) as missing: + store.head(key) + assert missing.value.code == StorageErrorCode.UNAVAILABLE + store.delete(key) + assert not store.exists(key) + finally: + provider.close() diff --git a/tests/provider/sqlite/test_graph_physical_sql.py b/tests/provider/sqlite/test_graph_physical_sql.py new file mode 100644 index 00000000..4b517d2d --- /dev/null +++ b/tests/provider/sqlite/test_graph_physical_sql.py @@ -0,0 +1,118 @@ +"""The optional-module SQL lease shares the native graph's physical owner.""" + +import aiosqlite +import pytest + +from teaql.core.meta import EntityDescriptor, PropertyDescriptor +from teaql.core.entity import EntityKey +from teaql.core.mutation import InsertCommand +from teaql.core.value import DataType, Value +from teaql.provider.sqlite import SimpleSchemaProvider +from teaql.provider.sqlite.dialect import SqliteDialect +from teaql.provider.sqlite.transport import SqliteTransport +from teaql.runtime import RuntimeModule +from teaql.sql.executor import SqlDataServiceExecutor +from teaql.sql.types import CompiledQuery + + +async def fixture(tmp_path): + path = str(tmp_path / "graph-owner.db") + descriptor = EntityDescriptor("Asset").table_name("asset") + descriptor.property(PropertyDescriptor("id", DataType.I64).is_id()) + descriptor.property(PropertyDescriptor("version", DataType.I64).is_version()) + descriptor.property(PropertyDescriptor("name", DataType.Text)) + schema = SimpleSchemaProvider() + schema.register_entity(descriptor) + context = RuntimeModule.new().entity(descriptor).into_context() + context.insert_resource("dataService", SqlDataServiceExecutor(SqliteDialect(), SqliteTransport(path), schema)) + async with aiosqlite.connect(path) as db: + await db.execute("CREATE TABLE asset (id INTEGER PRIMARY KEY, version INTEGER, name TEXT)") + await db.execute("CREATE TABLE module_binding (id INTEGER PRIMARY KEY, asset_id INTEGER NOT NULL)") + await db.commit() + return context, path + + +async def rows(path, table): + async with aiosqlite.connect(path) as db: + async with db.execute(f"SELECT * FROM {table}") as cursor: + return await cursor.fetchall() + + +@pytest.mark.asyncio +async def test_native_graph_and_module_rows_commit_or_rollback_together(tmp_path): + context, path = await fixture(tmp_path) + retained = [] + + async def save(graph, asset_id, *, fail=False): + command = (InsertCommand.new("Asset").value("id", asset_id) + .value("version", 1).value("name", "document")) + await graph.transaction.mutate(graph.context, graph.request(command, graph.scope(EntityKey("Asset", asset_id)))) + + async def bind(sql): + retained.append(sql) + assert not hasattr(sql, "commit_sql") + await sql.execute_sql(CompiledQuery( + "INSERT INTO module_binding(id, asset_id) VALUES (?, ?)", + [Value.I64(asset_id), Value.I64(asset_id)])) + assert len(await sql.fetch_all_sql(CompiledQuery( + "SELECT id FROM asset WHERE id = ?", [Value.I64(asset_id)]))) == 1 + + await graph.with_sql_transport(bind) + if fail: + raise RuntimeError("abort graph") + + await context.execute_graph_save(lambda graph: save(graph, 1), comment="save asset") + assert len(await rows(path, "asset")) == len(await rows(path, "module_binding")) == 1 + with pytest.raises(RuntimeError, match="abort graph"): + await context.execute_graph_save(lambda graph: save(graph, 2, fail=True), comment="save asset") + assert len(await rows(path, "asset")) == len(await rows(path, "module_binding")) == 1 + with pytest.raises(RuntimeError, match="BLOB_TRANSACTION_REQUIRED"): + await retained[0].execute_sql(CompiledQuery("DELETE FROM module_binding", [])) + + +@pytest.mark.asyncio +async def test_caught_callback_or_sql_error_still_blocks_graph_commit(tmp_path): + context, path = await fixture(tmp_path) + + async def save(graph, fail_in_sql): + command = (InsertCommand.new("Asset").value("id", 1) + .value("version", 1).value("name", "document")) + await graph.transaction.mutate(graph.context, graph.request(command, graph.scope(EntityKey("Asset", 1)))) + + async def bind(sql): + await sql.execute_sql(CompiledQuery( + "INSERT INTO module_binding(id, asset_id) VALUES (1, 1)", [])) + if fail_in_sql: + try: + await sql.execute_sql(CompiledQuery( + "INSERT INTO module_binding(id, asset_id) VALUES (1, 1)", [])) + except Exception: + pass + else: + raise ValueError("module rejected binding") + + try: + await graph.with_sql_transport(bind) + except Exception: + pass + + for fail_in_sql in (False, True): + with pytest.raises(RuntimeError, match="GRAPH_TRANSACTION_ROLLBACK_ONLY"): + await context.execute_graph_save( + lambda graph: save(graph, fail_in_sql), comment="save asset") + assert await rows(path, "asset") == await rows(path, "module_binding") == [] + + +@pytest.mark.asyncio +async def test_module_sql_requires_live_graph_and_transaction(tmp_path): + context, _ = await fixture(tmp_path) + saved = [] + + async def save(graph): + saved.append(graph) + await graph.with_sql_transport(lambda sql: sql.fetch_all_sql(CompiledQuery("SELECT 1", []))) + + await context.execute_graph_save(save, comment="inspect graph") + with pytest.raises(RuntimeError, match="no longer writable"): + await saved[0].with_sql_transport(lambda sql: sql.fetch_all_sql(CompiledQuery("SELECT 1", []))) + diff --git a/tests/provider/sqlite/testdata/blob-policy-cases.json b/tests/provider/sqlite/testdata/blob-policy-cases.json new file mode 100644 index 00000000..c8b4b794 --- /dev/null +++ b/tests/provider/sqlite/testdata/blob-policy-cases.json @@ -0,0 +1,159 @@ +{ + "schema": "teaql.blob-policy-cases/v1", + "cases": [ + { + "id": "defaults", + "input": {}, + "expected": { + "maxSizeBytes": 104857600, + "uploadSessionTtlSeconds": 3600, + "unboundRetentionSeconds": 86400, + "unreferencedRetentionSeconds": 604800, + "readUrlTtlSeconds": 600 + } + }, + { + "id": "intersections", + "input": { + "modelExtensions": [ + "pdf", + "docx", + "xlsx" + ], + "deploymentExtensions": [ + "pdf", + "docx" + ], + "profileExtensions": [ + "pdf", + "xlsx" + ] + }, + "expected": { + "allowedExtensions": [ + "pdf" + ], + "allowedContentTypes": [ + "application/pdf" + ] + } + }, + { + "id": "empty-intersection", + "input": { + "modelExtensions": [ + "pdf" + ], + "profileExtensions": [ + "docx" + ] + }, + "error": "BLOB_POLICY_INVALID" + }, + { + "id": "hard-size-limit", + "input": { + "maxSizeBytes": 104857601, + "maxUploadSizeBytes": 104857600 + }, + "error": "BLOB_POLICY_INVALID" + }, + { + "id": "url-capped-by-document", + "input": { + "requestedTtlSeconds": 600, + "documentRemainingSeconds": 120, + "storeMaximumSeconds": 90 + }, + "expected": { + "readUrlTtlSeconds": 90 + } + }, + { + "id": "upload-clock-independent", + "input": { + "beginAtSeconds": 0, + "completedAtSeconds": 3500, + "nowSeconds": 3600 + }, + "expected": { + "uploadSessionExpired": true, + "unboundEligibleAtSeconds": 89900 + } + }, + { + "id": "unbound-deadline-rejects-bind-before-gc", + "input": { + "completedAtSeconds": 0, + "unboundRetentionSeconds": 86400, + "nowSeconds": 86400 + }, + "error": "BLOB_REFERENCE_EXPIRED" + }, + { + "id": "active-reference-never-collected", + "input": { + "bindingCount": 1, + "nowSeconds": 315360000 + }, + "expected": { + "collectable": false + } + }, + { + "id": "maximum-retention-across-fields", + "input": { + "acceptedRetentionSeconds": [ + 604800, + 2592000, + 3600 + ], + "lastUnboundAtSeconds": 100, + "retentionFloorUntilSeconds": 3000000 + }, + "expected": { + "effectiveUnreferencedRetentionSeconds": 2592000, + "unreferencedEligibleAtSeconds": 3000000 + } + }, + { + "id": "profile-update-does-not-shorten", + "input": { + "acceptedUnreferencedRetentionSeconds": 2592000, + "currentProfileRetentionSeconds": 3600 + }, + "expected": { + "effectiveUnreferencedRetentionSeconds": 2592000 + } + }, + { + "id": "read-expiry-does-not-delete", + "input": { + "readUrlExpired": true, + "bindingCount": 1 + }, + "expected": { + "collectable": false, + "requiresReadAuthorization": true + } + }, + { + "id": "deleting-rejects-binding", + "input": { + "status": "deleting" + }, + "error": "BLOB_REFERENCE_EXPIRED" + }, + { + "id": "context-checked-read-is-explicit", + "input": { + "requireContextCheckedRead": true, + "storeCapabilities": [ + "presign", + "range" + ] + }, + "error": "BLOB_CAPABILITY_MISSING" + } + ] +} diff --git a/tests/runtime/test_blob_reference_value.py b/tests/runtime/test_blob_reference_value.py new file mode 100644 index 00000000..902ccc43 --- /dev/null +++ b/tests/runtime/test_blob_reference_value.py @@ -0,0 +1,78 @@ +import json +from pathlib import Path + +import pytest + +from teaql.core import BlobFieldSpec, BlobRef +from teaql.runtime import RuntimeModule, UserContext, install_runtime_capability, validate_capabilities + + +VECTORS = json.loads((Path(__file__).resolve().parents[2] / "test-vectors" / + "blob-value-cases.json").read_text()) + + +def test_shared_blob_ref_and_format_vectors(): + for key in VECTORS["valid"]: + reference = BlobRef.from_json(key) + assert reference == BlobRef.from_sql(key) + assert reference.to_sql() == key + assert repr(reference) == "BlobRef[masked]" + with pytest.raises(TypeError): + json.dumps(reference) + with pytest.raises(AttributeError): + reference._key = "blob_other" + for value in [*VECTORS["invalid"], *VECTORS["invalidJson"]]: + with pytest.raises(ValueError, match="BLOB_REFERENCE_INVALID"): + BlobRef.from_json(value) + + spec = BlobFieldSpec("Asset", "document", list(reversed(VECTORS["formats"])), + False, "default") + assert spec.allowed_extensions == tuple(sorted(VECTORS["formats"])) + assert spec.allowed_content_types == tuple(sorted(set(VECTORS["formats"].values()))) + with pytest.raises(ValueError, match="BLOB_FORMAT_NOT_ALLOWED"): + BlobFieldSpec("Asset", "document", ["application.docx"], False, "default") + + +class ReadyResource: + def __init__(self, capability: str, failure: bool = False): + self.capability = capability + self.failure = failure + + async def validate_ready(self, context): + if self.failure: + raise RuntimeError("private provider diagnostic") + + +@pytest.mark.asyncio +async def test_module_capabilities_fail_before_graph_transaction_and_schema(): + blob = RuntimeModule.new().require_capability("blob.v1") + secure = RuntimeModule.new().require_capability("secure_document.v1").require_capability("blob.v1") + both = blob.and_module(secure) + assert both.required_capabilities == ("blob.v1", "secure_document.v1") + assert blob.required_capabilities == ("blob.v1",) + context = both.into_context() + entered = False + + async def work(_session): + nonlocal entered + entered = True + + with pytest.raises(RuntimeError, match="BLOB_CAPABILITY_MISSING"): + await context.execute_graph_save(work, comment="save document") + assert not entered + with pytest.raises(RuntimeError, match="BLOB_CAPABILITY_MISSING"): + await context.ensure_schema() + + install_runtime_capability(context, ReadyResource("blob.v1")) + with pytest.raises(RuntimeError, match="DOCUMENT_CONFIGURATION"): + await validate_capabilities(context) + install_runtime_capability(context, ReadyResource("secure_document.v1")) + await validate_capabilities(context) + with pytest.raises(ValueError, match="RUNTIME_CAPABILITY_DUPLICATE"): + install_runtime_capability(context, ReadyResource("blob.v1")) + + failing = RuntimeModule.new().require_capability("blob.v1").into_context() + install_runtime_capability(failing, ReadyResource("blob.v1", failure=True)) + with pytest.raises(RuntimeError, match="BLOB_CAPABILITY_MISSING") as error: + await validate_capabilities(failing) + assert "private provider diagnostic" not in str(error.value) diff --git a/tests/runtime/test_object_storage.py b/tests/runtime/test_object_storage.py index b5712553..4e5cafd8 100644 --- a/tests/runtime/test_object_storage.py +++ b/tests/runtime/test_object_storage.py @@ -1,7 +1,13 @@ from datetime import timedelta, datetime, timezone +from concurrent.futures import ThreadPoolExecutor from io import BytesIO from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from threading import Thread +from urllib.error import HTTPError +from urllib.request import Request, urlopen +from uuid import uuid4 +import os +import time import pytest import boto3 from botocore.config import Config @@ -14,6 +20,7 @@ class Stub: size = 3 code = None seen = None + deletes = 0 def head(self, invocation, key): self.seen = invocation if self.code: @@ -24,6 +31,8 @@ def put(self, invocation, key, stream, options): return ObjectMetadata(key, options.size) def get(self, invocation, key): return ObjectContent(ObjectMetadata(key, self.size), BytesIO(b"abc")) + def delete(self, invocation, key): + self.deletes += 1 def assert_error(code, work): @@ -49,6 +58,16 @@ def test_object_storage_context_and_error_semantics(): stub.code = code assert_error(code, lambda: storage.exists(key)) +def test_conditional_delete_refuses_blind_provider(): + stub = Stub() + storage = UserContext().with_object_storage(stub).object_storage("collect old generation") + key = ObjectKey("PRIVATE-KEY-CANARY") + assert_error(StorageErrorCode.UNSUPPORTED, + lambda: storage.delete_if_version(key, '"old-etag"')) + assert_error(StorageErrorCode.INVALID_INPUT, + lambda: storage.delete_if_version(key, "*")) + assert stub.deletes == 0 + def test_object_storage_stream_bounds_ownership_and_completion(): events, stub = [], Stub() @@ -80,6 +99,16 @@ def test_object_storage_options_and_private_representations(): for seconds in (0, -1, 0.5, 604801): assert_error(StorageErrorCode.INVALID_INPUT, lambda: ReadUrlOptions(timedelta(seconds=seconds))) assert_error(StorageErrorCode.INVALID_INPUT, lambda: ReadUrlOptions(filename="bad\nname", disposition="attachment")) + assert_error(StorageErrorCode.INVALID_INPUT, lambda: ReadUrlOptions(filename="bad\r\nname", disposition="attachment")) + assert_error(StorageErrorCode.INVALID_INPUT, lambda: ReadUrlOptions(filename="bad/name", disposition="attachment")) + assert_error(StorageErrorCode.INVALID_INPUT, lambda: ReadUrlOptions(filename="\ud800", disposition="attachment")) + assert ReadUrlOptions(disposition="inline", filename="sample.jpg").content_disposition() == ( + 'inline; filename="sample.jpg"') + unicode_disposition = ReadUrlOptions( + disposition="inline", filename="报告 & 📄\u2028mid\u2029x.jpg").content_disposition() + assert unicode_disposition.startswith('inline; filename="__ & _') + assert "filename*=UTF-8''%E6%8A%A5%E5%91%8A%20%26%20%F0%9F%93%84" in unicode_disposition + assert "\r" not in unicode_disposition and "\n" not in unicode_disposition assert "CANARY" not in repr(ObjectKey("PRIVATE-KEY-CANARY")) assert "CANARY" not in repr(ObjectReadUrl("PRIVATE-URL-CANARY", None)) options = PutOptions(0, WriteMode.CREATE_ONLY, metadata={"tag": "PRIVATE-METADATA-CANARY"}) @@ -146,3 +175,89 @@ def read(self, *_): server.shutdown() server.server_close() provider.close() + + +def test_minio_real_signed_range_and_expiry(): + """Explicit local S3-compatible gate; the host owns the disposable bucket.""" + endpoint = os.environ.get("TEAQL_S3_TEST_ENDPOINT") + bucket = os.environ.get("TEAQL_S3_TEST_BUCKET") + access = os.environ.get("TEAQL_S3_TEST_ACCESS_KEY") + secret = os.environ.get("TEAQL_S3_TEST_SECRET_KEY") + if not all((endpoint, bucket, access, secret)): + pytest.skip("explicit S3-compatible endpoint, bucket and credentials required") + provider = S3ObjectStorage.connect(bucket, endpoint_url=endpoint, + aws_access_key_id=access, aws_secret_access_key=secret) + storage = UserContext().with_object_storage(provider).object_storage("F4 real S3-compatible Python") + key = ObjectKey("f4/python/" + uuid4().hex) + create_key = ObjectKey("f4/python/create/" + uuid4().hex) + race_key = ObjectKey("f4/python/race/" + uuid4().hex) + conditional_key = ObjectKey("f2/python/conditional-delete/" + uuid4().hex) + content = b"%PDF-1.7\nF4 real S3-compatible bytes\n%%EOF" + try: + written = storage.put_bytes(key, content, PutOptions(len(content), WriteMode.REPLACE, "application/pdf")) + head = storage.head(key) + assert head is not None and head.size == len(content) + assert head.etag == written.etag and head.content_type == "application/pdf" + assert storage.get_bytes(key, 1024) == content + ticket = storage.get_read_url(key, ReadUrlOptions(timedelta(seconds=2), "attachment", "report.pdf")) + # Do not log the signed URL or credentials. + with urlopen(Request(ticket.url, headers={"Range": "bytes=0-7"}), timeout=5) as response: + assert response.status == 206 and response.read() == content[:8] + assert response.headers["Content-Type"] == "application/pdf" + assert "report.pdf" in response.headers["Content-Disposition"] + time.sleep(max(0, ticket.expires_at.timestamp() - time.time() + 1.3)) + with pytest.raises(HTTPError) as expired: + urlopen(ticket.url, timeout=5) + assert expired.value.code == 403 + refreshed = storage.get_read_url(key, ReadUrlOptions(timedelta(seconds=30), "attachment", "report.pdf")) + assert refreshed.url != ticket.url and refreshed.expires_at > datetime.now(timezone.utc) + with urlopen(Request(refreshed.url, headers={"Range": "bytes=0-7"}), timeout=5) as response: + assert response.status == 206 and response.read() == content[:8] + if os.environ.get("TEAQL_S3_TEST_CREATE_ONLY") == "1": + create = PutOptions(len(content), WriteMode.CREATE_ONLY, "application/pdf") + created = storage.put_bytes(create_key, content, create) + assert_error(StorageErrorCode.CONFLICT, + lambda: storage.put_bytes(create_key, b"x" * len(content), create)) + assert storage.head(create_key).etag == created.etag + assert storage.get_bytes(create_key, 1024) == content + candidates = (content, b"y" * len(content)) + with ThreadPoolExecutor(max_workers=2) as pool: + futures = [pool.submit(storage.put_bytes, race_key, value, create) + for value in candidates] + outcomes = [] + for future in futures: + try: + future.result() + outcomes.append("winner") + except StorageError as error: + assert error.code == StorageErrorCode.CONFLICT + outcomes.append("conflict") + assert outcomes.count("winner") == outcomes.count("conflict") == 1 + assert storage.get_bytes(race_key, 1024) == candidates[outcomes.index("winner")] + if os.environ.get("TEAQL_S3_TEST_CONDITIONAL_DELETE") == "1": + first = storage.put_bytes(conditional_key, content, + PutOptions(len(content), WriteMode.CREATE_ONLY, "application/pdf")) + assert storage.delete_if_version( + conditional_key, '"00000000000000000000000000000000"') == "version_mismatch" + assert storage.head(conditional_key).etag == first.etag + replacement = b"next immutable generation" + second = storage.put_bytes(conditional_key, replacement, + PutOptions(len(replacement), WriteMode.REPLACE, "text/plain")) + assert storage.delete_if_version(conditional_key, first.etag) == "version_mismatch" + assert storage.head(conditional_key).etag == second.etag + assert storage.get_bytes(conditional_key, 1024) == replacement + assert storage.delete_if_version(conditional_key, second.etag) == "deleted" + assert storage.head(conditional_key) is None + assert storage.delete_if_version(conditional_key, second.etag) == "already_missing" + assert_error(StorageErrorCode.INVALID_INPUT, + lambda: storage.delete_if_version(conditional_key, "*")) + finally: + try: + storage.delete(key) + if os.environ.get("TEAQL_S3_TEST_CREATE_ONLY") == "1": + storage.delete(create_key) + storage.delete(race_key) + if os.environ.get("TEAQL_S3_TEST_CONDITIONAL_DELETE") == "1": + storage.delete(conditional_key) + finally: + provider.close() diff --git a/tests/test_view_preparation_blob.py b/tests/test_view_preparation_blob.py new file mode 100644 index 00000000..3dad41f7 --- /dev/null +++ b/tests/test_view_preparation_blob.py @@ -0,0 +1,173 @@ +from datetime import datetime, timedelta, timezone + +import pytest + +from teaql.core.blob_ref import BlobRef +from teaql.runtime.context import UserContext +from teaql.runtime.view_preparation import ( + AuthorizedBlobSlot, AuthorizedResponseGraph, PreparedSecureDocument, + ProjectedBlobURL, ViewOptions, ViewPreparationError, ViewPreparationService, +) + + +@pytest.mark.asyncio +async def test_authorized_view_is_detached_and_governed_document_precedes_blob_url(): + reference = BlobRef("blob_asset_1") + source = {"document": reference, "nested": [{"title": "untouched"}]} + stages = [] + ceiling = datetime.now(timezone.utc) + timedelta(minutes=5) + + class Authorizer: + async def authorize(self, _context, _result, _options): + stages.append("authorize") + return AuthorizedResponseGraph(source, (AuthorizedBlobSlot( + path=("document",), entity="Asset", entity_id="1", field="document", + scope="tenant-a", binding_version=4, reference=reference),)) + + class Secure: + async def prepare(self, _context, graph, _options): + stages.append("secure") + graph.presentation["governed"] = True + return PreparedSecureDocument(graph, "sealed-document", ceiling) + + class Blob: + async def project_batch(self, _context, graph, expiry): + stages.append("blob") + assert expiry == ceiling + assert graph.blob_slots[0].reference == reference + return (ProjectedBlobURL("https://files.example.invalid/read/asset", + ceiling - timedelta(minutes=1)),) + + context = UserContext() + context.insert_resource("response_graph_authorizer", Authorizer()) + context.insert_resource("secure_document_contributor", Secure()) + context.insert_resource("blob_response_contributor", Blob()) + view = await ViewPreparationService().prepare(context, source, + ViewOptions(mode="governed", business_outcome="committed")) + assert stages == ["authorize", "secure", "blob"] + assert source == {"document": reference, "nested": [{"title": "untouched"}]} + assert view.presentation == { + "document": "https://files.example.invalid/read/asset", + "nested": [{"title": "untouched"}], "governed": True, + } + assert view.document == "sealed-document" + assert view.expires_at == ceiling - timedelta(minutes=1) + assert view.cache_control == "private, no-store" + + +@pytest.mark.asyncio +async def test_independent_url_lifetimes_use_original_ceiling_and_minimum(): + first = BlobRef("blob_asset_1") + second = BlobRef("blob_asset_2") + ceiling = datetime.now(timezone.utc) + timedelta(seconds=90) + first_expiry = datetime.now(timezone.utc) + timedelta(seconds=20) + second_expiry = datetime.now(timezone.utc) + timedelta(seconds=40) + + class Authorizer: + async def authorize(self, _context, _result, _options): + return AuthorizedResponseGraph({"first": first, "second": second}, ( + AuthorizedBlobSlot(("first",), "Asset", "1", "first", "tenant-a", 4, first), + AuthorizedBlobSlot(("second",), "Asset", "1", "second", "tenant-a", 5, second), + )) + + class Blob: + async def project_batch(self, _context, _graph, expiry): + assert expiry == ceiling + return (ProjectedBlobURL("https://files.example.invalid/first", first_expiry), + ProjectedBlobURL("https://files.example.invalid/second", second_expiry)) + + context = UserContext() + context.insert_resource("response_graph_authorizer", Authorizer()) + context.insert_resource("blob_response_contributor", Blob()) + view = await ViewPreparationService().prepare(context, None, + ViewOptions(mode="ordinary", business_outcome="read_only", expires_at=ceiling)) + assert view.expires_at == first_expiry + assert view.presentation == { + "first": "https://files.example.invalid/first", + "second": "https://files.example.invalid/second", + } + + +@pytest.mark.asyncio +async def test_unlisted_key_and_provider_failure_fail_closed_after_commit(): + reference = BlobRef("blob_private_canary") + + class Authorizer: + def __init__(self): + self.listed = False + + async def authorize(self, _context, _result, _options): + slots = (AuthorizedBlobSlot(("document",), "Asset", "1", "document", + "tenant-a", 4, reference),) if self.listed else () + return AuthorizedResponseGraph({"document": reference}, slots) + + class FailingBlob: + async def project_batch(self, _context, _graph, _expiry): + raise RuntimeError("private URL https://secret.invalid/read?sig=canary") + + authorizer = Authorizer() + context = UserContext().insert_resource("response_graph_authorizer", authorizer) + service = ViewPreparationService() + options = ViewOptions(mode="ordinary", business_outcome="committed") + with pytest.raises(ViewPreparationError) as first: + await service.prepare(context, None, options) + assert first.value.code == "BLOB_CAPABILITY_MISSING" + assert first.value.business_state == "committed" + assert first.value.retry == "read" + assert reference.key not in str(first.value) + + authorizer.listed = True + context.insert_resource("blob_response_contributor", FailingBlob()) + with pytest.raises(ViewPreparationError) as second: + await service.prepare(context, None, options) + assert second.value.code == "VIEW_PREPARATION_FAILED" + assert "secret.invalid" not in str(second.value) + + +@pytest.mark.asyncio +async def test_provider_url_with_raw_whitespace_or_control_fails_after_commit(): + reference = BlobRef("blob_asset_1") + + class Authorizer: + async def authorize(self, _context, _result, _options): + return AuthorizedResponseGraph({"document": reference}, ( + AuthorizedBlobSlot(("document",), "Asset", "1", "document", + "tenant-a", 4, reference),)) + + class Blob: + url = "" + + async def project_batch(self, _context, _graph, _expiry): + return (ProjectedBlobURL(self.url, + datetime.now(timezone.utc) + timedelta(seconds=30)),) + + blob = Blob() + context = UserContext().insert_resource("response_graph_authorizer", Authorizer()) + context.insert_resource("blob_response_contributor", blob) + for invalid in (" https://files.example.invalid/read", + "https://files.example.invalid/read file", + "https://files.example.invalid/read\n", + "https://files.example.invalid/\tread", + "https://files.example.invalid/read\u2028next", + "https://files.example.invalid/read\u2029next"): + blob.url = invalid + with pytest.raises(ViewPreparationError) as failure: + await ViewPreparationService().prepare(context, None, + ViewOptions(mode="ordinary", business_outcome="committed")) + assert failure.value.code == "BLOB_SIGN_FAILED" + assert failure.value.business_state == "committed" + + +@pytest.mark.asyncio +async def test_invalid_or_expired_view_options_are_rejected_before_authorization(): + context = UserContext() + service = ViewPreparationService() + for options in ( + None, + ViewOptions(mode="downgrade", business_outcome="read_only"), + ViewOptions(mode="ordinary", business_outcome="committed", + expires_at=datetime.now(timezone.utc) - timedelta(seconds=1)), + ): + with pytest.raises(ViewPreparationError) as failure: + await service.prepare(context, {}, options) + assert failure.value.code == "VIEW_OPTIONS_INVALID"