From 9f569dff9a015dfcb51ccd09c8b86863d313333f Mon Sep 17 00:00:00 2001 From: Lia Date: Fri, 23 Oct 2026 08:06:53 +0000 Subject: [PATCH 1/7] =?UTF-8?q?=F0=9F=94=8C=20feat:=20Integrate=20Bounded?= =?UTF-8?q?=20PDF=20and=20DOCX=20Extraction=20Service?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/config.py | 3 + app/routes/document_routes.py | 69 +++--- app/services/extraction_client.py | 327 +++++++++++++++++++++++++ app/utils/document_loader.py | 12 + tests/extraction/conftest.py | 143 +++++++++++ tests/extraction/fixtures/body.docx | Bin 0 -> 36732 bytes tests/extraction/fixtures/pages.pdf | Bin 0 -> 950 bytes tests/extraction/test_client.py | 227 +++++++++++++++++ tests/extraction/test_processor.py | 144 +++++++++++ tests/extraction/test_routes.py | 363 ++++++++++++++++++++++++++++ 10 files changed, 1261 insertions(+), 27 deletions(-) create mode 100644 app/services/extraction_client.py create mode 100644 tests/extraction/conftest.py create mode 100644 tests/extraction/fixtures/body.docx create mode 100644 tests/extraction/fixtures/pages.pdf create mode 100644 tests/extraction/test_client.py create mode 100644 tests/extraction/test_processor.py create mode 100644 tests/extraction/test_routes.py diff --git a/app/config.py b/app/config.py index 3fc8560e..85d5d4d4 100644 --- a/app/config.py +++ b/app/config.py @@ -10,6 +10,7 @@ from starlette.middleware.base import BaseHTTPMiddleware from app.services.vector_store.factory import get_vector_store +from app.services.extraction_client import ExtractionSettings load_dotenv(find_dotenv()) @@ -41,6 +42,8 @@ def get_env_variable( return value +EXTRACTION_SETTINGS = ExtractionSettings.from_env() + RAG_HOST = os.getenv("RAG_HOST", "0.0.0.0") RAG_PORT = int(os.getenv("RAG_PORT", 8000)) diff --git a/app/routes/document_routes.py b/app/routes/document_routes.py index c3cdb2ee..4aefd004 100644 --- a/app/routes/document_routes.py +++ b/app/routes/document_routes.py @@ -7,6 +7,7 @@ import traceback import aiofiles import aiofiles.os +from anyio import CancelScope import asyncio import time from shutil import copyfileobj @@ -44,8 +45,11 @@ EMBEDDING_MAX_QUEUE_SIZE, PARALLEL_EXECUTION, RAG_DISTANCE_THRESHOLD, + EXTRACTION_SETTINGS, + PDF_EXTRACT_IMAGES, ) from app.scope import file_clause, files_clause, resolve_scope +from app.services.extraction_client import ExtractionClient, ExtractionError # Warn once at import time if the user set a threshold under Atlas, where # the score direction is inverted (Atlas vectorSearchScore: higher = better) @@ -94,6 +98,7 @@ def _apply_distance_threshold(documents): from app.utils.async_utils import run_in_executor from app.utils.document_loader import ( get_loader, + extraction_format, clean_text, process_documents, cleanup_temp_encoding_file, @@ -101,6 +106,7 @@ def _apply_distance_threshold(documents): from app.utils.health import is_health_ok router = APIRouter() +extraction_client = ExtractionClient(EXTRACTION_SETTINGS) _INGESTION_ATTEMPT_ID_KEY = "_rag_ingestion_attempt_id" _INGESTION_ATTEMPT_STARTED_AT_NS_KEY = "_rag_ingestion_attempt_started_at_ns" @@ -348,6 +354,31 @@ async def load_file_content( the ``/text`` endpoint) so text-formatted files are not semantically parsed. """ + format = extraction_format(filename, content_type) + settings = extraction_client.settings + if settings.enabled and format in settings.formats: + try: + if format == "pdf" and PDF_EXTRACT_IMAGES: + raise ExtractionError("UNSUPPORTED_CONTENT") + segments = await extraction_client.extract(file_path, format) + except ExtractionError as error: + raise HTTPException( + status_code=error.status_code, + detail={"code": error.code}, + ) from None + # Service pages are one-based; PyPDFLoader metadata is zero-based. + documents = [ + Document( + page_content=segment["text"], + metadata={ + "source": file_path, + **({"page": segment["index"] - 1} if format == "pdf" else {}), + }, + ) + for segment in segments + ] + return documents, True, format + loader = None try: loader, known_type, file_ext = get_loader( @@ -381,7 +412,9 @@ def extract_text_from_documents(documents: List[Document], file_ext: str) -> str async def cleanup_temp_file_async(file_path: str) -> None: """Clean up temporary file asynchronously.""" try: - await aiofiles.os.remove(file_path) + # Starlette cancellation scopes must not interrupt upload cleanup. + with CancelScope(shield=True): + await aiofiles.os.remove(file_path) except Exception as e: logger.error( "Failed to remove temporary file | Path: %s | Error: %s | Traceback: %s", @@ -1114,7 +1147,6 @@ async def embed_local_file( detail=ERROR_MESSAGES.FILE_NOT_FOUND, ) - loader = None try: logger.info( "Ingestion started | %s", @@ -1127,12 +1159,11 @@ async def embed_local_file( temp_file_path=file_path, ), ) - loader, known_type, file_ext = get_loader( - document.filename, document.file_content_type, file_path - ) - loop = asyncio.get_running_loop() - data = await loop.run_in_executor( - request.app.state.thread_pool, lambda: list(loader.lazy_load()) + data, known_type, file_ext = await load_file_content( + document.filename, + document.file_content_type, + file_path, + request.app.state.thread_pool, ) result = await store_data_in_vector_db( @@ -1147,7 +1178,7 @@ async def embed_local_file( temp_file_path=file_path, ) - if result: + if result and "error" not in result: return { "status": True, "file_id": document.file_id, @@ -1188,10 +1219,6 @@ async def embed_local_file( status_code=status.HTTP_400_BAD_REQUEST, detail=ERROR_MESSAGES.DEFAULT(e), ) - finally: - # Clean up temporary UTF-8 file if it was created for encoding conversion - if loader is not None: - cleanup_temp_encoding_file(loader) @router.post("/embed") @@ -1261,23 +1288,11 @@ async def embed_file( temp_file_path=validated_file_path, ) - if not result: - response_status = False - response_message = "Failed to process/store the file data." + if not result or "error" in result: raise HTTPException( status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail="Failed to process/store the file data.", ) - elif "error" in result: - response_status = False - response_message = "Failed to process/store the file data." - if isinstance(result["error"], str): - response_message = result["error"] - else: - raise HTTPException( - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - detail="An unspecified error occurred.", - ) except HTTPException as http_exc: response_status = False response_message = f"HTTP Exception: {http_exc.detail}" @@ -1423,7 +1438,7 @@ async def embed_file_upload( temp_file_path=validated_temp_file_path, ) - if not result: + if not result or "error" in result: raise HTTPException( status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail="Failed to process/store the file data.", diff --git a/app/services/extraction_client.py b/app/services/extraction_client.py new file mode 100644 index 00000000..ec421067 --- /dev/null +++ b/app/services/extraction_client.py @@ -0,0 +1,327 @@ +"""Bounded v1 extraction transport. Only caller-owned bytes cross the socket.""" + +import asyncio +import hashlib +import json +import math +import os +import re +from dataclasses import dataclass + +import aiofiles + +INPUT_BYTES = 10 * 1024 * 1024 +TEXT_BYTES = 1024 * 1024 +RESULT_BYTES = 3 * 1024 * 1024 +HEADER_BYTES = 4096 +CAPABILITY_BYTES = 8192 +CHUNK_BYTES = 64 * 1024 +OPERATION = "document.extract-text" +ERROR_STATUS = { + "INVALID_REQUEST": 400, + "INPUT_LIMIT": 413, + "OUTPUT_LIMIT": 413, + "DECOMPRESSION_LIMIT": 413, + "STRUCTURE_LIMIT": 413, + "BUSY": 429, + "UNAVAILABLE": 503, + "DEADLINE": 504, + **dict.fromkeys( + ( + "CANCELLED", + "INVALID_DOCUMENT", + "EMPTY_OUTPUT", + "ENCRYPTED_DOCUMENT", + "UNSUPPORTED_ENCODING", + "UNSUPPORTED_CONTENT", + "RESOURCE_LIMIT", + ), + 422, + ), +} + + +class ExtractionError(Exception): + def __init__(self, code: str, status_code: int = None): + self.code = code + self.status_code = status_code or ERROR_STATUS.get(code, 502) + super().__init__(code) + + +@dataclass(frozen=True) +class ExtractionSettings: + enabled: bool = False + socket: str = "/socket/extraction.sock" + deadline: float = 10.0 + formats: frozenset[str] = frozenset({"pdf", "docx"}) + + def __post_init__(self): + if not self.enabled: + return + if not os.path.isabs(self.socket) or "\x00" in self.socket: + raise ValueError( + "RAG_EXTRACTION_SOCKET must be an absolute Unix socket path" + ) + if not math.isfinite(self.deadline) or not 0 < self.deadline <= 10: + raise ValueError("RAG_EXTRACTION_DEADLINE_SECONDS must be in (0, 10]") + if not self.formats or not self.formats <= {"pdf", "docx"}: + raise ValueError("RAG_EXTRACTION_FORMATS must select pdf and/or docx") + + @classmethod + def from_env(cls): + enabled = os.getenv("RAG_EXTRACTION_ENABLED", "false").lower() in ( + "true", + "1", + "yes", + "on", + ) + if not enabled: + return cls() + return cls( + enabled=True, + socket=os.getenv("RAG_EXTRACTION_SOCKET", "/socket/extraction.sock"), + deadline=float(os.getenv("RAG_EXTRACTION_DEADLINE_SECONDS", "10")), + formats=frozenset( + part.strip().lower() + for part in os.getenv("RAG_EXTRACTION_FORMATS", "pdf,docx").split(",") + ), + ) + + +def _object(pairs): + result = {} + for key, value in pairs: + if key in result: + raise ValueError("duplicate key") + result[key] = value + return result + + +def _json(body: bytes): + try: + return json.loads( + body.decode("utf-8", "strict"), + object_pairs_hook=_object, + parse_constant=lambda _: (_ for _ in ()).throw(ValueError("constant")), + ) + except (UnicodeError, ValueError, RecursionError): + raise ExtractionError("INVALID_RESPONSE") from None + + +def _keys(value, expected): + return type(value) is dict and value.keys() == set(expected) + + +def _integer(value, minimum, maximum): + return type(value) is int and minimum <= value <= maximum + + +def validate_capabilities(value, format: str, size: int): + if not _keys( + value, ("version", "operation", "formats", "structure", "limitations", "limits") + ): + raise ExtractionError("INCOMPATIBLE_SERVICE") + if ( + type(value["version"]) is not int + or value["version"] != 1 + or value["operation"] != OPERATION + or type(value["formats"]) is not list + or not value["formats"] + or any( + type(item) is not str or item not in {"pdf", "docx"} + for item in value["formats"] + ) + or len(set(value["formats"])) != len(value["formats"]) + or format not in value["formats"] + ): + raise ExtractionError("INCOMPATIBLE_SERVICE") + for key in ("structure", "limitations"): + if not _keys(value[key], value["formats"]): + raise ExtractionError("INCOMPATIBLE_SERVICE") + for text in value[key].values(): + try: + if ( + type(text) is not str + or not text.strip() + or len(text.encode("utf-8")) > 2048 + ): + raise ValueError() + except (ValueError, UnicodeError): + raise ExtractionError("INCOMPATIBLE_SERVICE") from None + limits = value["limits"] + maximums = { + "inputBytes": INPUT_BYTES, + "textBytes": TEXT_BYTES, + "resultBytes": RESULT_BYTES, + "deadlineMs": 10_000, + "maxSegments": 128, + "concurrency": 2, + "expandedBytes": 32 * 1024 * 1024, + "archiveEntries": 512, + "tableColumns": 512, + } + if not _keys(limits, maximums) or any( + not _integer(limits[key], 1, maximum) for key, maximum in maximums.items() + ): + raise ExtractionError("INCOMPATIBLE_SERVICE") + if size > limits["inputBytes"]: + raise ExtractionError("INPUT_LIMIT") + return limits + + +def validate_result(value, format: str, digest: str, limits): + if not _keys( + value, + ("version", "operation", "format", "inputSha256", "textBytes", "segments"), + ): + raise ExtractionError("INVALID_RESPONSE") + if ( + type(value["version"]) is not int + or value["version"] != 1 + or value["operation"] != OPERATION + or value["format"] != format + or value["inputSha256"] != digest + or not _integer(value["textBytes"], 1, limits["textBytes"]) + or type(value["segments"]) is not list + or not 1 + <= len(value["segments"]) + <= (limits["maxSegments"] if format == "pdf" else 1) + ): + raise ExtractionError("INVALID_RESPONSE") + size = 0 + nonempty = False + for index, segment in enumerate(value["segments"], 1): + if ( + not _keys(segment, ("kind", "index", "text")) + or segment["kind"] != ("page" if format == "pdf" else "document") + or type(segment["index"]) is not int + or segment["index"] != index + or type(segment["text"]) is not str + ): + raise ExtractionError("INVALID_RESPONSE") + try: + size += len(segment["text"].encode("utf-8", "strict")) + except UnicodeError: + raise ExtractionError("INVALID_RESPONSE") from None + nonempty |= bool(segment["text"].strip()) + if size != value["textBytes"] or size > limits["textBytes"] or not nonempty: + raise ExtractionError("INVALID_RESPONSE") + return value["segments"] + + +class ExtractionClient: + def __init__(self, settings: ExtractionSettings): + self.settings = settings + self._admitted = 0 + + async def _request( + self, method: str, *, file=None, size=0, format=None, maximum=CAPABILITY_BYTES + ): + reader, writer = await asyncio.open_unix_connection( + self.settings.socket, limit=HEADER_BYTES + ) + # At most one chunk may be queued without drain, including on cancellation. + writer.transport.set_write_buffer_limits(high=CHUNK_BYTES, low=CHUNK_BYTES // 2) + digest = hashlib.sha256() + try: + path = "/v1/extract-text" if file is not None else "/v1/capabilities" + headers = ( + f"{method} {path} HTTP/1.1\r\nHost: extraction\r\nConnection: close\r\n" + ) + if file is not None: + headers += ( + f"Content-Length: {size}\r\nContent-Type: application/octet-stream\r\n" + f"X-Extraction-Version: 1\r\nX-Extraction-Format: {format}\r\n" + ) + writer.write((headers + "\r\n").encode("ascii")) + await writer.drain() + remaining = size + while remaining: + chunk = await file.read(min(CHUNK_BYTES, remaining)) + if not chunk: + raise ExtractionError("INVALID_REQUEST") + digest.update(chunk) + writer.write(chunk) + await writer.drain() + remaining -= len(chunk) + if file is not None and await file.read(1): + raise ExtractionError("INPUT_LIMIT") + try: + header = await reader.readuntil(b"\r\n\r\n") + except (asyncio.LimitOverrunError, asyncio.IncompleteReadError): + raise ExtractionError("INVALID_RESPONSE") from None + if len(header) > HEADER_BYTES: + raise ExtractionError("INVALID_RESPONSE") + lines = header[:-4].split(b"\r\n") + if not re.fullmatch(rb"HTTP/1\.1 [1-5][0-9]{2} [\x20-\x7e]*", lines[0]): + raise ExtractionError("INVALID_RESPONSE") + status = int(lines[0][9:12]) + fields = {} + for line in lines[1:]: + key, sep, val = line.partition(b":") + key = key.lower() + if not sep or not re.fullmatch(rb"[a-z0-9-]+", key) or key in fields: + raise ExtractionError("INVALID_RESPONSE") + if any(byte < 32 or byte > 126 for byte in val): + raise ExtractionError("INVALID_RESPONSE") + fields[key] = val.strip() + length = fields.get(b"content-length", b"") + if ( + not re.fullmatch(rb"[1-9][0-9]{0,7}", length) + or int(length) > maximum + or fields.get(b"content-type") != b"application/json; charset=utf-8" + or fields.get(b"connection", b"").lower() != b"close" + or b"transfer-encoding" in fields + or b"content-encoding" in fields + ): + raise ExtractionError("INVALID_RESPONSE") + try: + body = await reader.readexactly(int(length)) + if await reader.read(1): + raise ExtractionError("INVALID_RESPONSE") + except asyncio.IncompleteReadError: + raise ExtractionError("INVALID_RESPONSE") from None + value = _json(body) + if status != 200: + if ( + _keys(value, ("error",)) + and _keys(value["error"], ("code",)) + and type(value["error"]["code"]) is str + and ERROR_STATUS.get(value["error"]["code"]) == status + ): + raise ExtractionError(value["error"]["code"]) + raise ExtractionError("INVALID_RESPONSE") + return value, digest.hexdigest() + finally: + # Abort instead of flushing pending input on timeout/cancellation. + writer.transport.abort() + + async def extract(self, file_path: str, format: str): + if format not in self.settings.formats or format not in {"pdf", "docx"}: + raise ExtractionError("UNSUPPORTED_CONTENT") + if self._admitted >= 2: + raise ExtractionError("BUSY") + self._admitted += 1 + try: + async with asyncio.timeout(self.settings.deadline): + async with aiofiles.open(file_path, "rb") as file: + stat = await asyncio.to_thread(os.fstat, file.fileno()) + if not 0 < stat.st_size <= INPUT_BYTES: + raise ExtractionError("INPUT_LIMIT") + # No cache: each job checks the current socket's contract before upload. + capabilities, _ = await self._request("GET") + limits = validate_capabilities(capabilities, format, stat.st_size) + result, digest = await self._request( + "POST", + file=file, + size=stat.st_size, + format=format, + maximum=limits["resultBytes"], + ) + return validate_result(result, format, digest, limits) + except TimeoutError: + raise ExtractionError("DEADLINE") from None + except OSError: + raise ExtractionError("UNAVAILABLE") from None + finally: + self._admitted -= 1 diff --git a/app/utils/document_loader.py b/app/utils/document_loader.py index 5a5576b5..d25a64f6 100644 --- a/app/utils/document_loader.py +++ b/app/utils/document_loader.py @@ -32,6 +32,18 @@ ) +def extraction_format(filename: str, content_type: str) -> Optional[str]: + """Identify migrated formats independently of text/Markdown MIME overrides.""" + extension = filename.rsplit(".", 1)[-1].lower() + if extension == "pdf" or content_type == "application/pdf": + return "pdf" + if extension == "docx" or content_type == ( + "application/vnd.openxmlformats-officedocument.wordprocessingml.document" + ): + return "docx" + return None + + def detect_file_encoding(filepath: str) -> str: """ Detect the encoding of a file using BOM markers and chardet for broader support. diff --git a/tests/extraction/conftest.py b/tests/extraction/conftest.py new file mode 100644 index 00000000..20e79ec9 --- /dev/null +++ b/tests/extraction/conftest.py @@ -0,0 +1,143 @@ +import asyncio +import copy +import hashlib +import json +import os +import tempfile +from pathlib import Path + +import pytest + +from app.services.extraction_client import ExtractionClient, ExtractionSettings + +CAPABILITIES = { + "version": 1, + "operation": "document.extract-text", + "formats": ["pdf", "docx"], + "structure": { + "pdf": "ordered pages, one-based index", + "docx": "document body, paragraphs and tables", + }, + "limitations": {"pdf": "Text only; no OCR", "docx": "Body only"}, + "limits": { + "inputBytes": 10 * 1024 * 1024, + "textBytes": 1024 * 1024, + "resultBytes": 3 * 1024 * 1024, + "deadlineMs": 10000, + "maxSegments": 128, + "concurrency": 2, + "expandedBytes": 32 * 1024 * 1024, + "archiveEntries": 512, + "tableColumns": 512, + }, +} + + +class Processor: + def __init__(self, socket): + self.socket = socket + self.capabilities = copy.deepcopy(CAPABILITIES) + self.requests = [] + self.mutate = lambda value: value + self.raw = None + self.error = None + self.hold = False + self.entered = asyncio.Event() + self.closed = asyncio.Event() + self.active = set() + self.texts = ["First\x00 page café", "Second page"] + + def client(self, deadline=10): + return ExtractionClient( + ExtractionSettings(enabled=True, socket=self.socket, deadline=deadline) + ) + + async def handle(self, reader, writer): + task = asyncio.current_task() + self.active.add(task) + try: + headers = (await reader.readuntil(b"\r\n\r\n")).decode("ascii") + lines = headers.split("\r\n") + method, path, _ = lines[0].split() + fields = dict(line.split(": ", 1) for line in lines[1:] if line) + length = int(fields.get("Content-Length", "0")) + body = await reader.readexactly(length) + self.requests.append((method, path, fields, body)) + if path == "/v1/capabilities": + value = self.capabilities + status = 200 + else: + self.entered.set() + if self.hold: + await reader.read() + self.closed.set() + return + format = fields["X-Extraction-Format"] + texts = ( + self.texts if format == "pdf" else ["Body paragraph\nLeft\tRight"] + ) + value = self.mutate( + { + "version": 1, + "operation": "document.extract-text", + "format": format, + "inputSha256": hashlib.sha256(body).hexdigest(), + "textBytes": sum(len(text.encode("utf-8")) for text in texts), + "segments": [ + { + "kind": "page" if format == "pdf" else "document", + "index": i, + "text": text, + } + for i, text in enumerate(texts, 1) + ], + } + ) + status = 200 + if self.error: + status, code = self.error + value = {"error": {"code": code}} + if self.raw is not None: + writer.write(self.raw) + await writer.drain() + return + wire = json.dumps(value).encode("utf-8") + writer.write( + f"HTTP/1.1 {status} Result\r\nContent-Type: application/json; charset=utf-8\r\n" + f"Content-Length: {len(wire)}\r\nConnection: close\r\n\r\n".encode( + "ascii" + ) + + wire + ) + await writer.drain() + except (ConnectionError, asyncio.IncompleteReadError): + pass + finally: + writer.close() + try: + await writer.wait_closed() + except ConnectionError: + pass + self.active.discard(task) + + +@pytest.fixture +async def processor(): + # Keep AF_UNIX path below 108 bytes even in long worktree/pytest directories. + with tempfile.TemporaryDirectory( + prefix="rag-", dir=os.environ.get("TMPDIR") + ) as directory: + service = Processor(str(Path(directory, "s"))) + server = await asyncio.start_unix_server(service.handle, path=service.socket) + async with server: + yield service + for task in list(service.active): + task.cancel() + await asyncio.gather(*service.active, return_exceptions=True) + + +@pytest.fixture +def input_file(tmp_path): + file = tmp_path / "input" + file.write_bytes(b"caller-owned opaque document bytes") + return str(file) diff --git a/tests/extraction/fixtures/body.docx b/tests/extraction/fixtures/body.docx new file mode 100644 index 0000000000000000000000000000000000000000..77b5f53b237c22fe623d7d0d9ffb1a35f771a63d GIT binary patch literal 36732 zcmagFWmsIvwm*yqcMDDk?(XhRa0%|(xVsZH5Zv9}-607U9D+N+t${}WotZP|&YW}a z`+jL2s@D3+s@l7%_pVlvhlat1fPjF9Xcf`YuU0MooD2y8@f{8V0t?)#Ck}9MHFt0| zRQGZ;chP6^w6|+cR#aXQK@UHF#z|ooBJ&YP!>QPIq;X{UOdwI43FOgPVm?)X0^N=C zMQKc{M8Gk9&Pu=YBUxz>?D1`((i4|B)z{C`DtrfB>RY&0z;tH8y(jsZTy{}rhj?xn zuC6W1_w5uHB+E|SAfF*-*tuAqZg8?T2qKQUI$jf#bR-O3Rq#7(WNqOzcYYUzqq^$R zSLz*ltG(|k?$6@^9`8Wnfs#4TOH5Rn*|wR89YH*4QAQUUQsG8Ti9;^+?i2Kl&XyWZ z4J-Y{sGcK80Q8w%`#vU*F)j8ZxQ6nW8#Uc^Oq;OXQ-EzKRo(0%bdJ`m1ApG}iYC2P zm#$8nx}Wt?hzx=Eoj7XNabGYkbtAd{7q|?zuJ~%-(7VCAJKEy-_d=khNQ1k%Sw;JC zjli#H;hY-6^{Sns@hRWfdAVPmj>O;j0rNujmU$(V{JJCz1!;mrhJ5ZXQ6Kk>VSnP^ zBL~&o$vkzJ?76K37nmwAq|oX;CleQ8^nNye^m#xkt$ZsX)c>?ty0$0${q9}CBetRe zgvvig(Thr6rfC7NZ`MQ&gOP5%uK&Os}m>Wpjpua&xItvQkFWjXh;+-8%P`~ z776vGjh~e`_{)_1?CJg>p|6eGPjYa+V`{OGSxC4a1 zpMua8MDD4+9jarO5e=k36)~r}Kb2Ou@7j5Mx)*M7l`#qwIZANEP)k=bW5wesLX~yGx)GN>n2M`N zXwSm!c<5BI(yP2=It&q4Rx|K?e*-tsg*n7y*(6Xp-L@mW&-k8VLd$#7!fid|LxAHH z$q7q+>bY)n{E{F1w`o1D^M4j7G8{?G8a&OX!3Dwq7sw1?s^ScAbYV6HIGg{TJiAS`B>zxM4MTA`TvH^Grh@{7Y!93ES5Od1R1!sEBSv{^ z2?<0NMp-^&RLYRC!lVJr>{2B{i1U^mj5|S%uJZQbx*jXRN(;)0YhA~K=@n~$Kd~-K z=!x{Ok?MYyIZYiL8_}sd2o@O)l3nCOx~fN;CfnwxQx|8ajVX6%{YvE42xQchxB1hj zWmO`<`S7cH?(2Y|;`-6Kx$RX(7s^0kSF<+uI;`iGF&^qvWB;tReAH17LCXk)>@RmW zm)5MAUu3aMMYCXJJ7i`UdL{}E8v|aYU*0Ye(cg1l!r&iSq8iZ7dWgR~4O|iab6jp# zR`8C&<6;K`0fGMaaWQsu{9{0+toeZbGSI@&SBDkh&snmS;dDmz%eo6Ai(M6el%JnCz$a<4cNvD#d?~f;Q z5y=zx`um}aWR>Qn^|TKx%AYEspKGluy~?HpIi+`iM>;9VD5z3U#?S09V_uckG|Q1v zc#33%tVZ1z<5n;>NItH>2D4und6xEicmlH&BF7{Qj#snD_d|%gc&0_0BK|*sQjV3` zXEZAic%Pb%8_Cof%C#b|Vowu#wM9qscsZe;#c5)`%}^xU7z6rmFmknnR-F>sZX3(_ z6Hw)EQDGkV#g?9MCFp*NztJU5NNP#P3aI}MxIW?LK~Ac^g4TxobhZ1qpIwLbMZ}tD zxdY?9-{iEAWc^rOrQ9}C6}e!|2=)6E;ztuwLV8o$Gp&OI@kL#ICB0ajTuuTEcgn3W z|8~ed05$9vTS0skX?NXi(mN}HXLP1Zd)9+ZkeeXVkp~=)e_JC0N$M$r-SfAYF_NG* zyN%Rjp$V;x_xurAcp5%-%AZq*R%C^Xm8HA8GMS&4NiGmFLDz(@^S~3*e@+aMvBs`E z@WmizJ({D?JoosM2VpVQVs?WKE?kB|tX2 zO%!`8n&R}mfcTJuU_|!BLk`+}#HfSvv!2NSTutcg4>_@wQzbsEEhiNi=mQ31Xt-BBImDgO*#cPmMiK?G&3Z>_dgZ z%!r+5n^tF|Ik05#i)h=99&ZgeTyYt{R>G2EcdC86@5Y7n1>&Jh!?4w3u79ze_* zrJNQSa`;Xu-!ZGv+}RW4er`Kq>~ekW?NgbrDo7xVcB&-gu6>XGgr><}I5;!zWk!Nn zZ~;yC>x3ya@0%1E#fj&pgIQbOqBZodb=f`KQ`}B%CwsfRNUCkt%s)(uNRE6$Z0ZZH9H5|p8}P^rj=IFL6_Bc|gwl6h%)RueL~ zpIF^q{MDJmx>Utu5O9n}fAsnq9q`ZT><~zZ$p*g9je+lTMBw|}AJh5IRr>Ff`p*^n zi>Pu43x4?d3*tan5_XA0^xKmHs!lqo+G5`@sh^3r;(Hr#1f+>WAN3VFQI9qXDmf9BdRVjP4C=X>rhFYj?*xLzP9-Eu#FGJo+2(CVF zpoDjXVoHg&d5N`cKa6Qf9}5m#q+te;QQPCX{4CSlH(HhF`FS&43&RTvnPL8Ct>4Vm zrn@jhKwxUXLZJOsYZq59J9C%cHxBS_G-+F^!LPTspwRGX*FGT>Vsz30BGH}5*rz+E z{+wNrefH$X4?L^*Q^NpCauHGuRSiwKio^!~fgD^F+C8o8Bz5!W=iGoUyK{d-(zwj$ zvo-F)UX-48y>rdS(uuR{PJsV2zAry$|IAbOEGK2<+2Q#tQ+RO{@B)f|1wAgF$CV1V z9`ESigVruPQ|CJ$4`zP>J0l`?j=Y{51YYc4JJ!RP{eO}gq?BGv4IV_K?rn{&9Jf|J zj~!n%d9B)UmkN_s_4jovhx10pG`!weH|qL4&aaGldwddpb<_q7V%&A^T<-%t-=0=} zwKBYmC|w-%Zx$jMwDe!j8THX|a9=bIS2E5^oj9r!J`>J8RMx#~vvV`NYgxOO6RU#t^!rV9IeMXPuE@}FXn*G47W)S^Ki3)h=?MGrgfVw@#+!%X1w6kT3|a)hKi{l9yJmcRs`Fp&+7J+|_BcO_Naa4u z;Z{C=^)op47p%#3SbW7A>Pd-d``LS{bg&W+*uLL=t=H|9$|3CO(39c<(LVHXdEc>8 zQor1efn$a)zt7vlF4Y9eNA!#AYu$e|clqHUF-v8s{|Xpovz z2@Og3Mbi34&icE#_uP!0S`Skr8mNP!f@jtJ3pOc|b7|Tn_NWm_RKQLB^uizN0O0@j z8J>r1-oEPJkIO8*5pqykaWjV^peyNkf4cG9CBPs|no>LEF}dM=dpL8@7Qe(qv$|N8pZ*`P}n2YpyOQL$F+>)^g;1xeO5jYbI4@{hfH z%76>fMdi%a)My)+ACb27pXw?kc|S3zG|EM20iAM#Q?P8 znX|3C5?VinEo$?*<2|0fT>P@9&mtL_!SKZ)zlEO>^(8i>k&P*);7JlXx2V+kx+?Lc zP985O^FgKzQ}qjbWhlcQHa)pKdLdXwxm$|La zq*Q~qwYtDy?mOs_8vTw*PY1?lnp%-yM%fYK=B2vQiA4oWnA0-urGBoR` ziY5GXr6q^AT*vM+gugBc#18dR6G8mhMN5Wczh0GHC zD5tA3Y?BfIsl)S4PXt}*3Ku;zsSEZiqu3R0LU2MCY*F6tPAhOH?q8iSTkipoqW_Ao zX(pg#f?@VT`Z^%H4)quIDMunWl?)t5`$wuS7%dEp`;XK=sbI>f&uz#(|AXNC0|mGR z@edU#U9e{P|0VH%pc1^VYH)*7!BKw~=1vIKqlympiruu!NQT7t9}*eK;HkZk;QvrP zm*mlm-t{}`-%Uj>;!Tt8ojq6vIeZej^JG0Fnr?pQAjkln0A13B&g4EhM9H93>Msdp zZ7rTX8v*TDdQ0{$L=xI_`%az9qhoe23^9UCdMQum!)uQ~w|0GpVIR?_!Nqa+e5)nT zYrXrC+1%PK&)Q$}Ldpkw``-$-?qfHE^;>}wq;U@0Ov@t4ALGtBKNGw@IwFfRdQBOj zGpv>qzrVvo({aMms-HHkL4&QIIvA@j8#OPx3pm&IXj(hz`B;-Htt-Hs6u;Kw?lsD> z%$U$jv*YCJ!IS>xfSdZUTWM_6^y+o&v48_>-Sqz8_~rg=-HBTEOs4w)eXHda>Iqp;OCd2ixTC7F}8JAlBh1(@0qG{W0{*zi51+iwBifrin53# zY>l&C{36NnrwajD5|~==OXik`*e~Y4^;9j^3Wr^hG6g)LXT01e0Fy?6rO)JZf$UM; zJ4alXE4P5suloM?em*!~+|xcy^o_ly*ulja>^i_r`d+oX<>~NUL4NXlIp2PgsDy_$ zIGb^o?&olyxG!WCLVgy63vV66fQhsCC1G1WF`P7(Jh#6sd$k*Jr9umQAAKx~TSBb^ zl{55J7s#vCz}@k^JzR<>6-y!9CH~-}mzML}n%#G8i4$&=gtN(Z;`-l?!s%)Gq6XbC z-^|JqK~lG4FxvZ#EX-SS9lW+|iBiAo`Z$~vf)!LQbA4qC|4)C6Y^)AbR@V zJDpDWu^z_OZ{w9HR8wv9jzmf@u=)}n4o=iTm2wW0O_7~MHB8Q#L^S{t*H{vrHqhR_ zm4cmU9@B^eR4|3Jk=6-Mtj@L^;21bMbs0Wr{Qt*hT#b|&;IbiMjN@{|0u~H6J(smh@ ztc=?UL_udDF?R4~Uaj{;eCkd)V1zN|dWx{y(ZS&(u*^EO8#$t(%bAPN8>_bKJxdcC z(&8Ki|253V;nE2OTTOtY&jW;a@o;PFzy0u*RhC-69Z4MK#d_ z5X~pE0zs<7dNZ>neg?}IU8{a^aVZ_j3ky3+h12Y~F19mV-6NK*jNECca^&I^QwHPj zFB`R}KR@L7@uYFwTwJ%Fx=-9bcJ4M?%sUnbZwdGG_Hd5E9?w8iBTq7Ok)YUVXit;{ zW8p;6WuT(A(4BN>T&g_@QtM%Tv*kN9o{J&jnRfE8Uq7rXsSyOisBdu z3)Nt)(Zh*$?3y^M)^?3?bMkA}(ZA?($IzZ<%QeW!z@kr0C5*>qZJtvtk*y5F;?`>B zAm%l(iQ2;NF!29agX&}Pd_?J8{I#%#RD`?&*`9?9LK-phjBkB~pMbh5`YwFy;%Xl5 zj(BHmk-W=8`?jy#b2PLbiT4_H<*mkQIo4#2tiOP;+wdaN2jTeE!F`2_o#3leIWdV^ z2ozgL0|bw$4zcxsZLJSBe%z$V1!vY9Hfq@?ef*@OiCRBFHX08YpbQ3*at0~J^4QrZ zDEXcw{2}3!y@E0a46bEuw~7!`F3G!p}@RG`EO*Bko+8vOisPZKh}w1xxkoO2Y|zmSC(uXMM{zg0*yC(&>b ziu|zW7gJd(CTzJEO5Y4C!tT)l^xeOoJw>VTZXoyO3vUk~gyIOdzzv!1+p^kk5Mobl zpI`e_TWf-6Zyr`2jI9Bz=YAxk<2-_%Tz5@7La(WkBGidxXbx=s_~PSa&6cAj+5yQc z|7Q905c7Re&MoEeKE*5t6UL#1lbW0>C7*O&S<+$heI#Uw+)OMpkFs#ZlL2gDanMK0 z7|_c_lTcP~bR55_dftR6qWyYHI+Y*0NG{HB`-!q4pF^Nf*K$~jcCXgsJ)kaL7<&%j zoSA4sHcSmoSsKHVQc$EM8XTUa7nSQY@XJwWmhv9Bzh`3ccr@sivL$Vo)C1VeWjyI9dlU>7}F212z_T{1#xYgxcKk%erx^>LibbjqNpl`o8CQh;*8sPua z&a?pU*`S(a2(Pix%=Pr6Pdf?OdxvIi!SlO_Uhi5G8jeTzlrJuF4_|qx^GzMOijAJy*{A#b?ewoQ(23)1ieX`#v=Dvt6R^b!8p_k`35UH{q;x*h{bZLq2M zgn*UBlV0g32`EEUl>kt%-f*#DbT1D^v1d18{Xh}$favDEUN+gfBEin;qmQ7xDcK=n z-bC@NEUYlc@TM5U^k#pPrTxP2m1&c1GQC?0koFs<2H)F7Q3B9&$^@77bA^%F4Wb)% zr~KHFtr|&rGkFlxd)6rq?*gug*FXd}pSyt}pWC!gb`unrx}lPu%l;SxF46ngb)GlE zQV*LXuQX7zbjocb=aq&06^TEWYXR&DGjPIlMnz7WsT8!LPf3e6JI=tpYvWg33pqH( zZ$A%CPI$-rL|>R4ThoX6BZ%{%8Yg$k>0=4jSkt5{!H)wSGvJp>C(!-({Ip^ zG)^1yJ(5Tg7S)b(nF`jkG#abvA7Z>|H^ZQERG25oz1!5g%Tnzk%=mHU#*Xd3NOfO~ zNQHW9JSH8kt-|jD=h(`l%eXD)?Z)DT3JnlF^8=~p{i+Om)>+Frc}QmLi%@TC)ak+EBgt+v{`V`g zh3f&tT*^dEguO7Z?Ws7EFwW(oQD4*Aa{|f=U&8t5Af0$cyx_r)NLUGW+kV;!NzwUv zyf!?Yzg+m^W>6ED6Pv^=J2fiw9#StBEanz+vkABJ<*23}WR@xx$Vg(_T1cR|p;{Xd zPPKHZa*9{R#6TT&E~b(hu}ay0#FbHNZht*f79a@Aq8lyFNwi;4DCp_Hm*`z|n$32c zUCj99s6=86m<^~t&U#Z?z=x>|wK~z#;%ZbwT%fp~5!r<=V zKJK+E0-#w}D!ZDyyhwT?oR+BJI;3WpmG11r1?OAjIaNLS6#Gmzoo| zf0t*1p)%^3=a}~{FZ^yJ{+mqW=4=bROiNNzwu0yg&=25B%;+NyQ|6xN_EBYtfRC(e57n_9W2vS#+aC=;q zpftaq#w^&8RHeGIX>MgLyo^w3)YSY3nBDC8EGDRdDVvgaMovgP+4I_hbXC-Ohiwg+ zx4-PH>AdlY)}C?#GeTQUu$l*(i&k>;w%2eF`L44-lrJm<& z`*tiBc3H`;HE33glMyNr&?E=sxNRr!fl8eEwdLZ54XGdWZgcO#ln&}0Hsr5En}3^A zKPTZo_)%I3&un%4e;K@p4=_(o z&rM!Fpo=bF#Pd2&jtkIU=dN3rOAxb9ZTXW2JqWd6qpP`5IwV)EPXJ%f?PqwTt@5itzzI`+Q3{{UUk;0ZU1&4HIoUD!YMgr07DDW$!0tq@RS<5@-$a^`h zdqhbK{?9siFkgw0vIJQ=rpCjF)*z&*;b!lb&pVv(CzQGt6Nj=A=xAhTWJ5}kjdcOLpz(uOjz}eS+UbkB`o~H#>d;km$8^1$4$OY zqd2;nus)Eat*SuHo_sZ6)KR$H)h3A!$|aR&C`mv$xHVzjaz$*KEqD)42P)Mke$G{xn)2Gp$@5WZn+QI3)(D4J^`36ISnR-ZIK)S`xoe zQ`QUUi^yT@hAob?@%^lfxDc)|k)lAMYVxh*MNIr+dEf(eNQne%SrCOI|+pjCK#eMeH zQoOGpESZv=U%na68xXfplmu!RXS2U`ZfY^7FVXKP&@SJ}xS0**j`Ejp5PL|553DJi z%5ycvE@Tc7uL;}CCWO5%GeTWCEg ztOIb8lhU6_y#JkK#{?+nPNkld%}V021XB%}*&BrbwzITHs3)znzWvxq=CNd~pal0f zu8@WBl=^pQ?Pn!RLuwyML!uwN|J`m&D{IogRY%P4q{Og;b-*!cc6ce5)rdQf3GQFE zPY}&g6JDctnfx9?$N&r>jCA>qLqIo(R5yeJ&B0)Yy ze2*`I8a3HLG{>ECB8_&ckd6e=9bU7!gzu{f)(&a#^OF>gF;~VkIwmF+~aU>RsRa&F)iTzgPJW-hp1}}7%@01EH z*`^r@>Fm2PLL@$_L@2|mJ$+JEf6CxI;vbuia| zP*doEsaNVT{-91_O4oM%zo_phmB8EWr>p%j!6R6C6@Ugxieu)jnK43_DOx|~ZZqYw z{%g86QNMVXvvx3L_#h@Zk_TZ*(_4POa7wr%VziV$cG(kcYlRknk=KfY$us%T{~(Xe z2a|98pT41{Ji4CtQ~pJc11JAIwPQf5ts}M|0k}id| zllw(UElteo(O%rq6HZDV$~DV=XB7p2Z*(!o*&rN>>hxrN2FbJS(W#wAd~YLXgc+|I zFbD3(C!1Px@1wF#!u6htQ@{Jf`Ounj>iJXIFhXN~VEsu8odVT}Z4k0T8D4PlI#Ljg z&KsKaJ_`$wPHi>u3|Zamxd8^;2X>unLbS}I_m+uH$~OuW-c&~XYMdK9qDeUwd`DN| zYE8F~iB?K8M_TC99O#Ob_X7&u+{n{Ss6RM@_EE1~4E2a9MDj{Ex?PY9tQ+>ZN%-bv zXW#GSSY=3YDE7IR*z`DMPzh?d&F!I-TA$uo&t#R%*OC=jlVxbG(~qg1CrffvLvtwR zEyq5sE^59ZMnC-J!OR+Bl~DzXjZku%;rP}@ol%0Jh*g(eo&m@zkyX|pcJrFG);;z+ zb}hATrjN8Dh%8CILv5P#d4s8V455f+^+G9p*F66L8OHlzO>dekgUZ%_Ctod_Ly_$) zs>E7i7RJu9VxO?odPYq{T7TU!W5J(PdBR%v8Sz~PT!Hmow>l%xph9AnPh=J*gB%L? zsddg6qh+>mvD*UWFOMDEicEh4%iPGM;{R<@vDCWyj&mWqWWMHKCiQ-*o--=`X_B|< zx5-hR-zLxP|Nl%5TI)Xgt#g)I1HmRUSQLN1B^x@u!0#^mvE;OhOwS&*!xM6ts<_I8 zH{~FV*0(2Q{Wg!eqNrG;qHjjbnYYS-wRfI=M5To~(^BK%1u?mGrP;^e3B zv~`ooWifmam8C(dkVCB>gCmS2T_K)@A~Y)VHy~t$xlAD}E*h98Pa#5vHpch$KOiL# zVMs-GA<&^Dkl`u%U}XznyhH{25p1fd`D3t+NW`^+ewjC33S10zS1C3HdRN+b2CevU zMB{C)Nh2_0`=rC}fYKUe*G<9&%S;JGz{?~xc$rKm5kZD$2BU4m{zgNx5&Xdyg9k?Q z3~J=I;>u|sFj$~65b4(7?CP@*NgYPI;PMQ`Lo^m{#fwI;Y%Q5U_JKDDlnL5|jU<9F z2#kc>f^8#{?IL3?WA+p+f2$mE2{mpC&s^7gdL;m8LBDO_x`aBauGoTg5H4Xb2)r|W z*QH`NfTv*Lf~x44W{hS7e3JlWqlT0K{R;}b%@{y(6ixILHBeE$2-ek#MBRZ+8$~xT z@?I85L*uQa)cKn@}^5dXYQ-z4e$D{s(Bz%T5Cl(UK{^hqbFpKc=m;uQ}2qa z5WdU6x_P*ul@_rGqLnm{2&&m2<$o0Kardv{otj?{+l}Oe{-qG#fX46u-f*4A8jeH- zI+G7laadgRhx}dwdYLLO4MeJX90C-1aG~70IGclG=q@YYm10%IzzO^w!e>OG9Y;FB zW(^~y1+|~ojW|+-I@e_L|VZY48>n9P3e4%d={_8BBd`^MjsTngI9i1?xnGp(9%1-WuKabrfgUeSeFooJ6||J zpdA@UzfxOZv)aSbyT78--2Ws+erKx1qoXW4J_X|_KVBOmMYbPdl3ErKILhS4I_n)h zUi+-+aWt-sQye%N#tc=Xe_X=edu0)v;>|8<;!kYkpA>XlywRkd4Jt3dzGyRgQ*QVNRZCrpx1RPNRFl3f+HEKB#G>iBeR+9){_}%kw;+vp z#FYbTzM2!E-U_G3fu{@r>&m2%Fn1cFFeR%nbRQ#YFB3sf8tEtfuJrRaEuOjnmt4c+ z(^8CE5bbZG3qSv6+cnpHt@X_nRltSfcrEleGrqEZ1Xe!N7Z3t6q79FMAVX2FC$Ox} zeY;@iyE|GfK`ZrmZJ5QDz3;)n8`M&mN$>v{D$kcW)6|YM>u$;Y#TQYizv{;?bE?ao z3`#yf4*?u!a*5x>6e#NKP@QErGDY%5p&%D1n~00kd|CQh2Sdr;VwZ^dwv*bmB5V+Y zifNWWd9Nh=w1`qBkF|4!WYghT<=#LpHzd4{q`L1XGIkv&2S4JmVb?4DLzoj`8ooT47vgOtCe)E^9bWS zC@|jSABgyM2g>=Cjg*+=me0ml5L&xuyaj0<+x7VD38|B6(cdSN&RA5E&Pc%u%VDTr z9DDZHCKvv2?SI^VBZ%{aP6b>I5?{v83(oCX2YmmsNO({hoLdend_z=Li>K0$Hov5P z!S43(;jFJ{kvwHrueuR*hV!K2_py3YtKLKF*-nY>{==udV8cZufBUS!a-{X&f;S^r z$`2;f$3`GSr+O>_*-&%Douh;1B^B>xD`)i+MEcBjb zEdB*NisWBSdCtz<~Ff7oAx#O0dFo_tt6e$~H$myN?gnj)U(e(7fsHjhk4Scy&AZ)7_`*UWiWxJ$OTcUB@cI3agcp?veOY0CpRXM;OjTV0%#?G{N4>glaS| zT!s<-nQ)%K(g5I+knRBCkoaDCtSRUX3%tCObE-g}J=rjl}vs^x#j>vqKJgMk(UMwPNo^?qchc~y5LoXw< z#(S>8JDh}O*gUhB3aex=qp_{h0h^v`-%ntPMm0*Hh?q%6R04N4L-66_?hxXT?kp(( zrGOxY>MRWl0%lZCn_no4XSTgf6h4j+5dYG5S@LBWi6^d70hY(WF8L!ko9o-3^!%k5 zf6@yP^Y1`1gUxfJjry(VBQJ3>m;6`yTON|p^U_w&O%_P(ANPB}&N0F~?WbS`-M(#J zk;I}Zj(64wVf1bRQ>V!aIdsiB4Gi7aaMdaryf(0Cx@_|yqYU=ppJwheXOT6mPa=X1 zT@x!o4mV>(@V{&iMydzPt#1@5Emls2a)8r~`r6Ol--r>QgW>YLEMpE{emG)>j9C zOT|KW$&&4rhNlhb6F|?(YGOicZn7|-6SJJn7a3jdT%YCaqBgMS+-+9{35`WM`;EXV z#!g6w!S}O|eQwgk$aUX!7$8|YU4WjRi)*i*%`-sF#GI;*4ZL1fDd2_`#L87Em}=rG z_)Z)H)jIS&K}V;?L&E~fIj@RgF_(+I zY(lQvY7k00y{~NOyQ_tHVM1#CvHfy*aJH#ARK0vvRbiFgZoD+tqpyesp`wX7<}c3D z)zvSK?H`%xZTtGbISp_G^w%FZa-v~oVD1*S5iHf(t5)ht+@sfkfki)On>hJ6wOlwT;uw`vU!`ZtQoUR4&-!p)%@Z~sOUwe z`im;W8cb#Mn<_+y0T(*nujlP=oBiVO6htMHH#U4+ zY#oQ$6$ATibg}q>K@TiIL0S&c5Cm5wRww-&)}b#aw4s`ZfAl+yNvgJvv`#vwyQQhS zB=|&iigvn$_XI6BH|0mB#jZD%{?J(T;DPk?srxu}y_}h-gJn|bQzy0l)iOzcp&C3O zmgd87TLPMJ=U7Tr$mz)lj+kDWR1kRo7}5Knokr^M6K*STWg9dQP0ZJf&0RcpQI~#1 zFGr;PopJ0t0Sik#TXt2@$oJs(gYo5TR&6Yk%qoU7W{WJB8J$+}6bVh1q-MJ;*M6e= z;;--Sj`e(W1F0Cx{aoX=O)J>*%?7qx(a*=Cu@q}RWqr87qAII1A9(D#J@Qw2=Uxub4ugR@4ZEsj9Ick9jXT^g=s7qx@?QD_+3TKQl5sLWIr zzLfNw(V}^MSPOkm=My0kSY+pnW%wEY6@H!52&-cin;DPBTgd4dg*TLxvs7ulkSf8N zN%(SZJyR@KTGIB3<*h_-@XMRS;y^e!bO1IYYz4Qt)Vb;gvg}e|ooM!wh<&Gu&!N%p z+mm}z#^iR$L#_@*s8MI35#4s0ejhxF00V(sTI>P&TfEx=E+_DJRJwOziQNg-MGzZE zvNAylV%ajH_N^*DGX10z)YtVFejZS?bFbqB%-$za#A~- zmiZw@3K3k1b-u8l$|v*Jj>eKjDh+6CvDllBWC*nAs%2QTWvAFS;Jkp79DlF_%LRDa z`es;8|Wy2nd&2I0%A&p3m2^ zcD0hUurN1u{qv~4pMGkgRKw{b2&egISUD#Jg8bW<-y|l~h{4#-W%}*CP-W$uenw5= zksMts__V%AZXo8@c$u%Er~YZQB5xDg*XE6OtBE$|VaUBQ>M?1=!}HX; zc)D-gzCt;&9}PT@UVAQYPen1bWsZ)xeszNjk-fUJ@w48!jpM2vb926$wRmh;q7HBn zrtV$x_wsq>vc7h!H}v}`SbG~#Vh>8S@DB)hZP5?#u-RD|^DF8FF|`Ms`v-Ul8FrO- z+F8q>@GwcrhHIG+% z0o;Jv#RW~9>gTo>XKQ*LIEkNpu^(=y-EFw9x2!KS6!KCd&i$Xd*6r{}bCdYxyq}mP zcb@}Z7uWO~1Oy#O2KGvaSFZh2NqC7?UK#|6`APUl1zz2r_}8oMIy-AO7~Rn-=WD+` zQ&#s@VTucnFLciPIa{B3N1!-w+ zbmKla+;wrN?bElKgA|GwF z-+g;`yVZtAiN_!eev5hE;QLgktkl&jtgVdt@;GYPQaQn;v%0aNTv{4Jn!;R3su>?Y z27G+n0u2CXkDNfSIVr~h=PB`vJFkSK`>*B05fdxk0lso7iuG!amwWFFl|w8D!4hn%k+dpFn$=`mCJW^iD(EmEnWaZQw7aJy`CN zqYTi+t<#jptaxzdOWlTp>aCvXm=nC6By+8D^qP$pAl8J(Ur}d$ZKJhvRM)34@ws|r zfO#r9VsCaIqty_~Ek1G^!xszt^sZt@_5e_Nx2R^D`Tv^}g+# zt*iAGcP$YW!-Vbiv1NHN$t*&w=fLQ0Pd)hYJ+wBHw4iFd@ICr24OGd_V1v-vVRA8PS`^dm6ZKPY5EQS2g=_A{*Lm$ ztH9@Mg>_6T_zA_MFzo~fWvJ671jrEK-y%H zZ2pb22E~Is^Bd>y0e|ycC)ETyr}u6h9EJ%F`)v)?*11yl0rmF!D(*3Ck9<#$>Zw*X zsSCp2cP@ZYn6Y&X2t9hth^;v_MQ4<dArPJ(FP7Mk2cDu0K&M^u}%-qKEo`I4ei`m8mkg%ol zIX>E~>ZP5jdkc6K@AvE5!TiG*Xm|PM$+~{xqcd^jb(mWF8Bf-vFGl(k(7fIEnYX>y zBg|B|dSOieb`v=GEq+Wn(|*QpeK&A&*5gYm=b0mL-#*u00yKL*mCE0~Dt1QkeRZ*V zBh_!E<6e2Vwffh$8GxQkZT?$Aj2`be>QC6l95G40BkhQiXJN}8X ztZ|0Mz9~= zj6^HMvkGHqrzd3T96hpI^r04h+ixaT3;wl3?pgwN2tG%h!m|LQKdFbp(@rROJ}g{x zOG3&D|A&~DF3MiH6pivD>RuE`WuXJ#R`OXKs%BW64HYLST`jJBtn@YlS#2Dok{DZo z*d^|Cp(#8qq}Z>1$q|9A2&^yZ9CVw%R&w7y6&5TQs!0Z9wTk# zRbfA_>)XD%Xpwhxi^RZ_X2H8MbDu;zh&C``0pXp#VQfQ@&%=Lk9gC|~V|@eklnW7dw`yB>OK7oit+tY9 z7&0m_+o{qxcs@&XUFz}uMXtN3b0;OI=Zg#eq-pQ<C>#iRkB$U<`Jb$%yz4Lee+=|Drev*}%%TtWB|8=^>$N4{^_Kti`*1XU#zyUx!YA(UR*lOhA*0tO;=3^Za-zIX!wp|I*5k z-3Zcyhc2jeCwXA@vZ>mMcS$`s#i7_#Aq4z;uf`=eIPm#z#`(CKw^Hejv}WXchQA!h zH@(g1=a0y!OEL@}F30lfMCY&A(t8lO;sZ!9XV^Pw1%1Y~rZ&Wvd0id4h42TifBrwh z-U2v|ZD|q~Gcz+Yv&9S}W?9UPw#Z^;W@ct)W?4)YGs|LIyyJV{d;9M9@9rNH>Ig;5 zWM$^*K2xW9`jlA@vqhs35A2g$L;j#=Y_|EgiBA@8it~(0srma?tG7sM{(y1(vlVJA z52@@2KZ2k6+~B#9_-@ZW1*G|$+sJO&3dnVMyYfN#!_iC=U{DIrPM~aP7OvP2FZv znE>v{Rgl_-pOZ_p^Oza!J~H; zvc@nHKFNm@ocKbMMPyfwjD2Cpjkjl~U*n^jKS$f}6Mu=KpznLiM~$9cy-QGe&eApC5M^K4eJ#Fw@U5AE(atjez0>e4|h)8}zR zXV1=tn6vT;2#F@mU$+X{_ycxwhh8d`{@AC_RmDvivNQT2hKEaXI0| zkx5lwu`LNEV*UK;5*l)_Kns<5x$ms&j(rvfs2`ceHQmVa-peEx5v0;pvk6)`tgt%D zDp9UZUgE?3afmE_`C2mno%Wll(M8WsGoPvTa^JF6FS=3f#~h22#kyy(9r)jF8CZU= zBO`AClTSMeU^!DU8ZCVU1UJ5RJy5`vX6TYZ>UTn{)U?6i@U;xqzCcL+N=F)#PZ;3fjWl|?So3x%VX=w0D-niuklF3rkM>opcJJHc z(xp3*9hr~MafHN3M{C-i7c``uPDZ22ed|UjKC8AL10*NFn$q^+`kG9C%`c;?m{g7A z{i_gm^7Pv7yDJ2XJZ;YpOc>L_Vt+X?IBmNkl?x<>!emDqzy20hn3a;1Dn8dio(Qq#5C}m zF*?B~%P2SS4eRU>2D(oL6&Yx2r)VC>4s_SM;=pWV&dUcP{3;>aHRRsNtQT_I z;Pv8m^t#xR4I~E5=oH@u2X|RusfWrY+Y3po9C$aZdsK(caNb z>E(FKz28>w*Z)5|07v9q?$-@x&Ngl=WN?44efT!+HQ;U*t`1}?xyZ5w+$#zNzE=tY zInVmzOxg@&^o{T?Ey%y1t`K=mo0aA2`Rk%|2D(9^)<;M-GWGAyc%r)M(&{RKBe4Y> z36-m9#mdc|iBjGy&0i)BR?AT`zwKL7`*UO41sc2h+jAUBInZQ&Epc`v6YUN94lBR6 zx&OZP==6|l$@bz3-Ps6eOT7d8(O|`r+&YuPj~JWT8oP3ORas5xxgxSjpTFFUVjn_k zFKI$M;{Q!ymHLZ22#hWW>80n5X%(QhP+k+!fWb?9f4pdzI)}8p!i<`T&1jI?W}cdU zhcUMsU0)krKixaUl8`qK|2vyP_sIZ%<2?T$0gEeA(`7Lz4kNhOl5VAZU-rW?O@l+K zm(86aaJ`FxdcAz&ViKt+B*x}23HU_u7}C02)T+^^D~LjL`9hz@?G~q+Uaua&+axF zo;GRVA>aL5x;a4#%?!&tZIaiv8n^HyD!#hgtWDx64l&)3HQ-upnUxYXj1B7A+;=t$ zY~7|i04eTPk6UQ1#0z!rB3v#;GIep%cXLjGiX98DZ{hjPv9YLiH%}ZqXgfrcxqRws zzA%eEPR7pl2{Els&GwXzKz68>aj5od^(Hd_QgpYuh23{63UuJv6?iCscbJxSKy>Mw z<(+M5g)(eHK%jNF{JFkllAQX5r}iq#3_Y=rSB&pHW#NVxk6G<_cHz|lFC$dWDo3t8 zn*02X$pM>aZ`@Iv|R;&62QDO0*o+} z1`$H;kxz+T`$n3JR7pWV#MoJeFcxMJ_h~v-CswMbi7o*wATW!fE_jD58h%~VeIUa2 zLSt^BFm!Se=3)>SloF_9WBMTp1tCcoW>2Rb2rx`UG!YrPzfs8QKBL!_R(!S!>&t5R zq60bdrGkk}e0C2*a!G&6FbD>P41}=GiSbk|45V)s0fF^_BSMVZ6x(&2_!1EI-yu;z z{~hvt)dCpO?c%SHe2*cE^%nmMnFWPHCLGp>q|le6GEqzl=3-52gNd_F`w)#H5~-TT zEN(c1F@j)k1Vt`xctakDwAz{CN+%9tXbBaP=XoZj%mY+gQEGo>`BfA`I1ejhfhM#b z?7%Jr2E_SAq=sL7=OzTE;rWvGd#@5t`SDaZI_v!hn&7qmv*B9C$u|2EM6;lKqD zQ3x2+)qGgT8Jtaejmr#mJTesae3!L(I+oj7I~Y_~Q@Fcen2Yp*w0_1v9x(Wr&VqWa zBmjb8KoF+;P*fEMd9uV4Bde=C&kke0P>X{=Sm*ObJLAv`Z<&XIkhBsXbcjKaZ=v+V zkb` z<3^$Q562?#8UECnFGm6O%g4nrKC>c_)aPmYGhjad=K%u{lKmdNb$WQb1=fAjHeP{Q z0gt}R*N&4bGJkpq=!Rqj6oR}bAH%qJ`aSF3S8`RKx3&~Wr+%ZjFsKRqL^#BW%*1f? z3Ejk(N8t&g0g$6$P&$F1gZ_hlx&5fb@AOD7qr-!#Gwaj31EF^*>t2nD&U_qQO&mRK z9Nk^g6QRL!FlBc;*9QK4#6Jzv$?-AKSk#k7qb^>f7HP*?$-)=#lhqj8Q`?_N-bVF~ z8b+fq`{h^ZbWX)Yql>f0tB;0q^&j_Lk7jCL`zyKUD~X{S`@4#qi=le;$Frt1YE6}l zD>WOa4>u=uf_#xbdQd;~ePlU$#+vG;IC^Js#J0NJ*$`@C;}GNf7aT?wMmH{|H-bjD zJ4T5+j0hws;q@PBynXIGQT2U)4929b-Vzie+Upk?$T^hL2#UW2K}YVdI&ctb4QIx!cxmD_dA!Kl66^Oigc1 zDY(B?d%Y=VxmN+k15!W1)oyp`2d(5j*3ln5SbX~8{x;(N=Fgw2D0?Q?mW-dYSW;^W zFrU<9qp9DeUrW4wHe(z(4_(;~TVWQW#4|9}(K634u#C!gSRjFs)I6q<+tza{fvU|N zJpWtR%68AnPv8|yr+wWZwfm5{0%W>C^f=V@UL}WAyNy%pi>GKV63%d`Zm*=0*^|JV z&S)^6HxyE9syK0;+6+@mtnW*%-}{DA`FrOZwI4d^tinzW}x~LF8l;b2l z6)6tqYtQ$0V&toA7TNutFyC#aFC+nK;QqEYX$kim5ZK>l8hA_3O;fI5FrKCM4GA-l?T}CH zVjL4RrRo2o=M3o(UcdjJdK`Rl<;YtQ&(A4zP9E;eZVIKhQXlqq58!#91{)A*6*aBD zs(k;lBQg+3@j%jk<0U4jEk^@701r~3U9AjWjT&}L;{CF}>`m*J6xeyFnME!whn?!m;VavDlg+Ht4gKqi!B7G5FucX&Xp@rqWeR*=RKMxeg zvPpl}<*k`VBU!%bTMh^YO`fmn_Pv;;-e5AW4gjQkZM`3a((`TSi zY>0CRG%T_l`}B%eN-5N#mOv2&e_{Mf#F?-d#~%?uR{w7i(L>^ma^3oW3t-KIokO7S zA~IhuWIIEdz6hrML+cBZFQK0rH3dO8IpW+DffF#t_|zv{LYe^zDH(3wlV)m z;Xl(NYT&)buF%Qt%cZx;2JfEM(Z&72`|M}JM%#?^xkf&fo@sO4y>aX3Ja4_1R<(~p z=l$E;+S3%geqlagpj*us`Uw~|f+C~Ip&w^En+U3kPY*|!?0Mw6=g5qP$F zUBU1Qg(Yt+ps|y zwb%FCSEVm1|I(qFcMR0=?(%OP@8cT{Y-Ilr5^e?BS-!&$&q^sO`rrPt0zK`I6^|NvC^@z&a#!ee6cW({w6#k$7`q4kpw$d~H4H-=2>*^DV` zUfHN7JG*v>&ff2vKQRoZ*4{X`>j7NSN9qZdd^vam4xrZ70HZgL9o!n(KwD+gz9AG7 zJG|2`Dj)c4Gd5DuL>g~6E+JBDV{zwzpP^Rzd1IqRJPEc%c==?b`)jAlLIfVYhRoP-Rzp4?u z9?BR|CYso(#zY#DRNFZkOadqdRfldlELpTZK;z5jDpY*7KZ=?_Dv1m2TeJ#Q#^6-D zU1v#`S$cZwOCB{#hcmE3K#dWhgLKerc<6?BlQ8#4mW8WG%K_9k1)hR)I@R`i$MW{M`Ia2m@H0X*{I2~7+0hs@eSdH$QmjvP1r|rP9uX4 z;UU2^Q!G0!a=FpiWQqb!6I0Pt&pO9ZX;x3sE2Loe8hWbrU;-g`cO($MsmttFv-=dk~rd74$Q zi#YFVtA6Z#&GR*o2b!haFs6&OJXQ5zdVC1$uo_e2Hss=MIjCE(;uG>yvy{u{;eiUK zYOZe#_Z!}$8p%nnu-b(k;?bl}7YS?b65g-oA=OTW34%z!4l)qteX|m6HgZ~L-@$3? zgljVLq`0KD(J`7>yl`O|3H~H|O==jttZHo3Vu-*DEKZElOXWVUuYg1(sTn4stTzW; z+89d6ltS4k#-9pL2Hj$x1nG%74N+WqpS)ak_$yAwKS8w2uYP;51f!j-d_-n{&c{Zz^y_nJNlC%CT&A~ zAe}jLAuSK~pG>XyLY+B41yWCKN+!8Tm6b+SReJcJs70{=88QiU4%e21E|ng63%11&rUbE=^Bs-I<#sUSFt9bf$0cj+~%sO+@2dZ-|L8182Q zIv8$I;q&^Rh`Ufb@wFI);)My^Q+CPr7vn;*bX$&zP99Goxj!qx8-tZEqJswSA`s0+ zw4{0B94GG5kR-utO_iPCqD07+s)f&0((NFBq{^7m&vxl;=@@|*SXnV<4MjuN0D#eu<={@igIAVK*3AhY(?8d9t8Lc3lf1BID6w(1%(1afZ&A|2V}Y=FnyWq%Z9swBT|rUdR%& zn!}cetVOcBswx$!Hn2;ze~X4Bw5q;KtoE&hh zJ75@aTkK_UN-GU%s&03E7RFVHo}WNw8~fyYJPz^6k2>(4xU|tO4+&WfKgPP-9HDgW zhk2@~-|l_PNgVmCqmR`6a|0O}QWy`(o?n<)D;ZKg{s?9HDyjm?B`TC7c8exEGjkMJ*4MfDLDfe@z8QC5v$-d!CzozMnnj1J2%4wK~};9(^z{K z_mN|SUw2i8aBrZ8h*5m=I0AkLg5HGEAZ^CIkrGyx3G5;7ScC~R$=4hKx6?9t*+vqI zt(XsEu|W0h%WAgxPWJBiI}nja;uZ7;dVw|KH!#Zk#_A35NAIaWP1`ragn)sB{thG% zp7Q+$+Eh;z_})s}^-|Iq}Db3G3%!^O+q+vpSxuoxg5bYUIA}q!h!IaV^UwH#foiY5BA$k#hr$ z?+6@?J>(J-pd2N*Hx#6!#so!oEqm@g00NY)gjNI zdmQg3Ad6Hn<-k%i#uXQ<)6YFC(=2e%H(0yH;JtnZvBW&NjWXdopFwjv0_P=FNy5FS zZCQIfWS=DoTM#)F6oCz4V+89A(aUv{yWV`9PRg~}*nKUbgq>w0G8`x<2F?!^cx%L6 zmPcphJmCOA$WniG(7`f;x;_?b97*lo-}fq8bt6K8y>VaN7&OZPqL9{Qt9@-Z2ndY}a3TI;%zPV))HzT; z`6GOn=U4r_k3=^DrCVncwtIa3Hx7k*1|Ve31e1&dV5r5;{aSj zvyYT76h_&7So*B4Y8E!2uy#1UT9NE>th2{O09<79@9<00^Tv4pNI~>nm{NvB`VUZc zzvW)6oCXBX^qX_W=I&4?)T<#-#e$0#>EbGXJcRK!j2Kqt^3LU{Qqw}1j)gO{)`ZJz1&-5cg zQQ3mG7PL?|p*$r#?Yf1af$P^E*-ex0CEHxzO_;8a7xbM}#2TI;(H@#0x*a6Y(#@ z5dcURXPG4*>-+{V1Oke^q2{GI3)CA9gG}w*CkuzEwMCjI>=3xPvnyg5%bC`=%G8hciU2UVl993IX&FU&N6qn7m-TX)@mTy-DDT zeY)jonSZ?=Iy>Dmv3yi}+r4UFNW`B_7)v+>HT2=AB@6_AQk8*w=y9%AJ)t1Gl%t^} zn--%#P1ish0Zaj?p_+pnG{hMFb&PX$_NKa@(z55UJ;SY#7`BnLapngEeFZlPjOU?_ z^3umdYO)o->wtvWFHxjT9iz^$uN7ua0Viq6jaZ1oA%w%CqAtC3XHjJ{qzRebfgDOUc=fgyQMQ=QQi3s~qXE!;a{MS1<5!4Aa{9L(vc_v_ieuKq-1w;>hbZO`IT z=cPrkGV10fwpZr00(VgY9FVL`JkCQ2yV6t7(WF6KK!`Jhm<2kcjWpgwy?M}_$LD3G z`5vz%3E*yG3%_3b@Ul>urlq2Socu)i}|RhA>O?4yz{4fSVbfaIL}0oHbF4NVy2;eqoHwZkHoqTp}u-fa?x zMTk=(yzzm0{E*OptT?2&C+FTN7qEeN-8S#^5l7CRWQ+cj`>b_ z{$cnGHK<`UC~SECz?(3@!`&>5X9+5gGC^bT?Q3V!EKCq5*pL*>~ zwB%a@8E%Oumw-s;u zoUhvMAq0m#G6kC6AeTMnr%HFr@hu`OJ5F>S5>zU!0b6Ah>cKP0Pk`URTO{q-ZEBSC zZa$nozhi&->%Bv(80BXLBP__#R^kMtU@<{3UYe-$qNx7;`Yb6T&x!Kq7G^?He2dBg zOPJ(M+(&!{WvN?pLBl~(ffB};)vjpVf*{rvC@73Uo>8 zcAVu_RyMny3j=@5>Q;IoA;5J?%GxjM{adGJ6=`v(++uqfJX1z*TwG)WRmRoEcWMN( zOXs())Ty~S5cZUO{#RmVLf~9$8zCRI$e<#Sp8N3GQfFTODu*sgU0X4m##gux3t^Je z#!>MU(7}gW^uc2RE8(NEDXzQ@UOp@Sy$EnxoFtRohYyb(!2KA^ElWLHm(2^6_#|L# zF$eXPqYb^aryy3k}ht+KCAm=HmcODH4EBD__^5GSjOoXBZlyfjDG??p!(by+mfynD|;n zBT{6B@orWL4Galn3TRrUyP|btJ$2HjNf<4SW3#aves9L~=l%U;h``;)wGUQv`Ir<9 z{3TlEH9NEq1P}&sKlJMTC4glRFf8rV8JbBl+FM+I3Ea>C;5~f$T#zNyZhwFUu06dG{&j! zhX17wH{g#tF_izU&K3cvjt8dO@9hlpKVYT@&o1$v+H17vwZ#v`z>0M4>u7^lod`C{7Hg zZS4zhkAs5{+aJI{q zpilwWC(8Mk zie@d=anUhCQVJzbAoftYVqAov<7;5y$|^QaCw;wB@R8AM*7Fo1KXp8oXI?PXvtt${ z^Fn8ITm}Zhk=T^Edvw$@6(#Y-0-?w;Bu=I>e-;l&Wc>LC8axa>WE8I_k{lF7_T@q} z6+!wTC2*qx8x{5!@rVuC!_DsndBKy6b45*gE)uf$EpY?Z<6ZicA1Pn_8J1RumIvYp z1*}(;7-K#aRO!ND>I zYI}RZh1VpUF5P0VMJHUGnL=@M@7@l|Wae6Qw;?7_0Gm@q9$06%b)WJLsJ6ZkH4E`( zAiCSslXT4{A!yk11F|IZa~;aB-9g)!cW}nf2Lkt<;Bf#rla6|yP?~Nd@S20%T+HbU zxVyIjV!26s)@{BI>ST(-ALVA%IVg3Z#s<_N^ngPW4rL%(gV!k~v%@x{iyeNIW1Km&c|R~S2mo1tkPku+{u66Fg6);U!u=-X;z;)+gQIXBM7l z=k&^1b77Gs=IeUfY$|`1|6Dohi{du9q3}z=uGnCF6N>B7dHpAd}PUCAwxXtHJX;pDg9~+x19Wdbsc5-WIS<;d=+p z{Cs>^z!<3<7%9IiXIlISOf9iSv|K$x3qwmKiH1ecAZ32TwTfKvto5nP={w;;Z2CR- zTCQw^{pR{gIFK6H5Fx2$(FLrG%oX&eH(o@p%EnmE8rX87ly9PF9HYkm~yo*Is z;-8_LCL=U3{4mj285eL#m+M9P6_C$JqG8cPWX9!e4Kl|ldc(ZqxKViogsWNvbNHyn zXVmsVLfM;u0w=OEeJjkG*BTn4`RJQg8lD*00rrVTQ2b?&D-kJF#_5T<*p`g~#uMmc zWiaKg(G!Om7&G7W+jnBqq(b%)$c>9_yELNN04TLevNmaJ`JGX^T%NJ!Hp622D*Izb)Qdivi@Xo` ziJ!YCLriF+pHQR%(6`f34>wwvbr#HN+B8zyvn#3Ky}hd3He5erM!^nNP#F(ITdWqviX9D3e;ra1s#{l~D$|TGjuZZnkyz3*_Zgxr1iFVrl(n|6tzJKwbNh z5(L#;wt}iwQ|oSYguAz{xF>yoK6g-oJ2CSE^=q)worfP6Fm9gRh_yB4dfWnSgXbj? znx3%ixh5WNtoUgX8csU5<2;NW`FDLMvZ*cW7%a2xWoD5q$9my@7{9x)el)lB8cg-(3fjvfgul+*QlZI&S54rvc<{$< zTHU4ywdn7t89E;Hr`!pqsE$_E?Kwb#F;Cb3Jp2~^-R(8Chw8l4Q=vw(;@i#t=!=;o zLe@3{Y{}b62Lgim?^Dv%(#+0``L84EU!BZ!6dcyMaC&}eqI4WAKCin)bVyvRUM`m) zqP2>}Hi_%n(kPRhQuH8RI`#P(ZPpFKU9=3=RSUyk%)e#W6Sg3o7~s&ZMbIca9}O$~ zpoAVJdlK;OHBh-;Iu9iReF<}te|yw9RPjJ0*)BnP9KCGz3Va`4#@hjVMxt1hx@@N1 zvstPA+y++*gY5w$-8NI(*K!Dlsz;mL#u!T4D-M1z*^X*PgmD`mEr%2cov=(^Y~8-W z;DPdjb)6sjV|jB9W*r6rquA{d48-M~D4w`y$C`nl8FsySFgsD2J_A{k$1C|s_4ib1 zmmJP&Xp!Cc8=)tQaOeOSgfl7i4-MYeMp;%V8S;1xafSl<*@r=)Y$g#DS*@khRd<{_ z_2-9&$xJJ&iVP0OYyH}lOZE9q>fPCvVhQ>u4B!^n^D;7VV8a}c39ggT&YRMS$t5X@ zja#R#c;{3ZIw2)FiVJky=lJUA3ZKLlXzCO!3fJv2Eb(wN%mo!bZao|Ai>#(<)JoF z*1*FOv3XO7w88ZcB`ju5-yJ7u`K=>B7ubCx5Y{%1S?>Avw7;G@(_bmYtpDlxaA|+E zmgr3nq8Ikf3GMyph56m*nfCYj(^rQa?`Dv(#J+9c=ZjW@UOhh)nW6LA+_#&(wQ~o- zkGoK6i8e&2P?-%7udqP=6OaofMLL5fO1uWjd~obN3fDnmmlH8arhtc_%Kj^PQhy0Q zq> zDto#h&qU!PJf@AJb25cwJF-H!nD|%-B^OuZe6m;YD9#UCqfBLh6qrC?9=KCIDYQGy z)Jf)QVoVO!1e2I-3+595ML4~GTG1CrFl{Q$i3~c+pa6kQejahP;F*}+KQHd|5 zQ_BRS!3ZU+WEl4ZLms5RnKG^EHWQkiL`?1C%S*-5`n-bU*5l?WOcSr%KA%7idm3#k z1PJbVv8j@*mK10SRvg!Pro;rl?}c^5#%nc*ZVsBP5JXv<55T!#DtCv-kHZuUeW@Td z_sl%|x+W;k#Fg(|=jNQ-jIQfCzLDsc;iSBEJqJI%jFHpHwZb_| zfrCiq0)P5m>mNx`THyrjI7ncU!_-=BolFQt@|g6^gK0JmLxm)HdRw`S-{=g=?mJH2 zFj7*GusQF4N%0Y5=q`pYFHIZ;=NH_4Q{78R>T0K3TwJCumb|>RDpwB2QIw9A-B0s8 z_s|XGPtJhIUZKq5%0MkTv5RrRkhSgxWDHGOC?g=TrQp zrLc^pJazV0Q$ckX&5wZqhqBFxWo|6jy}-bh=B+X=2E)NTyXXT#DW3cQFK#XcrQp@JIw>}fSvBbMP*ITzW1C?-ddV$JH;5$_TxS!MmCfo_ zOFD}>$-mSiPsUMpEjdJ6SR_4+>6+a*Oz1jY$xznavcXv>&|AQZ-OiNx_|hi9wcXzP zO#RDe*wmS?kHAuVema!7?21Z*VSk4M(D!bMyRJor9V&YD=iSaZPDoZZe7+?7!BpCi zwSIMP$6ZM{CdEl@0;8)Bz=5dXX=0oRk~8as*R%bV-nEdXmZSmFkVXp*BKmeMB=%B1 z!kbFHH)AVzpe}#0!)mC-Wwgr{)G|D^^jK8bS+%mf+xF4hjkdgfSgxd+?~@R^<2k2J=Y*mw#Av6uH9KUf>MW~`9q%%3=&mp>yJo6MjJ9j~VL6>S(l!ONAvIJwd5kFy1p-mFgfVkS zi$S|$YSSE8Zi?p70;S1ykT)Gt>b2Fy@bzo%xPI||zF{5z$8I_UtaB{STkYrH(ErTE za{TyKN`YT^P{8ce^*b3|2asTQ#dSs{Y4o1wu2WXX$adCw6x|9SE|GGO8zC#H< z4;F#WszI?-wUE+wYjcMrRRqVT8z0Bm*JH(6gwO~Avv4d&JgE+YeH(McX$nBE7!HWE z)lBiCYUaR`Vm`8>>|OKKAjDBZeP*hG;xW3cX3W{746bd1%QSTkT3Y)bx&$WPT7{H`5YUjNTz9 zz135o3XaRRMfmZ{#m*PGvQ304cCM_XBH8c6@!pB+%wpBoKAEJ&sfk{aA0@ zZ4nL;>{h)~MpqzVhmGGDGhI_Bnh8(nLs~sYME?=)s`L!XyM}_ge0isa(A&xz*gprf zxl>+Y7bw^mC>-Y>!v9GAs|ovGqnaI8Zy(5lA@Q|m;C64EH4nlPBTO2E)@DieGoia5 zr7iiocyz_PmjWNB$>`S_*Q+Id0-?hod&8<(I_fzp3K+FY{``=vu3ksyPU+-Wm@3K4%9Rpw82H?6g4zsG~qyvdql7E(Bd3)fYkZt1od!;#$7 z^8Wj;)DxD<_;UU zPubJDPO3eyn<}5yq$eQvni-z|Lbzk98NS0Rt2)dcYzwVgnOFGA#s#;3JMWgIv}vN@ z)Kwcs9ag>*0S{N)foT(rQoDX2=|rWp2T7$x#G$D)cQLZ8QK#JRq`H*PO0M2#DJ7FW zb?$s88ksL1CKoY9%1-1!Ys^MJgB4a5z$?ei!+W-zqyWiHDRUX3$knOnxBdM--Or1Y z0G657H$9MMoW9lq;yYxFWdxj)46MTn13mAz-;dKpyqI4%0qa7L+f-6q8!?yR-5Poo z=&wkcyK|?@^IwJq91IJPP@<(2ZAKvVP}Jj9x=M4U&-I(B+oX&A>R&sGyScwAPSziq z(lQLoVXlj?wG>>ir|GoB%reIRhW|l&>nyF5UWiRallgstLO~ig%d;iE$|6kOUk>`gUm5Fl^2E)HmPc( z^7A12VUZIazHLEs;}XhQaS5$WI}+XVcLxuV^qiaZwc9oKf@9`kD5`pk=;53f0FSHQ zhgjt0OSAF0<1gb*s`}&T;TR%+e1vWpDP!uHTj7f2-W|2nnjRVVmHPo$o9@}&tL|%g zeevk-Yte1!{?D49M-GC$B$%P>voZO8bffq5hof+O0Ym6BJyXzW${S8Wp_rk2yZh<; z{(;m)9Bv%OlUT078D0O3b>p1}4`4eHw|X`QV=vFG$5e{|X1l>oU#a#tjxMJ%6${q9 ze?&X|RGc-z`m{aGcia4#`vs;4iprq-sM-nYNSWjOL^#9y(>8ABO|!biD@+ZNd`(F7 z=WO1@zHd`6tC$Avn%>)l-}sIzV!nLzz$-{1;N$NiL4zvk>-}>7eB;`DyDP-^nSnhq z``zAi-?qKBF3-fmum%@vu}^^gq6X^mYV;BHXr=FJs!jxvfn~4@Z?zA>#l|on7s}dD zjD3nXc#Zlaf9nJOpEFj$*s?`8BnU_l2?z)_aK-{Q=>BV(0At7n^HuJy~~~ zE;GDd$MyxI1qOq&w`g9lEC}Fc?jrY1J(Kp$HI6)$vKN?Kt{^9^?E||YHei+r69tWn z#yTCg?K0c&3{e3bV<8-=x^URM1ekt-*1H`}r!XuS$cn@pAGPGuz*!K43esbYyd6Kj z733Q9t5y=ItHrGTEJJAg`A_j2QsaQJ>7V$=6IvW&-1}b7UBq)2*Rzk%g=%;kDdQKV8f7Jwc6;syC@qcul$n zFB{a!bHluSwOW*3Ob>)4@T?=_6d?^mJ%$r>)MvKV&nmS81kl?@nb&QMAD0b3nY$jlOLG8!q-oie@aS|g5h}K6x!av~{6ft1s(x+xr*xm3{3)&Rpg`#VA zOc0JW#@C{YeW&BJs8NMU;#>N?u~l~wcG&j-4W5i?EpLoMQ7k#k^kO}+*lwB<#jKny zm*VwC`Iy$jNTBpcb|l#nfofqV?=nmILw1ShAONK*diX%fQ*XGbl|61Ia5xGc0!D;T za1$kH(h!U~DFBicfy}dOClo{n;vUZOhW)EB_z$QRUX17o^axM0FGAVTl?XM;nmqUm zLDE>PiQZkuad2#@7;&O3@lOYWY_S-H&2dw>Y3I^9xixNINWRmB8LCz2BBVU5#b*t9 zUhcv5zVsy?*JKcOFjWACuMgf z{?AzdueFQ#wJDxZ=glCXfx|36So)Q0act5|%cO_cO>vzO%|d8Ta6esdC9KHR3?olo z;A zaijIO0&Wz+E|A1ugj}9DJ&7m6>e-PQ*%#yU?2ZmoknZQNTI!B>ukq_dgpAg|hw%=Be zyc?Nm#a#1!n}}QFz_`GHV;FQ|*mch_w3-4xfi^w}jhIH?k2Oe+Z#5r0N`g(f z+$7Hm{SMsU8+|>r$lN@dAo-KkZBkZe$@ea=<%@s;ud3megn2eE4>yI&J<8vn|7&;U z4o1KKynf0PkKp3dudcPr)jhem^|f4e&EsXu=hZ#C`Fvje!s64{rpMK#-d??a@6T(u zt?yNC&wjqH-|pV$llN+W+y8xY`f&VL8Oy3Q-z)CF%s*H2`|kI@>Gc-%4hBXc zf1WE=es#oo-%FW13+42dp9R1JmrFMAFKc3x z%2?2dmH2Pm@$Ha(fHxzP2s3DSoP%MzLwM}kprsS10$Zr3fMaf;ZU+oB0L2&>qMd<* z!KryA@gbE3sl~CN))~41m1^D&KY&W_0mqTKVY-2I1LHBE5s)@MxM`2Bv7?PUI1{Mk zB2c3sipJ+a5qRB`oL>YUiALA{<@Rmy*Ffz#EDQ_+DBA0RB5>`A1qIk#?QnYYo&`W_ zTsY7zp8*UA@XP_wh|*l3-?3@WYmgQ)0$P3=Xq60#<>|nwMzD5>&yfwm8Vc-J*3T@~ z2R4O6SsD0IOu7slln0wsTvC~n3ZAY+4+HcAC1Ls-7>lh?G@~6aiEa-1JSoDQC;2dQ zkS0;kwWCkGAhcg9hH6KfhCw$0eIf#30!t~>1e9qBbp7a~#t8k>%c1(w$B@yDK<|Ge zjF?b|wKtA#0D4CkVL(PbvH{q-z3674_Z<;tB{m?Nh1!!uHwC?`g)pU}5gIBe9WQkK i=z;V(fL_!=2M7!EuZI$MRIGAQqf=XODE-L&ahy8=)NAOq; zMCpYSmJ-?HnK$pvcqa3!{6wBcF!}%X`VS-qsP7)&{2avG=xVzpgGZa^O;sPm#RX7X z@pU+GE1x%nVy;%yfwy=B(NXQwALBRAN zAlI!0vDn^OPk?Di5Hr;g-=O$G>nF0!QVmd;PA~71H;95dy){dhmxcRyzmdp!Jr5^d z{GqB*Gre=dg44i%i3~B#LcC?-hFH9haFuASJ9e1#2>aUcx##ek+LgV>X~G;kYe+r9 ztVEGZbhu@WsY4F~EGs<98`D|bxay##oju0#0k2E$$Ubn7&&BTXzf0rUNF$$n4!y?o z3mO-+)NS|K{)x DfClz) literal 0 HcmV?d00001 diff --git a/tests/extraction/test_client.py b/tests/extraction/test_client.py new file mode 100644 index 00000000..5ecde4fb --- /dev/null +++ b/tests/extraction/test_client.py @@ -0,0 +1,227 @@ +import asyncio +import json +from pathlib import Path + +import pytest + +from app.services.extraction_client import ( + ExtractionClient, + ExtractionError, + ExtractionSettings, + INPUT_BYTES, + TEXT_BYTES, +) + + +@pytest.mark.parametrize("format", ["pdf", "docx"]) +async def test_raw_upload_and_valid_segments(processor, input_file, format): + segments = await processor.client().extract(input_file, format) + assert len(segments) == (2 if format == "pdf" else 1) + assert [segment["index"] for segment in segments] == list( + range(1, len(segments) + 1) + ) + assert len(processor.requests) == 2 + method, path, headers, body = processor.requests[1] + assert (method, path) == ("POST", "/v1/extract-text") + assert body == Path(input_file).read_bytes() + assert headers == { + "Host": "extraction", + "Connection": "close", + "Content-Length": str(len(body)), + "Content-Type": "application/octet-stream", + "X-Extraction-Version": "1", + "X-Extraction-Format": format, + } + + +@pytest.mark.parametrize("size", [0, INPUT_BYTES + 1]) +async def test_input_bounds_before_any_connection(processor, input_file, size): + with open(input_file, "wb") as file: + file.truncate(size) + with pytest.raises(ExtractionError, match="INPUT_LIMIT"): + await processor.client().extract(input_file, "pdf") + assert not processor.requests + + +@pytest.mark.parametrize( + "change", + [ + lambda r: {**r, "version": True}, + lambda r: {**r, "version": 2}, + lambda r: {**r, "operation": "execute"}, + lambda r: {**r, "format": "docx"}, + lambda r: {**r, "inputSha256": "0" * 64}, + lambda r: {**r, "textBytes": True}, + lambda r: {**r, "textBytes": r["textBytes"] + 1}, + lambda r: {**r, "textBytes": TEXT_BYTES + 1}, + lambda r: {**r, "metadata": {"user_id": "attacker"}}, + lambda r: {**r, "segments": []}, + lambda r: {**r, "segments": r["segments"] * 65}, + lambda r: {**r, "segments": list(reversed(r["segments"]))}, + lambda r: {**r, "segments": [{**r["segments"][0], "index": True}]}, + lambda r: {**r, "segments": [{**r["segments"][0], "index": 0}]}, + lambda r: {**r, "segments": [{**r["segments"][0], "kind": "document"}]}, + lambda r: {**r, "segments": [{**r["segments"][0], "text": "\ud800"}]}, + lambda r: {**r, "segments": [{**r["segments"][0], "text": None}]}, + lambda r: {**r, "segments": [{**r["segments"][0], "file_id": "forged"}]}, + lambda r: { + **r, + "textBytes": 1, + "segments": [{"kind": "page", "index": 1, "text": " "}], + }, + ], +) +async def test_strict_result_rejection(processor, input_file, change): + processor.mutate = change + with pytest.raises(ExtractionError, match="INVALID_RESPONSE"): + await processor.client().extract(input_file, "pdf") + + +@pytest.mark.parametrize( + "change", + [ + lambda c: c.update(version=True), + lambda c: c.update(operation="other"), + lambda c: c.update(formats=["docx"]), + lambda c: c.update(formats=["pdf", "pdf"]), + lambda c: c.update(structure={"pdf": "pages"}), + lambda c: c["limitations"].update(pdf="\ud800"), + lambda c: c["limits"].update(inputBytes=True), + lambda c: c["limits"].update(resultBytes=4 * 1024 * 1024), + lambda c: c["limits"].update(concurrency=0), + lambda c: c.update(extra="untrusted"), + ], +) +async def test_incompatible_capabilities_never_upload(processor, input_file, change): + change(processor.capabilities) + with pytest.raises(ExtractionError, match="INCOMPATIBLE_SERVICE"): + await processor.client().extract(input_file, "pdf") + assert len(processor.requests) == 1 + + +def response(body=b"{}", fields=None): + fields = fields or f"Content-Length: {len(body)}\r\n".encode() + return ( + b"HTTP/1.1 200 OK\r\nContent-Type: application/json; charset=utf-8\r\n" + b"Connection: close\r\n" + fields + b"\r\n" + body + ) + + +@pytest.mark.parametrize( + "raw", + [ + response(b"\xff"), + response(b'{"version":1,"version":1}'), + response(b'{"textBytes":NaN}'), + response(b"[]"), + response(b"x" * 20, b"Content-Length: 21\r\n"), + response(b"{}trailing", b"Content-Length: 2\r\n"), + response(b"{}", b"Content-Length: 3145729\r\n"), + response(b"{}", b"Content-Length: 2\r\nContent-Length: 2\r\n"), + response(b"{}", b"Content-Length: 2\r\nTransfer-Encoding: chunked\r\n"), + response(b"{}", b"Content-Length: 2\r\nContent-Encoding: gzip\r\n"), + response(b"{}", b"Content-Length: 2\r\nX-Pad: " + b"x" * 4096 + b"\r\n"), + b"HTTP/1.1 302 Redirect\r\nLocation: http://example.com\r\n\r\n", + ], +) +async def test_bounded_strict_http_and_json(processor, input_file, raw): + processor.raw = raw + with pytest.raises(ExtractionError, match="INVALID_RESPONSE"): + await processor.client().extract(input_file, "pdf") + + +@pytest.mark.parametrize( + "status,code", + [ + (400, "INVALID_REQUEST"), + (413, "INPUT_LIMIT"), + (413, "STRUCTURE_LIMIT"), + (422, "UNSUPPORTED_CONTENT"), + (429, "BUSY"), + (503, "UNAVAILABLE"), + (504, "DEADLINE"), + ], +) +async def test_safe_error_codes(processor, input_file, status, code): + processor.error = (status, code) + with pytest.raises(ExtractionError) as caught: + await processor.client().extract(input_file, "pdf") + assert (caught.value.code, caught.value.status_code) == (code, status) + + +async def test_deadline_closes_transport(processor, input_file): + processor.hold = True + client = processor.client(deadline=0.05) + with pytest.raises(ExtractionError, match="DEADLINE"): + await client.extract(input_file, "pdf") + await asyncio.wait_for(processor.closed.wait(), 1) + assert client._admitted == 0 + + +async def test_cancellation_closes_transport_and_releases_admission( + processor, input_file +): + processor.hold = True + client = processor.client() + task = asyncio.create_task(client.extract(input_file, "pdf")) + await asyncio.wait_for(processor.entered.wait(), 1) + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + await asyncio.wait_for(processor.closed.wait(), 1) + assert client._admitted == 0 + + +async def test_client_admission_is_fail_fast(processor, input_file): + processor.hold = True + client = processor.client() + tasks = [asyncio.create_task(client.extract(input_file, "pdf")) for _ in range(2)] + while len([r for r in processor.requests if r[0] == "POST"]) < 2: + await asyncio.sleep(0) + try: + with pytest.raises(ExtractionError, match="BUSY"): + await client.extract(input_file, "pdf") + finally: + for task in tasks: + task.cancel() + await asyncio.gather(*tasks, return_exceptions=True) + assert client._admitted == 0 + + +async def test_capabilities_rechecked_after_service_change(processor, input_file): + client = processor.client() + await client.extract(input_file, "pdf") + processor.capabilities["version"] = 2 + with pytest.raises(ExtractionError, match="INCOMPATIBLE_SERVICE"): + await client.extract(input_file, "pdf") + assert [r[0] for r in processor.requests] == ["GET", "POST", "GET"] + + +async def test_outage_is_safe(input_file, tmp_path): + client = ExtractionClient( + ExtractionSettings(enabled=True, socket=str(tmp_path / "missing")) + ) + with pytest.raises(ExtractionError, match="UNAVAILABLE"): + await client.extract(input_file, "pdf") + + +def test_disabled_settings_ignore_unrelated_stale_values(monkeypatch): + monkeypatch.delenv("RAG_EXTRACTION_ENABLED", raising=False) + monkeypatch.setenv("RAG_EXTRACTION_DEADLINE_SECONDS", "stale") + assert not ExtractionSettings.from_env().enabled + + +@pytest.mark.parametrize( + "kwargs", + [ + {"socket": "relative"}, + {"deadline": 0}, + {"deadline": 11}, + {"deadline": float("nan")}, + {"formats": frozenset()}, + {"formats": frozenset({"doc"})}, + ], +) +def test_opt_in_config_is_validated(kwargs): + with pytest.raises(ValueError): + ExtractionSettings(enabled=True, **kwargs) diff --git a/tests/extraction/test_processor.py b/tests/extraction/test_processor.py new file mode 100644 index 00000000..212c3807 --- /dev/null +++ b/tests/extraction/test_processor.py @@ -0,0 +1,144 @@ +"""Opt-in real client qualification against the shipped hardened processor image.""" + +import os +import subprocess +import time +import uuid +from pathlib import Path + +import aiofiles +import pytest + +pytestmark = pytest.mark.integration + + +def docker(*args, check=True): + result = subprocess.run( + ["docker", *args], capture_output=True, text=True, timeout=40 + ) + if check: + assert result.returncode == 0, result.stdout + result.stderr + return result + + +def test_python_client_to_hardened_processor(): + image = os.getenv("RAG_EXTRACTION_TEST_IMAGE") + if not image: + pytest.skip( + "RAG_EXTRACTION_TEST_IMAGE must name a built Code Interpreter extraction runtime" + ) + run_id = uuid.uuid4().hex + name = f"rag-extraction-{run_id}" + volume = f"rag-extraction-{run_id}" + caller = f"rag-extraction-caller-{run_id}" + root = Path(__file__).resolve().parents[2] + client = root / "app/services/extraction_client.py" + fixtures = Path(__file__).parent / "fixtures" + library = Path(aiofiles.__file__).parent + docker("volume", "create", volume) + try: + docker( + "run", + "-d", + "--name", + name, + "--init", + "--network", + "none", + "--read-only", + "--user", + "10001:10001", + "--cap-drop", + "ALL", + "--security-opt", + "no-new-privileges:true", + "--memory", + "768m", + "--cpus", + "2", + "--pids-limit", + "32", + "--tmpfs", + "/jobs:rw,noexec,nosuid,nodev,size=96m,nr_inodes=1024,uid=10001,gid=10001,mode=0700", + "--mount", + f"type=volume,src={volume},dst=/socket", + image, + ) + for _ in range(30): + status = docker( + "inspect", "--format", "{{.State.Health.Status}}", name + ).stdout.strip() + if status == "healthy": + break + if status == "unhealthy": + pytest.fail( + "Hardened extraction processor failed its isolation/health probe" + ) + time.sleep(1) + else: + pytest.fail("Hardened extraction processor did not become healthy") + script = """ +import asyncio +import sys +sys.path.insert(0, "/") +from client import ExtractionClient, ExtractionSettings +async def check(): + client = ExtractionClient(ExtractionSettings(enabled=True)) + pdf = await client.extract("/fixtures/pages.pdf", "pdf") + assert pdf == [ + {"kind": "page", "index": 1, "text": "First page text"}, + {"kind": "page", "index": 2, "text": "Second page text"}, + ], pdf + docx = await client.extract("/fixtures/body.docx", "docx") + assert len(docx) == 1 and docx[0]["kind"] == "document" and docx[0]["index"] == 1 + text = docx[0]["text"] + assert text.index("First paragraph") < text.index("Left cell") < text.index("Right cell") < text.index("Last paragraph") +asyncio.run(check()) +print("Python client to hardened PDF/DOCX processor passed") +""" + result = docker( + "run", + "--rm", + "--name", + caller, + "--init", + "--network", + "none", + "--read-only", + "--user", + "10001:10001", + "--group-add", + "10001", + "--cap-drop", + "ALL", + "--security-opt", + "no-new-privileges:true", + "--mount", + f"type=volume,src={volume},dst=/socket,readonly", + "--mount", + f"type=bind,src={client},dst=/client.py,readonly", + "--mount", + f"type=bind,src={fixtures},dst=/fixtures,readonly", + "--mount", + f"type=bind,src={library},dst=/usr/local/lib/python3.13/site-packages/aiofiles,readonly", + image, + "python", + "-B", + "-c", + script, + ) + assert "processor passed" in result.stdout + assert ( + docker( + "exec", + name, + "node", + "-e", + 'if(require("node:fs").readdirSync("/jobs").length)process.exit(1)', + ).returncode + == 0 + ) + finally: + docker("rm", "-f", caller, check=False) + docker("rm", "-f", name, check=False) + docker("volume", "rm", volume, check=False) diff --git a/tests/extraction/test_routes.py b/tests/extraction/test_routes.py new file mode 100644 index 00000000..b2884491 --- /dev/null +++ b/tests/extraction/test_routes.py @@ -0,0 +1,363 @@ +import asyncio +import copy +from concurrent.futures import ThreadPoolExecutor +from datetime import datetime, timedelta, timezone +from pathlib import Path +from unittest.mock import Mock + +import httpx +import jwt +import pytest +from fastapi import FastAPI +from langchain_core.documents import Document + +from app.middleware import security_middleware +from app.routes import document_routes as routes +from app.services.extraction_client import ExtractionClient, ExtractionSettings +from app.services.vector_store.async_pg_vector import AsyncPgVector + +ROUTES = ["/text", "/embed", "/embed-upload", "/local/embed"] +MIMES = { + "pdf": "application/pdf", + "docx": "application/vnd.openxmlformats-officedocument.wordprocessingml.document", +} + + +class RecordingStore(AsyncPgVector): + def __init__(self): + self._bind = None + self.calls = [] + self.persisted = [] + self.rollbacks = [] + self.fail_on = None + + async def aadd_documents(self, documents, ids=None, executor=None): + self.calls.append(copy.deepcopy(documents)) + if len(self.calls) == self.fail_on: + raise RuntimeError("insertion failed") + self.persisted.extend(copy.deepcopy(documents)) + return [f"row-{len(self.calls)}-{index}" for index in range(len(documents))] + + async def delete_by_metadata(self, metadata, executor=None): + self.rollbacks.append(metadata) + self.persisted = [ + doc + for doc in self.persisted + if not all( + doc.metadata.get(key) == value for key, value in metadata.items() + ) + ] + + async def get_filtered_ids(self, ids, owners=None, executor=None): + return [ + file_id + for file_id in ids + if any( + doc.metadata["file_id"] == file_id and doc.metadata["user_id"] in owners + for doc in self.persisted + ) + ] + + async def get_documents_by_ids(self, ids, owners=None, executor=None): + return [ + doc + for doc in reversed(self.persisted) + if doc.metadata["file_id"] in ids and doc.metadata["user_id"] in owners + ] + + +@pytest.fixture +async def api(processor, monkeypatch, tmp_path): + app = FastAPI() + app.middleware("http")(security_middleware) + app.include_router(routes.router) + store = RecordingStore() + uploads = tmp_path / "uploads" + uploads.mkdir() + monkeypatch.setattr(routes, "RAG_UPLOAD_DIR", str(uploads)) + monkeypatch.setattr(routes, "extraction_client", processor.client()) + monkeypatch.setattr(routes, "PDF_EXTRACT_IMAGES", False) + monkeypatch.setattr(routes, "vector_store", store) + monkeypatch.setattr(routes, "PARALLEL_EXECUTION", 1) + monkeypatch.setenv("JWT_SECRET", "test-extraction-secret") + # The migrated path must never construct either legacy parser. + monkeypatch.setattr( + "app.utils.document_loader.PyPDFLoader", + Mock(side_effect=AssertionError("local PDF parser")), + ) + monkeypatch.setattr( + "app.utils.document_loader.Docx2txtLoader", + Mock(side_effect=AssertionError("local DOCX parser")), + ) + token = jwt.encode( + { + "id": "caller", + "exp": datetime.now(timezone.utc) + timedelta(minutes=5), + }, + "test-extraction-secret", + algorithm="HS256", + ) + with ThreadPoolExecutor(max_workers=2) as executor: + app.state.thread_pool = executor + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=app), + base_url="http://rag", + headers={"Authorization": f"Bearer {token}"}, + ) as client: + yield client, store, uploads + + +async def post( + client, + uploads, + route, + format="pdf", + content=b"opaque bytes", + mime=None, + filename=None, + entity=None, +): + filename = filename or f"report.{format}" + mime = mime or MIMES.get(format, "text/plain") + if route == "/local/embed": + local = uploads / filename + local.write_bytes(content) + return await client.post( + route, + params={"entity_id": entity} if entity else None, + json={ + "filepath": filename, + "filename": filename, + "file_content_type": mime, + "file_id": "caller-file", + }, + ) + data = {"file_id": "caller-file"} + if entity: + data["entity_id"] = entity + return await client.post( + route, + data=data, + files={ + "uploaded_file" if route == "/embed-upload" else "file": ( + filename, + content, + mime, + ), + }, + ) + + +@pytest.mark.parametrize("route", ROUTES) +@pytest.mark.parametrize("format", ["pdf", "docx"]) +async def test_migrated_routes_parse_once_and_keep_caller_metadata( + api, processor, route, format +): + client, store, uploads = api + response = await post(client, uploads, route, format) + assert response.status_code == 200, response.text + result = response.json() + assert result["file_id"] == "caller-file" + assert result["filename"] == f"report.{format}" + assert result["known_type"] is True + assert [r[0] for r in processor.requests] == ["GET", "POST"] + if route == "/text": + assert result["text"] == ( + "First page café\nSecond page" + if format == "pdf" + else "Body paragraph\nLeft\tRight" + ) + assert not store.calls + else: + assert result["status"] is True + docs = store.persisted + assert docs + assert all( + doc.metadata["user_id"] == "caller" + and doc.metadata["file_id"] == "caller-file" + for doc in docs + ) + assert all(str(uploads) in doc.metadata["source"] for doc in docs) + assert all("\x00" not in doc.page_content for doc in docs) + assert [doc.metadata["_rag_chunk_index"] for doc in docs] == list( + range(len(docs)) + ) + assert len({doc.metadata["_rag_ingestion_attempt_id"] for doc in docs}) == 1 + if format == "pdf": + assert [doc.metadata["page"] for doc in docs] == [0, 1] + context = await client.get("/documents/caller-file/context") + assert context.status_code == 200 + assert context.text.index("First page") < context.text.index("Second page") + assert "# PAGE 0" not in context.text + assert "# PAGE 1" in context.text # Existing zero-based context convention. + else: + assert all("page" not in doc.metadata for doc in docs) + remaining = list(uploads.rglob("*.*")) + assert len(remaining) == (1 if route == "/local/embed" else 0) + + +@pytest.mark.parametrize("route", ROUTES) +@pytest.mark.parametrize( + "failure,status", + [ + ("unsupported", 422), + ("outage", 503), + ("incompatible", 502), + ("identity", 502), + ], +) +async def test_selected_format_failure_never_falls_back_or_persists( + api, processor, route, failure, status +): + client, store, uploads = api + if failure == "unsupported": + processor.error = (422, "UNSUPPORTED_CONTENT") + elif failure == "outage": + routes.extraction_client = ExtractionClient( + ExtractionSettings(enabled=True, socket="/socket/missing.sock") + ) + elif failure == "incompatible": + processor.capabilities["version"] = 2 + else: + processor.mutate = lambda value: { + **value, + "user_id": "forged", + "file_id": "forged", + "_rag_chunk_index": 99, + } + response = await post(client, uploads, route) + assert response.status_code == status, response.text + assert not store.calls and not store.persisted and not store.rollbacks + assert len(list(uploads.rglob("*.pdf"))) == (1 if route == "/local/embed" else 0) + + +@pytest.mark.parametrize("route", ROUTES) +async def test_unauthorized_request_cannot_reach_processor(api, processor, route): + client, store, uploads = api + client.headers.pop("Authorization") + response = await post(client, uploads, route) + assert response.status_code == 401 + assert not processor.requests and not store.calls + + +@pytest.mark.parametrize("route", ROUTES) +async def test_cancellation_cleans_upload_and_never_stores(api, processor, route): + client, store, uploads = api + processor.hold = True + task = asyncio.create_task(post(client, uploads, route)) + await asyncio.wait_for(processor.entered.wait(), 2) + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + await asyncio.wait_for(processor.closed.wait(), 1) + assert not store.calls + assert len(list(uploads.rglob("*.pdf"))) == (1 if route == "/local/embed" else 0) + + +@pytest.mark.parametrize("route", ["/embed", "/embed-upload", "/local/embed"]) +async def test_existing_attempt_rollback_and_no_success_on_storage_error( + api, processor, monkeypatch, route +): + client, store, uploads = api + monkeypatch.setattr(routes, "EMBEDDING_BATCH_SIZE", 1) + store.fail_on = 2 + old = Document( + page_content="earlier attempt", + metadata={ + "file_id": "caller-file", + "user_id": "caller", + "_rag_ingestion_attempt_id": "old", + }, + ) + store.persisted.append(old) + response = await post(client, uploads, route) + assert response.status_code == 500, response.text + assert store.persisted == [old] + assert len(store.rollbacks) == 1 + assert store.rollbacks[0]["file_id"] == "caller-file" + assert store.rollbacks[0]["_rag_ingestion_attempt_id"] != "old" + assert len([r for r in processor.requests if r[0] == "POST"]) == 1 + + +@pytest.mark.parametrize( + "filename,mime", + [ + ("notes.md", "text/markdown"), + ("plain.txt", "text/plain"), + ("code.py", "text/x-python"), + ], +) +async def test_raw_text_parity_keeps_original_text_and_local_path( + api, processor, filename, mime +): + client, store, uploads = api + content = "# Heading\n\n**bold** and café\n- item\n".encode() + response = await post(client, uploads, "/text", "txt", content, mime, filename) + assert response.status_code == 200 + assert response.json()["text"] == content.decode().rstrip("\n") + assert response.json()["known_type"] is True + assert not processor.requests and not store.calls + + +async def test_mime_selection_returns_canonical_pdf_and_cleans_text(api, processor): + client, store, uploads = api + response = await post( + client, uploads, "/text", mime="application/pdf", filename="opaque.bin" + ) + assert response.status_code == 200 + assert "\x00" not in response.json()["text"] + assert processor.requests[-1][2]["X-Extraction-Format"] == "pdf" + + +async def test_selected_docx_cannot_be_hijacked_by_markdown_mime(api, processor): + client, store, uploads = api + response = await post(client, uploads, "/text", "docx", mime="text/markdown") + assert response.status_code == 200 + assert processor.requests[-1][2]["X-Extraction-Format"] == "docx" + + +async def test_requested_pdf_image_extraction_is_not_silently_downgraded( + api, processor, monkeypatch +): + client, store, uploads = api + monkeypatch.setattr(routes, "PDF_EXTRACT_IMAGES", True) + response = await post(client, uploads, "/text") + assert response.status_code == 422 + assert response.json()["detail"]["code"] == "UNSUPPORTED_CONTENT" + assert not processor.requests and not store.calls + + +async def test_format_policy_leaves_unselected_formats_on_existing_path( + api, processor, monkeypatch +): + client, store, uploads = api + routes.extraction_client = ExtractionClient( + ExtractionSettings( + enabled=True, + socket=processor.socket, + formats=frozenset({"docx"}), + ) + ) + local = Mock() + local.lazy_load.return_value = [Document(page_content="existing PDF path")] + get_loader = Mock(return_value=(local, True, "pdf")) + monkeypatch.setattr(routes, "get_loader", get_loader) + response = await post(client, uploads, "/text") + assert response.status_code == 200 + assert response.json()["text"] == "existing PDF path" + get_loader.assert_called_once() + assert not processor.requests + + +async def test_disabled_by_default_keeps_existing_loader(api, processor, monkeypatch): + client, store, uploads = api + routes.extraction_client = ExtractionClient(ExtractionSettings()) + local = Mock() + local.lazy_load.return_value = [Document(page_content="legacy extraction")] + get_loader = Mock(return_value=(local, True, "pdf")) + monkeypatch.setattr(routes, "get_loader", get_loader) + response = await post(client, uploads, "/text") + assert response.status_code == 200 + assert response.json()["text"] == "legacy extraction" + get_loader.assert_called_once() + assert not processor.requests From d155c680776384ab401aa4c23e644c5729b81da0 Mon Sep 17 00:00:00 2001 From: Lia Date: Fri, 23 Oct 2026 08:28:24 +0000 Subject: [PATCH 2/7] =?UTF-8?q?=F0=9F=94=8C=20fix:=20Cancel=20Remote=20Ext?= =?UTF-8?q?raction=20on=20HTTP=20Disconnect?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/routes/document_routes.py | 34 +++++- tests/extraction/conftest.py | 3 + tests/extraction/fixtures/.gitattributes | 2 + tests/extraction/test_routes.py | 136 +++++++++++++++++++++++ 4 files changed, 174 insertions(+), 1 deletion(-) create mode 100644 tests/extraction/fixtures/.gitattributes diff --git a/app/routes/document_routes.py b/app/routes/document_routes.py index 4aefd004..2da465d6 100644 --- a/app/routes/document_routes.py +++ b/app/routes/document_routes.py @@ -341,12 +341,40 @@ def _make_unique_temp_path(user_id: str, filename: str) -> Optional[str]: return str(Path(RAG_UPLOAD_DIR, user_id, unique_name).resolve()) +async def _extract_for_request(request: Optional[Request], file_path: str, format: str): + if request is None: + return await extraction_client.extract(file_path, format) + + async def wait_for_disconnect(): + # FastAPI has already consumed the multipart/JSON body before this route. + while (await request.receive())["type"] != "http.disconnect": + pass + + disconnected = asyncio.create_task(wait_for_disconnect()) + extraction = asyncio.create_task(extraction_client.extract(file_path, format)) + try: + await asyncio.wait( + (disconnected, extraction), return_when=asyncio.FIRST_COMPLETED + ) + if disconnected.done(): + disconnected.result() + raise ExtractionError("CANCELLED", status_code=499) + return extraction.result() + finally: + disconnected.cancel() + extraction.cancel() + # Drain both tasks before cleanup or persistence, including ASGI cancellation. + with CancelScope(shield=True): + await asyncio.gather(disconnected, extraction, return_exceptions=True) + + async def load_file_content( filename: str, content_type: str, file_path: str, executor, raw_text: bool = False, + request: Optional[Request] = None, ) -> tuple: """Load file content using appropriate loader. @@ -360,7 +388,7 @@ async def load_file_content( try: if format == "pdf" and PDF_EXTRACT_IMAGES: raise ExtractionError("UNSUPPORTED_CONTENT") - segments = await extraction_client.extract(file_path, format) + segments = await _extract_for_request(request, file_path, format) except ExtractionError as error: raise HTTPException( status_code=error.status_code, @@ -1164,6 +1192,7 @@ async def embed_local_file( document.file_content_type, file_path, request.app.state.thread_pool, + request=request, ) result = await store_data_in_vector_db( @@ -1266,6 +1295,7 @@ async def embed_file( file.content_type, validated_file_path, request.app.state.thread_pool, + request=request, ) logger.debug( @@ -1424,6 +1454,7 @@ async def embed_file_upload( uploaded_file.content_type, validated_temp_file_path, request.app.state.thread_pool, + request=request, ) result = await store_data_in_vector_db( @@ -1562,6 +1593,7 @@ async def extract_text_from_file( validated_temp_file_path, request.app.state.thread_pool, raw_text=True, + request=request, ) # Extract text content from loaded documents diff --git a/tests/extraction/conftest.py b/tests/extraction/conftest.py index 20e79ec9..4ffc95a4 100644 --- a/tests/extraction/conftest.py +++ b/tests/extraction/conftest.py @@ -42,6 +42,7 @@ def __init__(self, socket): self.raw = None self.error = None self.hold = False + self.before_response = None self.entered = asyncio.Event() self.closed = asyncio.Event() self.active = set() @@ -101,6 +102,8 @@ async def handle(self, reader, writer): writer.write(self.raw) await writer.drain() return + if self.before_response is not None: + self.before_response() wire = json.dumps(value).encode("utf-8") writer.write( f"HTTP/1.1 {status} Result\r\nContent-Type: application/json; charset=utf-8\r\n" diff --git a/tests/extraction/fixtures/.gitattributes b/tests/extraction/fixtures/.gitattributes new file mode 100644 index 00000000..d5a03a01 --- /dev/null +++ b/tests/extraction/fixtures/.gitattributes @@ -0,0 +1,2 @@ +*.pdf binary +*.docx binary diff --git a/tests/extraction/test_routes.py b/tests/extraction/test_routes.py index b2884491..6ceea6d4 100644 --- a/tests/extraction/test_routes.py +++ b/tests/extraction/test_routes.py @@ -11,6 +11,7 @@ from fastapi import FastAPI from langchain_core.documents import Document +from app.config import LogMiddleware from app.middleware import security_middleware from app.routes import document_routes as routes from app.services.extraction_client import ExtractionClient, ExtractionSettings @@ -69,6 +70,7 @@ async def get_documents_by_ids(self, ids, owners=None, executor=None): @pytest.fixture async def api(processor, monkeypatch, tmp_path): app = FastAPI() + app.add_middleware(LogMiddleware) app.middleware("http")(security_middleware) app.include_router(routes.router) store = RecordingStore() @@ -361,3 +363,137 @@ async def test_disabled_by_default_keeps_existing_loader(api, processor, monkeyp assert response.json()["text"] == "legacy extraction" get_loader.assert_called_once() assert not processor.requests + + +class ASGIConnection: + """Deliver the complete upload, then a server disconnect without cancelling app.""" + + def __init__(self, app, request, body): + self.incoming = asyncio.Queue() + self.received = [] + self.sent = [] + chunks = [body[:17], body[17:53], body[53:]] + for index, chunk in enumerate(chunks): + self.incoming.put_nowait( + { + "type": "http.request", + "body": chunk, + "more_body": index < len(chunks) - 1, + } + ) + self.scope = { + "type": "http", + "asgi": {"version": "3.0", "spec_version": "2.3"}, + "http_version": "1.1", + "method": request.method, + "scheme": "http", + "path": request.url.path, + "raw_path": request.url.raw_path, + "query_string": request.url.query, + "root_path": "", + "headers": [(key.lower(), value) for key, value in request.headers.raw], + "client": ("127.0.0.1", 12345), + "server": ("rag", 80), + } + self.task = asyncio.create_task(app(self.scope, self.receive, self.send)) + + async def receive(self): + message = await self.incoming.get() + self.received.append(message["type"]) + return message + + async def send(self, message): + self.sent.append(message) + + def disconnect(self): + self.incoming.put_nowait({"type": "http.disconnect"}) + + async def close(self): + if not self.task.done(): + self.task.cancel() + await asyncio.gather(self.task, return_exceptions=True) + + +def disconnect_request(client, uploads, route, format): + filename = f"report.{format}" + if route == "/local/embed": + (uploads / filename).write_bytes(b"opaque document bytes") + return client.build_request( + "POST", + route, + json={ + "filepath": filename, + "filename": filename, + "file_content_type": MIMES[format], + "file_id": "caller-file", + }, + ) + return client.build_request( + "POST", + route, + data={"file_id": "caller-file"}, + files={ + "uploaded_file" if route == "/embed-upload" else "file": ( + filename, + b"opaque document bytes", + MIMES[format], + ), + }, + ) + + +@pytest.mark.parametrize("route", ROUTES) +@pytest.mark.parametrize("format", ["pdf", "docx"]) +async def test_asgi_disconnect_stops_extraction_and_cleans_upload( + api, processor, route, format +): + client, store, uploads = api + processor.hold = True + request = disconnect_request(client, uploads, route, format) + connection = ASGIConnection(client._transport.app, request, await request.aread()) + try: + await asyncio.wait_for(processor.entered.wait(), 1) + assert connection.received == ["http.request"] * 3 + connection.disconnect() + await asyncio.wait_for(processor.closed.wait(), 0.5) + await asyncio.wait_for(asyncio.shield(connection.task), 1) + assert "http.disconnect" in connection.received + assert connection.task.cancelling() == 0 + assert not store.calls and not store.persisted and not store.rollbacks + assert not any( + message["type"] == "http.response.start" and 200 <= message["status"] < 300 + for message in connection.sent + ) + assert len(list(uploads.rglob(f"*.{format}"))) == ( + 1 if route == "/local/embed" else 0 + ) + assert routes.extraction_client._admitted == 0 + finally: + await connection.close() + + +@pytest.mark.parametrize("route", ROUTES) +@pytest.mark.parametrize("format", ["pdf", "docx"]) +async def test_asgi_disconnect_wins_over_processor_success( + api, processor, route, format +): + client, store, uploads = api + request = disconnect_request(client, uploads, route, format) + body = await request.aread() + processor.before_response = lambda: connection.disconnect() + connection = ASGIConnection(client._transport.app, request, body) + try: + await asyncio.wait_for(asyncio.shield(connection.task), 1) + assert connection.received == ["http.request"] * 3 + ["http.disconnect"] + assert connection.task.cancelling() == 0 + assert not store.calls and not store.persisted and not store.rollbacks + assert not any( + message["type"] == "http.response.start" and 200 <= message["status"] < 300 + for message in connection.sent + ) + assert len(list(uploads.rglob(f"*.{format}"))) == ( + 1 if route == "/local/embed" else 0 + ) + assert routes.extraction_client._admitted == 0 + finally: + await connection.close() From 0c19824b3f58c6ddd68afd445584be1c1482874c Mon Sep 17 00:00:00 2001 From: Lia Date: Fri, 23 Oct 2026 09:28:01 +0000 Subject: [PATCH 3/7] =?UTF-8?q?=F0=9F=94=8C=20fix:=20Preserve=20Ingestion?= =?UTF-8?q?=20Cancellation=20Through=20Storage?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/routes/document_routes.py | 167 +++++++++++++- tests/extraction/test_routes.py | 386 ++++++++++++++++++++++++++++++++ 2 files changed, 541 insertions(+), 12 deletions(-) diff --git a/app/routes/document_routes.py b/app/routes/document_routes.py index 2da465d6..5b93453a 100644 --- a/app/routes/document_routes.py +++ b/app/routes/document_routes.py @@ -10,6 +10,9 @@ from anyio import CancelScope import asyncio import time +import inspect +from contextvars import ContextVar +from threading import Lock from shutil import copyfileobj from typing import List, Iterable, Optional, Union, TYPE_CHECKING from concurrent.futures import ThreadPoolExecutor @@ -26,7 +29,7 @@ ) from langchain_core.documents import Document from langchain_text_splitters import RecursiveCharacterTextSplitter -from functools import lru_cache +from functools import lru_cache, wraps if TYPE_CHECKING: from app.services.vector_store.async_pg_vector import AsyncPgVector @@ -341,8 +344,115 @@ def _make_unique_temp_path(user_id: str, filename: str) -> Optional[str]: return str(Path(RAG_UPLOAD_DIR, user_id, unique_name).resolve()) +_active_extraction_request = ContextVar("rag_extraction_request", default=None) + + +class _ExtractionRequest: + def __init__(self): + self.disconnected = False + self.finish_storage = False + self.cleanup_started = False + self._mutation_lock = Lock() + + def mark_disconnected(self): + with self._mutation_lock: + self.disconnected = True + + def checkpoint(self): + if self.disconnected: + raise asyncio.CancelledError() + + def start_uncancellable_storage(self): + with self._mutation_lock: + self.checkpoint() + # These store APIs expose no safe attempt-scoped cancellation/undo boundary. + self.finish_storage = True + + +def _with_extraction_request(file_argument, content_type="content_type"): + """Keep selected requests alive until cancellable work or started storage drains.""" + + def decorate(handler): + signature = inspect.signature(handler) + + @wraps(handler) + async def wrapped(*args, **kwargs): + arguments = signature.bind(*args, **kwargs).arguments + file = arguments[file_argument] + settings = extraction_client.settings + format = extraction_format(file.filename, getattr(file, content_type)) + if not settings.enabled or format not in settings.formats: + return await handler(*args, **kwargs) + + request = arguments["request"] + lifecycle = _ExtractionRequest() + + async def wait_for_disconnect(): + # FastAPI consumed the complete multipart/JSON body before the handler. + while (await request.receive())["type"] != "http.disconnect": + pass + lifecycle.mark_disconnected() + + token = _active_extraction_request.set(lifecycle) + try: + operation = asyncio.create_task(handler(*args, **kwargs)) + finally: + _active_extraction_request.reset(token) + disconnected = asyncio.create_task(wait_for_disconnect()) + try: + await asyncio.wait( + (operation, disconnected), return_when=asyncio.FIRST_COMPLETED + ) + if disconnected.done(): + disconnected.result() + raise HTTPException(status_code=499, detail={"code": "CANCELLED"}) + return operation.result() + finally: + if not operation.done() and not ( + lifecycle.finish_storage or lifecycle.cleanup_started + ): + operation.cancel() + disconnected.cancel() + # Do not clean uploads or return while workers/rollback can still run. + with CancelScope(shield=True): + await asyncio.gather( + operation, disconnected, return_exceptions=True + ) + + return wrapped + + return decorate + + +async def _drain_request_worker(future): + """Caller cancellation must not abandon a submitted executor worker.""" + try: + return await asyncio.shield(future) + except asyncio.CancelledError: + with CancelScope(shield=True): + await asyncio.gather(future, return_exceptions=True) + raise + + +async def _run_sync_request_store(executor, store, documents, ids): + lifecycle = _active_extraction_request.get() + + def insert(): + if not lifecycle.finish_storage: + lifecycle.start_uncancellable_storage() + return store.add_documents(documents, ids=ids) + + if not lifecycle.finish_storage: + lifecycle.checkpoint() + future = asyncio.get_running_loop().run_in_executor(executor, insert) + return await _drain_request_worker(future) + + async def _extract_for_request(request: Optional[Request], file_path: str, format: str): - if request is None: + lifecycle = _active_extraction_request.get() + if request is None or lifecycle is not None: + if lifecycle is not None: + lifecycle.checkpoint() return await extraction_client.extract(file_path, format) async def wait_for_disconnect(): @@ -439,6 +549,9 @@ def extract_text_from_documents(documents: List[Document], file_ext: str) -> str async def cleanup_temp_file_async(file_path: str) -> None: """Clean up temporary file asynchronously.""" + lifecycle = _active_extraction_request.get() + if lifecycle is not None: + lifecycle.cleanup_started = True try: # Starlette cancellation scopes must not interrupt upload cleanup. with CancelScope(shield=True): @@ -745,6 +858,9 @@ async def insert_batch( ) -> List[str]: """Track started inserts so rollback waits for executor-backed work.""" nonlocal cleanup_needed + lifecycle = _active_extraction_request.get() + if lifecycle is not None: + lifecycle.checkpoint() insert_task = asyncio.create_task( vector_store.aadd_documents( batch_documents, ids=batch_ids, executor=executor @@ -848,6 +964,9 @@ async def embedding_consumer(): # Wait for all tasks to complete await asyncio.gather(producer_task, *consumer_tasks, return_exceptions=False) + lifecycle = _active_extraction_request.get() + if lifecycle is not None: + lifecycle.checkpoint() for batch_num in range(1, num_batches + 1): all_ids.extend(successful_batch_ids[batch_num]) @@ -977,21 +1096,27 @@ async def _process_documents_batched_sync( try: # Wrap sync call in executor to avoid blocking the event loop - batch_result_ids = await loop.run_in_executor( - executor, - lambda docs=batch_documents, ids=batch_ids: vector_store.add_documents( - docs, ids=ids - ), - ) + if _active_extraction_request.get() is not None: + batch_result_ids = await _run_sync_request_store( + executor, vector_store, batch_documents, batch_ids + ) + else: + batch_result_ids = await loop.run_in_executor( + executor, + lambda docs=batch_documents, ids=batch_ids: vector_store.add_documents( + docs, ids=ids + ), + ) all_ids.extend(batch_result_ids) except Exception as batch_error: logger.error("Batch %d failed: %s", batch_idx + 1, batch_error) # Rollback entire file from vector store - if ( - all_ids - ): # any batch succeeded (i.e., any chunks for this file were inserted) + if all_ids and _active_extraction_request.get() is not None: + # Sync stores cannot safely distinguish this attempt from older rows. + logger.warning("Retaining committed sync batches for file %s", file_id) + elif all_ids: logger.warning("Rolling back file %s due to batch failure", file_id) try: await loop.run_in_executor( @@ -1064,7 +1189,10 @@ async def store_data_in_vector_db( start_time = time.perf_counter() # Run document preparation in executor to avoid blocking the event loop loop = asyncio.get_running_loop() - docs = await loop.run_in_executor( + lifecycle = _active_extraction_request.get() + if lifecycle is not None: + lifecycle.checkpoint() + preparation = loop.run_in_executor( executor, _prepare_documents_sync, data, @@ -1072,6 +1200,11 @@ async def store_data_in_vector_db( user_id, clean_content, ) + if lifecycle is None: + docs = await preparation + else: + docs = await _drain_request_worker(preparation) + lifecycle.checkpoint() logger.info( "Documents prepared | %s", @@ -1091,10 +1224,16 @@ async def store_data_in_vector_db( if EMBEDDING_BATCH_SIZE <= 0: # synchronously embed the file and insert into vector store in one go if isinstance(vector_store, AsyncPgVector): + if lifecycle is not None: + lifecycle.start_uncancellable_storage() _tag_documents_for_ingestion(docs, file_id) ids = await vector_store.aadd_documents( docs, ids=[file_id] * len(docs), executor=executor ) + elif lifecycle is not None: + ids = await _run_sync_request_store( + executor, vector_store, docs, [file_id] * len(docs) + ) else: ids = vector_store.add_documents(docs, ids=[file_id] * len(docs)) else: @@ -1155,6 +1294,7 @@ async def store_data_in_vector_db( @router.post("/local/embed") +@_with_extraction_request("document", content_type="file_content_type") async def embed_local_file( document: StoreDocument, request: Request, entity_id: str = None ): @@ -1251,6 +1391,7 @@ async def embed_local_file( @router.post("/embed") +@_with_extraction_request("file") async def embed_file( request: Request, file_id: str = Form(...), @@ -1413,6 +1554,7 @@ async def load_document_context(request: Request, id: str, entity_id: str = None @router.post("/embed-upload") +@_with_extraction_request("uploaded_file") async def embed_file_upload( request: Request, file_id: str = Form(...), @@ -1564,6 +1706,7 @@ async def query_embeddings_by_file_ids(request: Request, body: QueryMultipleBody @router.post("/text") +@_with_extraction_request("file") async def extract_text_from_file( request: Request, file_id: str = Form(...), diff --git a/tests/extraction/test_routes.py b/tests/extraction/test_routes.py index 6ceea6d4..eb3c2cc0 100644 --- a/tests/extraction/test_routes.py +++ b/tests/extraction/test_routes.py @@ -497,3 +497,389 @@ async def test_asgi_disconnect_wins_over_processor_success( assert routes.extraction_client._admitted == 0 finally: await connection.close() + + +@pytest.mark.parametrize("route", ["/embed", "/embed-upload", "/local/embed"]) +@pytest.mark.parametrize("phase", ["preparation", "insertion"]) +@pytest.mark.parametrize("format", ["pdf", "docx"]) +async def test_disconnect_after_extraction_drains_and_preserves_old_rows( + api, processor, monkeypatch, route, phase, format +): + import threading + + client, store, uploads = api + monkeypatch.setattr(routes, "EMBEDDING_BATCH_SIZE", 1) + old = Document( + page_content="previous attempt", + metadata={ + "file_id": "caller-file", + "user_id": "caller", + "_rag_ingestion_attempt_id": "old", + }, + ) + store.persisted.append(old) + events = [] + preparation_entered = threading.Event() + preparation_release = threading.Event() + insert_entered = asyncio.Event() + insert_release = asyncio.Event() + prepare = routes._prepare_documents_sync + insert = store.aadd_documents + rollback = store.delete_by_metadata + + def delayed_prepare(*args): + events.append("preparation:start") + preparation_entered.set() + assert preparation_release.wait(3), "test preparation gate timed out" + result = prepare(*args) + events.append("preparation:done") + return result + + async def delayed_insert(*args, **kwargs): + events.append("insert:start") + insert_entered.set() + await insert_release.wait() + result = await insert(*args, **kwargs) + events.append("insert:commit") + return result + + async def track_rollback(*args, **kwargs): + events.append("rollback") + return await rollback(*args, **kwargs) + + if phase == "preparation": + monkeypatch.setattr(routes, "_prepare_documents_sync", delayed_prepare) + else: + monkeypatch.setattr(store, "aadd_documents", delayed_insert) + monkeypatch.setattr(store, "delete_by_metadata", track_rollback) + request = disconnect_request(client, uploads, route, format) + connection = ASGIConnection(client._transport.app, request, await request.aread()) + try: + if phase == "preparation": + assert await asyncio.to_thread(preparation_entered.wait, 1) + else: + await asyncio.wait_for(insert_entered.wait(), 1) + assert [r[0] for r in processor.requests] == ["GET", "POST"] + events.append("disconnect:queued") + connection.disconnect() + await asyncio.sleep(0.05) + assert not connection.task.done() + assert not store.rollbacks + assert list(uploads.rglob(f"*.{format}")) + preparation_release.set() + insert_release.set() + await asyncio.wait_for(asyncio.shield(connection.task), 1) + statuses = [ + message["status"] + for message in connection.sent + if message["type"] == "http.response.start" + ] + assert store.persisted == [old] + assert statuses == [499] + assert connection.task.cancelling() == 0 + if phase == "preparation": + assert not store.calls and not store.rollbacks + assert events.index("preparation:done") > events.index("disconnect:queued") + else: + assert events == [ + "insert:start", + "disconnect:queued", + "insert:commit", + "rollback", + ] + assert store.rollbacks[0]["_rag_ingestion_attempt_id"] != "old" + assert len(list(uploads.rglob(f"*.{format}"))) == ( + 1 if route == "/local/embed" else 0 + ) + finally: + preparation_release.set() + insert_release.set() + await connection.close() + + +class SyncRecordingStore: + def __init__(self, entered, release, events, old): + self.entered = entered + self.release = release + self.events = events + self.calls = [] + self.persisted = [old] + self.rollbacks = [] + + def add_documents(self, documents, ids=None): + self.calls.append(copy.deepcopy(documents)) + self.events.append("insert:start") + self.entered.set() + assert self.release.wait(3), "test sync insertion gate timed out" + self.persisted.extend(copy.deepcopy(documents)) + self.events.append("insert:commit") + return [ + f"sync-row-{len(self.calls)}-{index}" for index in range(len(documents)) + ] + + def delete_scoped(self, *args, **kwargs): + self.rollbacks.append((args, kwargs)) + raise AssertionError("disconnect must not delete previous attempts by file_id") + + +@pytest.mark.parametrize("route", ["/embed", "/embed-upload", "/local/embed"]) +@pytest.mark.parametrize("mode", ["async_single", "sync_single", "sync_batch"]) +@pytest.mark.parametrize("format", ["pdf", "docx"]) +async def test_disconnect_after_uncancellable_storage_starts_drains_without_undo( + api, processor, monkeypatch, route, mode, format +): + import threading + + client, store, uploads = api + monkeypatch.setattr( + routes, "EMBEDDING_BATCH_SIZE", 1 if mode == "sync_batch" else 0 + ) + old = Document( + page_content="previous attempt", + metadata={ + "file_id": "caller-file", + "user_id": "caller", + "_rag_ingestion_attempt_id": "old", + }, + ) + events = [] + sync_entered = threading.Event() + sync_release = threading.Event() + async_entered = asyncio.Event() + async_release = asyncio.Event() + if mode.startswith("sync"): + store = SyncRecordingStore(sync_entered, sync_release, events, old) + monkeypatch.setattr(routes, "vector_store", store) + else: + store.persisted.append(old) + insert = store.aadd_documents + + async def delayed_insert(*args, **kwargs): + events.append("insert:start") + async_entered.set() + await async_release.wait() + result = await insert(*args, **kwargs) + events.append("insert:commit") + return result + + monkeypatch.setattr(store, "aadd_documents", delayed_insert) + request = disconnect_request(client, uploads, route, format) + connection = ASGIConnection(client._transport.app, request, await request.aread()) + try: + if mode.startswith("sync"): + assert await asyncio.to_thread(sync_entered.wait, 1) + else: + await asyncio.wait_for(async_entered.wait(), 1) + events.append("disconnect:queued") + connection.disconnect() + await asyncio.sleep(0.05) + assert "http.disconnect" in connection.received + assert not connection.task.done() + assert store.persisted == [old] and not store.rollbacks + assert list(uploads.rglob(f"*.{format}")) + sync_release.set() + async_release.set() + await asyncio.wait_for(asyncio.shield(connection.task), 1) + assert old in store.persisted + assert len(store.persisted) == (3 if format == "pdf" else 2) + assert not store.rollbacks + assert events.index("insert:commit") > events.index("disconnect:queued") + expected_calls = 2 if mode == "sync_batch" and format == "pdf" else 1 + assert len(store.calls) == expected_calls + assert [ + m["status"] for m in connection.sent if m["type"] == "http.response.start" + ] == [499] + assert connection.task.cancelling() == 0 + assert len(list(uploads.rglob(f"*.{format}"))) == ( + 1 if route == "/local/embed" else 0 + ) + finally: + sync_release.set() + async_release.set() + await connection.close() + + +@pytest.mark.parametrize("route", ["/embed", "/embed-upload", "/local/embed"]) +@pytest.mark.parametrize("policy", ["disabled", "unselected"]) +async def test_legacy_request_keeps_post_extraction_disconnect_behavior( + api, monkeypatch, route, policy +): + import threading + + client, store, uploads = api + if policy == "disabled": + routes.extraction_client = ExtractionClient(ExtractionSettings()) + else: + routes.extraction_client = ExtractionClient( + ExtractionSettings( + enabled=True, + formats=frozenset({"docx"}), + ) + ) + loader = Mock() + loader.lazy_load.return_value = [Document(page_content="legacy text")] + monkeypatch.setattr(routes, "get_loader", Mock(return_value=(loader, True, "pdf"))) + entered = threading.Event() + release = threading.Event() + prepare = routes._prepare_documents_sync + + def delayed_prepare(*args): + entered.set() + assert release.wait(3), "test legacy preparation gate timed out" + return prepare(*args) + + monkeypatch.setattr(routes, "_prepare_documents_sync", delayed_prepare) + request = disconnect_request(client, uploads, route, "pdf") + connection = ASGIConnection(client._transport.app, request, await request.aread()) + try: + assert await asyncio.to_thread(entered.wait, 1) + connection.disconnect() + await asyncio.sleep(0.05) + release.set() + await asyncio.wait_for(asyncio.shield(connection.task), 1) + assert "http.disconnect" not in connection.received + assert len(store.persisted) == 1 + assert not store.rollbacks + assert [ + m["status"] for m in connection.sent if m["type"] == "http.response.start" + ] == [200] + finally: + release.set() + await connection.close() + + +@pytest.mark.parametrize("disconnect", [False, True]) +async def test_selected_sync_partial_failure_retains_old_and_committed_rows( + api, monkeypatch, disconnect +): + import threading + + client, _, uploads = api + monkeypatch.setattr(routes, "EMBEDDING_BATCH_SIZE", 1) + entered = threading.Event() + release = threading.Event() + events = [] + old = Document( + page_content="old", metadata={"file_id": "caller-file", "user_id": "caller"} + ) + store = SyncRecordingStore(entered, release, events, old) + insert = store.add_documents + + def fail_second_batch(documents, ids=None): + if store.calls: + raise RuntimeError("second sync batch failed") + return insert(documents, ids) + + monkeypatch.setattr(store, "add_documents", fail_second_batch) + monkeypatch.setattr(routes, "vector_store", store) + request = disconnect_request(client, uploads, "/embed", "pdf") + connection = ASGIConnection(client._transport.app, request, await request.aread()) + try: + assert await asyncio.to_thread(entered.wait, 1) + if disconnect: + connection.disconnect() + await asyncio.sleep(0.05) + assert "http.disconnect" in connection.received + release.set() + await asyncio.wait_for(asyncio.shield(connection.task), 1) + assert len(store.persisted) == 2 and old in store.persisted + assert not store.rollbacks + assert [ + m["status"] for m in connection.sent if m["type"] == "http.response.start" + ] == [499 if disconnect else 500] + finally: + release.set() + await connection.close() + + +async def test_disconnect_during_upload_cleanup_waits_for_removal(api, monkeypatch): + client, _, uploads = api + entered = asyncio.Event() + release = asyncio.Event() + remove = routes.aiofiles.os.remove + + async def delayed_remove(*args, **kwargs): + entered.set() + await release.wait() + await remove(*args, **kwargs) + + monkeypatch.setattr(routes.aiofiles.os, "remove", delayed_remove) + request = disconnect_request(client, uploads, "/text", "pdf") + connection = ASGIConnection(client._transport.app, request, await request.aread()) + try: + await asyncio.wait_for(entered.wait(), 1) + connection.disconnect() + await asyncio.sleep(0.05) + assert not connection.task.done() + release.set() + await asyncio.wait_for(asyncio.shield(connection.task), 1) + assert not list(uploads.rglob("*.pdf")) + assert [ + m["status"] for m in connection.sent if m["type"] == "http.response.start" + ] == [499] + finally: + release.set() + await connection.close() + + +@pytest.mark.parametrize("batch_size", [0, 1]) +async def test_disconnect_before_queued_sync_mutation_prevents_write( + api, monkeypatch, batch_size +): + import threading + + client, _, uploads = api + app = client._transport.app + original_executor = app.state.thread_pool + monkeypatch.setattr(routes, "EMBEDDING_BATCH_SIZE", batch_size) + blocker_entered = threading.Event() + blocker_release = threading.Event() + submitted = asyncio.Event() + old = Document( + page_content="old", metadata={"file_id": "caller-file", "user_id": "caller"} + ) + store = SyncRecordingStore(threading.Event(), threading.Event(), [], old) + monkeypatch.setattr(routes, "vector_store", store) + prepare = routes._prepare_documents_sync + run_sync = routes._run_sync_request_store + + def blocker(): + blocker_entered.set() + assert blocker_release.wait(3), "test executor gate timed out" + + def prepare_then_block_executor(*args): + docs = prepare(*args) + app.state.thread_pool.submit(blocker) + return docs + + async def observe_submission(*args): + submitted.set() + return await run_sync(*args) + + monkeypatch.setattr(routes, "_prepare_documents_sync", prepare_then_block_executor) + monkeypatch.setattr(routes, "_run_sync_request_store", observe_submission) + with ThreadPoolExecutor(max_workers=1) as executor: + app.state.thread_pool = executor + request = disconnect_request(client, uploads, "/embed", "pdf") + connection = ASGIConnection(app, request, await request.aread()) + try: + await asyncio.wait_for(submitted.wait(), 1) + assert blocker_entered.is_set() + connection.disconnect() + await asyncio.sleep(0.05) + assert "http.disconnect" in connection.received + assert not connection.task.done() + assert not store.calls + blocker_release.set() + await asyncio.wait_for(asyncio.shield(connection.task), 1) + assert store.persisted == [old] + assert not store.calls and not store.rollbacks + assert [ + m["status"] + for m in connection.sent + if m["type"] == "http.response.start" + ] == [499] + finally: + blocker_release.set() + await connection.close() + app.state.thread_pool = original_executor From c9d5bc8a57b184fb591f2bdef620f385a1ecd589 Mon Sep 17 00:00:00 2001 From: Lia Date: Fri, 23 Oct 2026 09:47:29 +0000 Subject: [PATCH 4/7] =?UTF-8?q?=F0=9F=94=8C=20fix:=20Protect=20Ingestion?= =?UTF-8?q?=20Rollback=20From=20Repeated=20Cancellation?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/routes/document_routes.py | 137 +++++++----- tests/extraction/test_routes.py | 373 ++++++++++++++++++++++++++++++++ 2 files changed, 459 insertions(+), 51 deletions(-) diff --git a/app/routes/document_routes.py b/app/routes/document_routes.py index 5b93453a..f6141ec4 100644 --- a/app/routes/document_routes.py +++ b/app/routes/document_routes.py @@ -362,6 +362,9 @@ def checkpoint(self): if self.disconnected: raise asyncio.CancelledError() + def begin_cleanup(self): + self.cleanup_started = True + def start_uncancellable_storage(self): with self._mutation_lock: self.checkpoint() @@ -408,16 +411,17 @@ async def wait_for_disconnect(): raise HTTPException(status_code=499, detail={"code": "CANCELLED"}) return operation.result() finally: - if not operation.done() and not ( - lifecycle.finish_storage or lifecycle.cleanup_started + if ( + not operation.done() + and not operation.cancelling() + and not (lifecycle.finish_storage or lifecycle.cleanup_started) ): operation.cancel() disconnected.cancel() # Do not clean uploads or return while workers/rollback can still run. - with CancelScope(shield=True): - await asyncio.gather( - operation, disconnected, return_exceptions=True - ) + await _drain_request_worker( + asyncio.gather(operation, disconnected, return_exceptions=True) + ) return wrapped @@ -425,13 +429,31 @@ async def wait_for_disconnect(): async def _drain_request_worker(future): - """Caller cancellation must not abandon a submitted executor worker.""" - try: - return await asyncio.shield(future) - except asyncio.CancelledError: - with CancelScope(shield=True): - await asyncio.gather(future, return_exceptions=True) - raise + """Defer repeated cancellation until submitted work or cleanup has finished.""" + cancellation = None + with CancelScope(shield=True): + while True: + try: + result = await asyncio.shield(future) + break + except asyncio.CancelledError as error: + lifecycle = _active_extraction_request.get() + if lifecycle is not None: + lifecycle.begin_cleanup() + # Executor futures can store CancelledError without being cancelled. + if future.cancelled() or ( + future.done() + and isinstance(future.exception(), asyncio.CancelledError) + ): + raise + cancellation = error + except Exception: + if cancellation is not None: + raise cancellation + raise + if cancellation is not None: + raise cancellation + return result async def _run_sync_request_store(executor, store, documents, ids): @@ -551,11 +573,16 @@ async def cleanup_temp_file_async(file_path: str) -> None: """Clean up temporary file asynchronously.""" lifecycle = _active_extraction_request.get() if lifecycle is not None: - lifecycle.cleanup_started = True + lifecycle.begin_cleanup() try: # Starlette cancellation scopes must not interrupt upload cleanup. with CancelScope(shield=True): - await aiofiles.os.remove(file_path) + if lifecycle is None: + await aiofiles.os.remove(file_path) + else: + await _drain_request_worker( + asyncio.create_task(aiofiles.os.remove(file_path)) + ) except Exception as e: logger.error( "Failed to remove temporary file | Path: %s | Error: %s | Traceback: %s", @@ -955,42 +982,7 @@ async def embedding_consumer(): producer_task = None consumer_tasks: List[asyncio.Task] = [] - try: - # Start producer and consumers concurrently - producer_task = asyncio.create_task(batch_producer()) - consumer_tasks = [ - asyncio.create_task(embedding_consumer()) for _ in range(consumer_count) - ] - - # Wait for all tasks to complete - await asyncio.gather(producer_task, *consumer_tasks, return_exceptions=False) - lifecycle = _active_extraction_request.get() - if lifecycle is not None: - lifecycle.checkpoint() - - for batch_num in range(1, num_batches + 1): - all_ids.extend(successful_batch_ids[batch_num]) - - logger.info( - "Async pipeline completed | user_id=%s | file_id=%s | inserted_ids=%d | %s", - user_id, - file_id, - len(all_ids), - get_process_memory_details(), - ) - - return all_ids - - except (Exception, asyncio.CancelledError) as e: - failure_event.set() - logger.error( - "Pipeline failed | user_id=%s | file_id=%s | inserted_batches=%d | error=%s | %s", - user_id, - file_id, - len(successful_batch_ids), - e, - get_process_memory_details(), - ) + async def cleanup_failed_pipeline(): if producer_task is not None or consumer_tasks: # if one of the tasks is still running, cancel it if producer_task is not None and not producer_task.done(): @@ -1039,6 +1031,49 @@ async def embedding_consumer(): get_process_memory_details(), ) + try: + # Start producer and consumers concurrently + producer_task = asyncio.create_task(batch_producer()) + consumer_tasks = [ + asyncio.create_task(embedding_consumer()) for _ in range(consumer_count) + ] + + # Wait for all tasks to complete + await asyncio.gather(producer_task, *consumer_tasks, return_exceptions=False) + lifecycle = _active_extraction_request.get() + if lifecycle is not None: + lifecycle.checkpoint() + + for batch_num in range(1, num_batches + 1): + all_ids.extend(successful_batch_ids[batch_num]) + + logger.info( + "Async pipeline completed | user_id=%s | file_id=%s | inserted_ids=%d | %s", + user_id, + file_id, + len(all_ids), + get_process_memory_details(), + ) + + return all_ids + + except (Exception, asyncio.CancelledError) as e: + failure_event.set() + logger.error( + "Pipeline failed | user_id=%s | file_id=%s | inserted_batches=%d | error=%s | %s", + user_id, + file_id, + len(successful_batch_ids), + e, + get_process_memory_details(), + ) + lifecycle = _active_extraction_request.get() + if lifecycle is None: + await cleanup_failed_pipeline() + else: + lifecycle.begin_cleanup() + await _drain_request_worker(asyncio.create_task(cleanup_failed_pipeline())) + # Re-raise the original error raise diff --git a/tests/extraction/test_routes.py b/tests/extraction/test_routes.py index eb3c2cc0..f869a078 100644 --- a/tests/extraction/test_routes.py +++ b/tests/extraction/test_routes.py @@ -883,3 +883,376 @@ async def observe_submission(*args): blocker_release.set() await connection.close() app.state.thread_pool = original_executor + + +@pytest.fixture +def freeze_disconnect_wrapper(monkeypatch): + entered = asyncio.Event() + release = asyncio.Event() + original_wait = asyncio.wait + state = {} + + async def frozen_wait(tasks, **kwargs): + result = await original_wait(tasks, **kwargs) + watcher = next( + ( + task + for task in tasks + if "_with_extraction_request" in task.get_coro().__qualname__ + and task.get_coro().__name__ == "wait_for_disconnect" + ), + None, + ) + if watcher in result[0]: + state["operation"] = next(task for task in tasks if task is not watcher) + entered.set() + await release.wait() + return result + + monkeypatch.setattr(asyncio, "wait", frozen_wait) + return entered, release, state + + +@pytest.mark.parametrize("route", ["/embed", "/embed-upload", "/local/embed"]) +@pytest.mark.parametrize("format", ["pdf", "docx"]) +@pytest.mark.parametrize("repeat_cancel", [False, True]) +async def test_completed_batch_disconnect_cannot_cancel_queued_attempt_rollback( + api, monkeypatch, freeze_disconnect_wrapper, route, format, repeat_cancel +): + import threading + + client, store, uploads = api + app = client._transport.app + original_executor = app.state.thread_pool + monkeypatch.setattr(routes, "EMBEDDING_BATCH_SIZE", 500) + wrapper_entered, wrapper_release, state = freeze_disconnect_wrapper + insert_entered = asyncio.Event() + insert_release = asyncio.Event() + rollback_queued = asyncio.Event() + worker_entered = threading.Event() + worker_release = threading.Event() + events = [] + queued_deletes = [] + old = Document( + page_content="old attempt", + metadata={ + "file_id": "caller-file", + "user_id": "caller", + "_rag_ingestion_attempt_id": "old", + }, + ) + store.persisted.append(old) + insert = store.aadd_documents + + def occupy_worker(): + worker_entered.set() + assert worker_release.wait(3), "test rollback executor gate timed out" + + async def final_insert(*args, executor=None, **kwargs): + executor.submit(occupy_worker) + insert_entered.set() + await insert_release.wait() + result = await insert(*args, executor=executor, **kwargs) + events.append("insert:commit") + return result + + def delete_current_attempt(metadata): + events.append("rollback:execute") + store.rollbacks.append(metadata) + store.persisted = [ + doc + for doc in store.persisted + if not all( + doc.metadata.get(key) == value for key, value in metadata.items() + ) + ] + + async def queued_rollback(metadata, executor=None): + future = asyncio.get_running_loop().run_in_executor( + executor, delete_current_attempt, metadata + ) + queued_deletes.append(future) + events.append("rollback:queued") + rollback_queued.set() + await future + + monkeypatch.setattr(store, "aadd_documents", final_insert) + monkeypatch.setattr(store, "delete_by_metadata", queued_rollback) + with ThreadPoolExecutor(max_workers=1) as executor: + app.state.thread_pool = executor + request = disconnect_request(client, uploads, route, format) + connection = ASGIConnection(app, request, await request.aread()) + try: + await asyncio.wait_for(insert_entered.wait(), 1) + assert await asyncio.to_thread(worker_entered.wait, 1) + connection.disconnect() + await asyncio.wait_for(wrapper_entered.wait(), 1) + insert_release.set() + await asyncio.wait_for(rollback_queued.wait(), 1) + assert len(store.persisted) > 1 and not store.rollbacks + wrapper_release.set() + await asyncio.sleep(0.05) + assert state["operation"].cancelling() == 0 + if repeat_cancel: + for _ in range(2): + state["operation"].cancel() + await asyncio.sleep(0) + await asyncio.sleep(0.05) + assert not connection.task.done(), events + assert not queued_deletes[0].cancelled(), events + assert list(uploads.rglob(f"*.{format}")) + worker_release.set() + await asyncio.wait_for(asyncio.shield(connection.task), 1) + assert events == ["insert:commit", "rollback:queued", "rollback:execute"] + assert store.persisted == [old] + assert len(store.rollbacks) == 1 + assert store.rollbacks[0]["_rag_ingestion_attempt_id"] != "old" + assert connection.task.cancelling() == 0 + assert [ + m["status"] + for m in connection.sent + if m["type"] == "http.response.start" + ] == [499] + assert len(list(uploads.rglob(f"*.{format}"))) == ( + 1 if route == "/local/embed" else 0 + ) + finally: + wrapper_release.set() + insert_release.set() + worker_release.set() + await connection.close() + app.state.thread_pool = original_executor + + +@pytest.mark.parametrize("route", ["/embed", "/embed-upload", "/local/embed"]) +@pytest.mark.parametrize("repeat_cancel", [False, True]) +async def test_disconnect_drain_owns_pending_insert_until_attempt_rollback( + api, monkeypatch, freeze_disconnect_wrapper, route, repeat_cancel +): + import threading + + client, store, uploads = api + app = client._transport.app + original_executor = app.state.thread_pool + monkeypatch.setattr(routes, "EMBEDDING_BATCH_SIZE", 1) + monkeypatch.setattr(routes, "PARALLEL_EXECUTION", 2) + wrapper_entered, wrapper_release, state = freeze_disconnect_wrapper + second_batch = asyncio.Event() + insert_draining = asyncio.Event() + worker_entered = threading.Event() + worker_release = threading.Event() + insert_tasks = [] + insert_futures = [] + events = [] + old = Document( + page_content="old", + metadata={ + "file_id": "caller-file", + "user_id": "caller", + "_rag_ingestion_attempt_id": "old", + }, + ) + store.persisted.append(old) + original_queue = asyncio.Queue + original_gather = asyncio.gather + + class ControlledQueue(original_queue): + async def get(self): + item = await super().get() + if isinstance(item, tuple) and len(item) == 4 and item[2] >= 2: + await second_batch.wait() + return item + + def observe_gather(*tasks, **kwargs): + if insert_tasks and insert_tasks[0] in tasks: + insert_draining.set() + return original_gather(*tasks, **kwargs) + + def commit_after_release(documents): + events.append("insert:start") + worker_entered.set() + assert worker_release.wait(3), "test insert executor gate timed out" + store.calls.append(copy.deepcopy(documents)) + store.persisted.extend(copy.deepcopy(documents)) + events.append("insert:commit") + return ["new-row"] + + async def blocked_insert(documents, ids=None, executor=None): + insert_tasks.append(asyncio.current_task()) + future = asyncio.get_running_loop().run_in_executor( + executor, commit_after_release, documents + ) + insert_futures.append(future) + return await future + + def remove_attempt(metadata): + events.append("rollback") + store.rollbacks.append(metadata) + store.persisted = [ + doc + for doc in store.persisted + if not all( + doc.metadata.get(key) == value for key, value in metadata.items() + ) + ] + + async def rollback(metadata, executor=None): + await asyncio.get_running_loop().run_in_executor( + executor, remove_attempt, metadata + ) + + monkeypatch.setattr(asyncio, "Queue", ControlledQueue) + monkeypatch.setattr(asyncio, "gather", observe_gather) + monkeypatch.setattr(store, "aadd_documents", blocked_insert) + monkeypatch.setattr(store, "delete_by_metadata", rollback) + with ThreadPoolExecutor(max_workers=1) as executor: + app.state.thread_pool = executor + request = disconnect_request(client, uploads, route, "pdf") + connection = ASGIConnection(app, request, await request.aread()) + try: + assert await asyncio.to_thread(worker_entered.wait, 1) + connection.disconnect() + await asyncio.wait_for(wrapper_entered.wait(), 1) + second_batch.set() + await asyncio.wait_for(insert_draining.wait(), 1) + assert state["operation"].cancelling() == 0 + wrapper_release.set() + await asyncio.sleep(0.05) + assert state["operation"].cancelling() == 0 + if repeat_cancel: + # The real disconnect started cleanup; stress subsequent owner cancellation. + state["operation"].cancel() + await asyncio.sleep(0) + state["operation"].cancel() + await asyncio.sleep(0.05) + assert not connection.task.done() + assert not insert_futures[0].cancelled() + assert not insert_tasks[0].cancelled() + assert store.persisted == [old] and not store.rollbacks + assert list(uploads.rglob("*.pdf")) + worker_release.set() + await asyncio.wait_for(asyncio.shield(connection.task), 1) + assert events == ["insert:start", "insert:commit", "rollback"] + assert len(insert_tasks) == 1 and len(store.calls) == 1 + assert store.persisted == [old] + assert store.rollbacks[0]["_rag_ingestion_attempt_id"] != "old" + assert [ + m["status"] + for m in connection.sent + if m["type"] == "http.response.start" + ] == [499] + assert len(list(uploads.rglob("*.pdf"))) == ( + 1 if route == "/local/embed" else 0 + ) + assert connection.task.cancelling() == 0 + finally: + second_batch.set() + wrapper_release.set() + worker_release.set() + await connection.close() + app.state.thread_pool = original_executor + + +@pytest.mark.parametrize("route", ["/embed", "/embed-upload", "/local/embed"]) +async def test_repeated_cancellation_during_preparation_still_drains_without_writes( + api, monkeypatch, freeze_disconnect_wrapper, route +): + import threading + + client, store, uploads = api + wrapper_entered, wrapper_release, state = freeze_disconnect_wrapper + worker_entered = threading.Event() + worker_release = threading.Event() + worker_done = threading.Event() + futures = [] + prepare = routes._prepare_documents_sync + drain = routes._drain_request_worker + + def blocked_prepare(*args): + worker_entered.set() + assert worker_release.wait(3), "test preparation executor gate timed out" + result = prepare(*args) + worker_done.set() + return result + + async def observe_drain(future): + if routes._active_extraction_request.get() is not None and not futures: + futures.append(future) + return await drain(future) + + monkeypatch.setattr(routes, "_prepare_documents_sync", blocked_prepare) + monkeypatch.setattr(routes, "_drain_request_worker", observe_drain) + request = disconnect_request(client, uploads, route, "pdf") + connection = ASGIConnection(client._transport.app, request, await request.aread()) + try: + assert await asyncio.to_thread(worker_entered.wait, 1) + connection.disconnect() + await asyncio.wait_for(wrapper_entered.wait(), 1) + wrapper_release.set() + await asyncio.sleep(0.05) + for _ in range(2): + state["operation"].cancel() + await asyncio.sleep(0) + await asyncio.sleep(0.05) + assert not connection.task.done() and not worker_done.is_set() + assert not futures[0].cancelled() + assert not store.calls and not store.persisted and not store.rollbacks + assert list(uploads.rglob("*.pdf")) + worker_release.set() + await asyncio.wait_for(asyncio.shield(connection.task), 1) + assert worker_done.is_set() + assert not store.calls and not store.persisted and not store.rollbacks + assert [ + m["status"] for m in connection.sent if m["type"] == "http.response.start" + ] == [499] + assert len(list(uploads.rglob("*.pdf"))) == ( + 1 if route == "/local/embed" else 0 + ) + assert connection.task.cancelling() == 0 + finally: + wrapper_release.set() + worker_release.set() + await connection.close() + + +async def test_repeated_cancellation_cannot_abandon_upload_unlink( + api, monkeypatch, freeze_disconnect_wrapper +): + client, _, uploads = api + wrapper_entered, wrapper_release, state = freeze_disconnect_wrapper + unlink_entered = asyncio.Event() + unlink_release = asyncio.Event() + unlink_tasks = [] + remove = routes.aiofiles.os.remove + + async def delayed_remove(*args, **kwargs): + unlink_tasks.append(asyncio.current_task()) + unlink_entered.set() + await unlink_release.wait() + await remove(*args, **kwargs) + + monkeypatch.setattr(routes.aiofiles.os, "remove", delayed_remove) + request = disconnect_request(client, uploads, "/text", "pdf") + connection = ASGIConnection(client._transport.app, request, await request.aread()) + try: + await asyncio.wait_for(unlink_entered.wait(), 1) + connection.disconnect() + await asyncio.wait_for(wrapper_entered.wait(), 1) + wrapper_release.set() + await asyncio.sleep(0.05) + for _ in range(2): + state["operation"].cancel() + await asyncio.sleep(0) + await asyncio.sleep(0.05) + assert not connection.task.done() and not unlink_tasks[0].cancelled() + assert list(uploads.rglob("*.pdf")) + unlink_release.set() + await asyncio.wait_for(asyncio.shield(connection.task), 1) + assert not list(uploads.rglob("*.pdf")) + assert [ + m["status"] for m in connection.sent if m["type"] == "http.response.start" + ] == [499] + finally: + wrapper_release.set() + unlink_release.set() + await connection.close() From 434382d0e60825af7f5d661c25f543251b7d1435 Mon Sep 17 00:00:00 2001 From: Lia Date: Fri, 23 Oct 2026 10:38:18 +0000 Subject: [PATCH 5/7] =?UTF-8?q?=F0=9F=94=8C=20feat:=20Require=20Isolated?= =?UTF-8?q?=20PDF=20and=20DOCX=20Extraction=20by=20Default?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- api-compose.yaml | 12 ++ app/routes/document_routes.py | 8 +- app/services/extraction_client.py | 38 +++---- app/utils/document_loader.py | 67 ++--------- docker-compose.yaml | 14 ++- extraction-compose.yaml | 19 ++++ tests/extraction/conftest.py | 2 +- tests/extraction/test_client.py | 66 +++++++++-- tests/extraction/test_deployment.py | 112 +++++++++++++++++++ tests/extraction/test_processor.py | 121 ++++++++++---------- tests/extraction/test_routes.py | 121 +++++++++++--------- tests/utils/test_document_loader.py | 165 +++++----------------------- tests/utils/test_lazy_load.py | 164 +-------------------------- 13 files changed, 401 insertions(+), 508 deletions(-) create mode 100644 extraction-compose.yaml create mode 100644 tests/extraction/test_deployment.py diff --git a/api-compose.yaml b/api-compose.yaml index aa2f95b8..fee0f1a0 100644 --- a/api-compose.yaml +++ b/api-compose.yaml @@ -1,14 +1,26 @@ version: '3.8' services: + extraction: + extends: + file: extraction-compose.yaml + service: extraction + rag_api: build: . + group_add: + - '10001' + depends_on: + extraction: + condition: service_healthy ports: - "8000:8000" volumes: - ./uploads:/app/uploads + - extraction-socket:/socket:ro env_file: - .env volumes: + extraction-socket: pgdata2: diff --git a/app/routes/document_routes.py b/app/routes/document_routes.py index f6141ec4..659ac588 100644 --- a/app/routes/document_routes.py +++ b/app/routes/document_routes.py @@ -382,9 +382,8 @@ def decorate(handler): async def wrapped(*args, **kwargs): arguments = signature.bind(*args, **kwargs).arguments file = arguments[file_argument] - settings = extraction_client.settings format = extraction_format(file.filename, getattr(file, content_type)) - if not settings.enabled or format not in settings.formats: + if format is None: return await handler(*args, **kwargs) request = arguments["request"] @@ -515,8 +514,7 @@ async def load_file_content( parsed. """ format = extraction_format(filename, content_type) - settings = extraction_client.settings - if settings.enabled and format in settings.formats: + if format is not None: try: if format == "pdf" and PDF_EXTRACT_IMAGES: raise ExtractionError("UNSUPPORTED_CONTENT") @@ -526,7 +524,7 @@ async def load_file_content( status_code=error.status_code, detail={"code": error.code}, ) from None - # Service pages are one-based; PyPDFLoader metadata is zero-based. + # Service pages are one-based; RAG metadata remains zero-based. documents = [ Document( page_content=segment["text"], diff --git a/app/services/extraction_client.py b/app/services/extraction_client.py index ec421067..20b4841d 100644 --- a/app/services/extraction_client.py +++ b/app/services/extraction_client.py @@ -50,41 +50,35 @@ def __init__(self, code: str, status_code: int = None): @dataclass(frozen=True) class ExtractionSettings: - enabled: bool = False socket: str = "/socket/extraction.sock" deadline: float = 10.0 - formats: frozenset[str] = frozenset({"pdf", "docx"}) def __post_init__(self): - if not self.enabled: - return - if not os.path.isabs(self.socket) or "\x00" in self.socket: + if ( + not isinstance(self.socket, str) + or not os.path.isabs(self.socket) + or "\x00" in self.socket + ): raise ValueError( "RAG_EXTRACTION_SOCKET must be an absolute Unix socket path" ) - if not math.isfinite(self.deadline) or not 0 < self.deadline <= 10: + if ( + type(self.deadline) not in (int, float) + or not math.isfinite(self.deadline) + or not 0 < self.deadline <= 10 + ): raise ValueError("RAG_EXTRACTION_DEADLINE_SECONDS must be in (0, 10]") - if not self.formats or not self.formats <= {"pdf", "docx"}: - raise ValueError("RAG_EXTRACTION_FORMATS must select pdf and/or docx") @classmethod def from_env(cls): - enabled = os.getenv("RAG_EXTRACTION_ENABLED", "false").lower() in ( - "true", - "1", - "yes", - "on", - ) - if not enabled: - return cls() + for key in ("RAG_EXTRACTION_ENABLED", "RAG_EXTRACTION_FORMATS"): + if key in os.environ: + raise ValueError( + f"{key} is obsolete; PDF/DOCX service extraction is required" + ) return cls( - enabled=True, socket=os.getenv("RAG_EXTRACTION_SOCKET", "/socket/extraction.sock"), deadline=float(os.getenv("RAG_EXTRACTION_DEADLINE_SECONDS", "10")), - formats=frozenset( - part.strip().lower() - for part in os.getenv("RAG_EXTRACTION_FORMATS", "pdf,docx").split(",") - ), ) @@ -297,7 +291,7 @@ async def _request( writer.transport.abort() async def extract(self, file_path: str, format: str): - if format not in self.settings.formats or format not in {"pdf", "docx"}: + if format not in {"pdf", "docx"}: raise ExtractionError("UNSUPPORTED_CONTENT") if self._admitted >= 2: raise ExtractionError("BUSY") diff --git a/app/utils/document_loader.py b/app/utils/document_loader.py index d25a64f6..93284d1f 100644 --- a/app/utils/document_loader.py +++ b/app/utils/document_loader.py @@ -4,15 +4,14 @@ import codecs import tempfile -from typing import Iterator, List, Optional +from typing import List, Optional import chardet from langchain_core.documents import Document -from app.config import known_source_ext, PDF_EXTRACT_IMAGES, CHUNK_OVERLAP, logger +from app.config import known_source_ext, CHUNK_OVERLAP, logger from langchain_community.document_loaders import ( TextLoader, - PyPDFLoader, CSVLoader, Docx2txtLoader, UnstructuredEPubLoader, @@ -94,7 +93,7 @@ def get_loader( filepath: str, raw_text: bool = False, ): - """Get the appropriate document loader based on file type and\or content type. + """Get the appropriate document loader based on file type and/or content type. When ``raw_text`` is True, text-formatted files (e.g. Markdown) are loaded verbatim with :class:`TextLoader` so their original formatting is @@ -103,14 +102,14 @@ def get_loader( (``raw_text=False``) so semantic loaders continue to strip formatting for better vector search quality. """ + if extraction_format(filename, file_content_type) is not None: + raise ValueError("PDF/DOCX require the asynchronous extraction service") file_ext = filename.split(".")[-1].lower() known_type = True # File Content Type reference: # ref.: https://developer.mozilla.org/en-US/docs/Web/HTTP/Guides/MIME_types/Common_types - if file_ext == "pdf" or file_content_type == "application/pdf": - loader = SafePyPDFLoader(filepath, extract_images=PDF_EXTRACT_IMAGES) - elif file_ext == "csv" or file_content_type == "text/csv": + if file_ext == "csv" or file_content_type == "text/csv": # Detect encoding for CSV files encoding = detect_file_encoding(filepath) @@ -170,10 +169,7 @@ def get_loader( loader = UnstructuredMarkdownLoader(filepath) elif file_ext == "epub" or file_content_type == "application/epub+zip": loader = UnstructuredEPubLoader(filepath) - elif file_ext in ["doc", "docx"] or file_content_type in [ - "application/msword", - "application/vnd.openxmlformats-officedocument.wordprocessingml.document", - ]: + elif file_ext == "doc" or file_content_type == "application/msword": loader = Docx2txtLoader(filepath) elif file_ext in ["xls", "xlsx"] or file_content_type in [ "application/vnd.ms-excel", @@ -253,52 +249,3 @@ def process_documents(documents: List[Document]) -> str: processed_text += new_content return processed_text.strip() - - -class SafePyPDFLoader: - """ - A wrapper around PyPDFLoader that handles image extraction failures gracefully. - Falls back to text-only extraction when image extraction fails. - - This is a workaround for issues with PyPDFLoader that can occur when extracting images - from PDFs, which can lead to KeyError exceptions if the PDF is malformed or has unsupported - image formats. This class attempts to load the PDF with image extraction enabled, and if it - fails due to a KeyError related to image filters, it falls back to loading the PDF - without image extraction. - ref.: https://github.com/langchain-ai/langchain/issues/26652 - """ - - def __init__(self, filepath: str, extract_images: bool = False): - self.filepath = filepath - self.extract_images = extract_images - self._temp_filepath = None # For compatibility with cleanup function - - def lazy_load(self) -> Iterator[Document]: - """Lazy load PDF documents with automatic fallback on image extraction errors.""" - loader = PyPDFLoader(self.filepath, extract_images=self.extract_images) - - if not self.extract_images: - # No image extraction: no fallback needed, stream directly - yield from loader.lazy_load() - return - - # extract_images=True: must collect eagerly so that a mid-stream - # KeyError doesn't leave already-yielded pages duplicated by the - # fallback (yield from + try/except would deliver partial + full). - try: - pages = list(loader.lazy_load()) - except KeyError as e: - if "/Filter" in str(e): - logger.warning( - f"PDF image extraction failed for {self.filepath}, falling back to text-only: {e}" - ) - fallback_loader = PyPDFLoader(self.filepath, extract_images=False) - pages = list(fallback_loader.lazy_load()) - else: - # Re-raise if it's a different error - raise - yield from pages - - def load(self) -> List[Document]: - """Load PDF documents with automatic fallback on image extraction errors.""" - return list(self.lazy_load()) diff --git a/docker-compose.yaml b/docker-compose.yaml index 45f59cc8..e5bc27a9 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -1,4 +1,9 @@ services: + extraction: + extends: + file: extraction-compose.yaml + service: extraction + db: image: ankane/pgvector:latest environment: @@ -12,6 +17,8 @@ services: fastapi: build: . + group_add: + - '10001' environment: - DB_HOST=db - DB_PORT=5432 @@ -20,10 +27,15 @@ services: - "8000:8000" volumes: - ./uploads:/app/uploads + - extraction-socket:/socket:ro depends_on: - - db + db: + condition: service_started + extraction: + condition: service_healthy env_file: - .env volumes: + extraction-socket: pgdata2: diff --git a/extraction-compose.yaml b/extraction-compose.yaml new file mode 100644 index 00000000..bad0e713 --- /dev/null +++ b/extraction-compose.yaml @@ -0,0 +1,19 @@ +services: + extraction: + build: + context: https://github.com/LibreChat-AI/code-interpreter.git#77bcd2ec2e63be523bfab2dfa11159a2ceec0088 + dockerfile: packages/extraction/Dockerfile + target: runtime + init: true + user: '10001:10001' + read_only: true + cap_drop: [ALL] + security_opt: [no-new-privileges:true] + network_mode: none + mem_limit: 768m + cpus: 2 + pids_limit: 32 + tmpfs: + - /jobs:rw,noexec,nosuid,nodev,size=96m,nr_inodes=1024,uid=10001,gid=10001,mode=0700 + volumes: + - extraction-socket:/socket diff --git a/tests/extraction/conftest.py b/tests/extraction/conftest.py index 4ffc95a4..cb8b52f6 100644 --- a/tests/extraction/conftest.py +++ b/tests/extraction/conftest.py @@ -50,7 +50,7 @@ def __init__(self, socket): def client(self, deadline=10): return ExtractionClient( - ExtractionSettings(enabled=True, socket=self.socket, deadline=deadline) + ExtractionSettings(socket=self.socket, deadline=deadline) ) async def handle(self, reader, writer): diff --git a/tests/extraction/test_client.py b/tests/extraction/test_client.py index 5ecde4fb..d5ff29fe 100644 --- a/tests/extraction/test_client.py +++ b/tests/extraction/test_client.py @@ -198,30 +198,74 @@ async def test_capabilities_rechecked_after_service_change(processor, input_file async def test_outage_is_safe(input_file, tmp_path): - client = ExtractionClient( - ExtractionSettings(enabled=True, socket=str(tmp_path / "missing")) - ) + client = ExtractionClient(ExtractionSettings(socket=str(tmp_path / "missing"))) with pytest.raises(ExtractionError, match="UNAVAILABLE"): await client.extract(input_file, "pdf") -def test_disabled_settings_ignore_unrelated_stale_values(monkeypatch): - monkeypatch.delenv("RAG_EXTRACTION_ENABLED", raising=False) - monkeypatch.setenv("RAG_EXTRACTION_DEADLINE_SECONDS", "stale") - assert not ExtractionSettings.from_env().enabled +def test_default_settings_require_service_without_environment(monkeypatch): + for key in ( + "RAG_EXTRACTION_ENABLED", + "RAG_EXTRACTION_FORMATS", + "RAG_EXTRACTION_SOCKET", + "RAG_EXTRACTION_DEADLINE_SECONDS", + ): + monkeypatch.delenv(key, raising=False) + settings = ExtractionSettings.from_env() + assert settings.socket == "/socket/extraction.sock" and settings.deadline == 10 + assert not hasattr(settings, "enabled") and not hasattr(settings, "formats") + + +@pytest.mark.parametrize( + "key,value", + [ + ("RAG_EXTRACTION_ENABLED", "false"), + ("RAG_EXTRACTION_ENABLED", "true"), + ("RAG_EXTRACTION_ENABLED", ""), + ("RAG_EXTRACTION_FORMATS", "docx"), + ("RAG_EXTRACTION_FORMATS", "pdf,docx"), + ("RAG_EXTRACTION_FORMATS", ""), + ], +) +def test_obsolete_opt_out_configuration_is_rejected(monkeypatch, key, value): + monkeypatch.setenv(key, value) + with pytest.raises(ValueError, match="obsolete"): + ExtractionSettings.from_env() @pytest.mark.parametrize( "kwargs", [ {"socket": "relative"}, + {"socket": ""}, {"deadline": 0}, {"deadline": 11}, + {"deadline": True}, {"deadline": float("nan")}, - {"formats": frozenset()}, - {"formats": frozenset({"doc"})}, + {"deadline": float("inf")}, + ], +) +def test_required_config_is_validated(kwargs): + with pytest.raises(ValueError): + ExtractionSettings(**kwargs) + + +@pytest.mark.parametrize( + "key,value", + [ + ("RAG_EXTRACTION_SOCKET", "relative"), + ("RAG_EXTRACTION_DEADLINE_SECONDS", "stale"), ], ) -def test_opt_in_config_is_validated(kwargs): +def test_invalid_environment_never_disables_extraction(monkeypatch, key, value): + monkeypatch.setenv(key, value) with pytest.raises(ValueError): - ExtractionSettings(enabled=True, **kwargs) + ExtractionSettings.from_env() + + +@pytest.mark.parametrize( + "kwargs", [{"enabled": False}, {"formats": frozenset({"docx"})}] +) +def test_client_constructor_cannot_opt_out(kwargs): + with pytest.raises(TypeError): + ExtractionSettings(**kwargs) diff --git a/tests/extraction/test_deployment.py b/tests/extraction/test_deployment.py new file mode 100644 index 00000000..9bb8d35b --- /dev/null +++ b/tests/extraction/test_deployment.py @@ -0,0 +1,112 @@ +"""Contracts for the processor required by both standard RAG deployments.""" + +import json +import shutil +import subprocess +import uuid +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parents[2] +PROCESSOR_REF = "77bcd2ec2e63be523bfab2dfa11159a2ceec0088" +PROCESSOR_CONTEXT = ( + f"https://github.com/LibreChat-AI/code-interpreter.git#{PROCESSOR_REF}" +) + + +def stage_compose_files(directory): + for name in ("docker-compose.yaml", "api-compose.yaml", "extraction-compose.yaml"): + shutil.copyfile(ROOT / name, directory / name) + # Never read deployment credentials while rendering or qualifying the service. + (directory / ".env").write_text("") + return directory + + +def compose_command(directory, project, *files): + command = ["docker", "compose", "--env-file", "/dev/null", "-p", project] + for file in files: + command.extend(["-f", str(directory / file)]) + return command + + +def render(directory, *files): + command = compose_command(directory, f"rag-contract-{uuid.uuid4().hex}", *files) + result = subprocess.run( + [*command, "config", "--no-env-resolution", "--format", "json"], + capture_output=True, + text=True, + check=True, + timeout=15, + ) + return json.loads(result.stdout) + + +def assert_hardened_processor(service): + assert service["build"]["context"] == PROCESSOR_CONTEXT + assert service["build"]["dockerfile"] == "packages/extraction/Dockerfile" + assert service["build"]["target"] == "runtime" + assert service["init"] is True and service["user"] == "10001:10001" + assert service["read_only"] is True + assert service["cap_drop"] == ["ALL"] + assert service["security_opt"] == ["no-new-privileges:true"] + assert service["network_mode"] == "none" + assert int(service["mem_limit"]) == 768 * 1024 * 1024 + assert float(service["cpus"]) == 2 and int(service["pids_limit"]) == 32 + assert service["tmpfs"] == [ + "/jobs:rw,noexec,nosuid,nodev,size=96m,nr_inodes=1024,uid=10001,gid=10001,mode=0700" + ] + assert len(service["volumes"]) == 1 + mount = service["volumes"][0] + assert mount["type"] == "volume" and mount["source"] == "extraction-socket" + assert mount["target"] == "/socket" and not mount.get("read_only", False) + for key in ( + "environment", + "env_file", + "ports", + "expose", + "profiles", + "privileged", + "cap_add", + "command", + "entrypoint", + ): + assert not service.get(key), key + + +@pytest.mark.parametrize( + "recipe,caller", + [("docker-compose.yaml", "fastapi"), ("api-compose.yaml", "rag_api")], +) +def test_standard_deployment_requires_one_hardened_processor(tmp_path, recipe, caller): + config = render(stage_compose_files(tmp_path), recipe) + assert_hardened_processor(config["services"]["extraction"]) + app = config["services"][caller] + assert "10001" in app["group_add"] + assert app["depends_on"]["extraction"]["condition"] == "service_healthy" + mount = next(item for item in app["volumes"] if item["target"] == "/socket") + assert ( + mount["type"] == "volume" + and mount["source"] == "extraction-socket" + and mount["read_only"] is True + ) + assert not app.get("profiles") + assert ( + not {"RAG_EXTRACTION_ENABLED", "RAG_EXTRACTION_FORMATS"} + & app.get("environment", {}).keys() + ) + + +def test_combined_recipes_share_one_processor_and_socket(tmp_path): + config = render( + stage_compose_files(tmp_path), "docker-compose.yaml", "api-compose.yaml" + ) + assert set(config["services"]) == {"db", "fastapi", "rag_api", "extraction"} + assert_hardened_processor(config["services"]["extraction"]) + for caller in ("fastapi", "rag_api"): + mount = next( + item + for item in config["services"][caller]["volumes"] + if item["target"] == "/socket" + ) + assert mount["source"] == "extraction-socket" and mount["read_only"] is True diff --git a/tests/extraction/test_processor.py b/tests/extraction/test_processor.py index 212c3807..b9b83d47 100644 --- a/tests/extraction/test_processor.py +++ b/tests/extraction/test_processor.py @@ -1,14 +1,20 @@ -"""Opt-in real client qualification against the shipped hardened processor image.""" +"""Current Python client qualification against the default hardened deployment.""" -import os +import json import subprocess -import time import uuid from pathlib import Path import aiofiles import pytest +from tests.extraction.test_deployment import ( + assert_hardened_processor, + compose_command, + render, + stage_compose_files, +) + pytestmark = pytest.mark.integration @@ -21,69 +27,73 @@ def docker(*args, check=True): return result -def test_python_client_to_hardened_processor(): - image = os.getenv("RAG_EXTRACTION_TEST_IMAGE") - if not image: - pytest.skip( - "RAG_EXTRACTION_TEST_IMAGE must name a built Code Interpreter extraction runtime" - ) +def test_python_client_to_default_hardened_processor(tmp_path): + directory = stage_compose_files(tmp_path) + assert_hardened_processor( + render(directory, "docker-compose.yaml")["services"]["extraction"] + ) run_id = uuid.uuid4().hex - name = f"rag-extraction-{run_id}" - volume = f"rag-extraction-{run_id}" + project = f"rag-extraction-{run_id}" + volume = f"{project}_extraction-socket" caller = f"rag-extraction-caller-{run_id}" + compose = compose_command(directory, project, "docker-compose.yaml") + image = f"{project}-extraction" + + def run_compose(*args, check=True, timeout=40): + result = subprocess.run( + [*compose, *args], capture_output=True, text=True, timeout=timeout + ) + if check: + assert result.returncode == 0, result.stdout + result.stderr + return result + root = Path(__file__).resolve().parents[2] client = root / "app/services/extraction_client.py" fixtures = Path(__file__).parent / "fixtures" library = Path(aiofiles.__file__).parent - docker("volume", "create", volume) try: - docker( - "run", - "-d", - "--name", - name, - "--init", - "--network", - "none", - "--read-only", - "--user", - "10001:10001", - "--cap-drop", - "ALL", - "--security-opt", - "no-new-privileges:true", - "--memory", - "768m", - "--cpus", - "2", - "--pids-limit", - "32", - "--tmpfs", - "/jobs:rw,noexec,nosuid,nodev,size=96m,nr_inodes=1024,uid=10001,gid=10001,mode=0700", - "--mount", - f"type=volume,src={volume},dst=/socket", - image, + run_compose("build", "extraction", timeout=240) + run_compose("up", "-d", "--wait", "--wait-timeout", "30", "extraction") + name = run_compose("ps", "-q", "extraction").stdout.strip() + image = docker("inspect", "--format", "{{.Config.Image}}", name).stdout.strip() + runtime = json.loads(docker("inspect", name).stdout)[0] + host = runtime["HostConfig"] + assert ( + host["NetworkMode"] == "none" + and host["ReadonlyRootfs"] + and not host["Privileged"] + ) + assert ( + host["CapDrop"] == ["ALL"] + and "no-new-privileges:true" in host["SecurityOpt"] + ) + assert runtime["Config"]["User"] == "10001:10001" + assert host["Memory"] == 768 * 1024 * 1024 and host["NanoCpus"] == 2 * 10**9 + assert host["PidsLimit"] == 32 and set(host["Tmpfs"]) == {"/jobs"} + assert ( + len(runtime["Mounts"]) == 1 + and runtime["Mounts"][0]["Destination"] == "/socket" ) - for _ in range(30): - status = docker( - "inspect", "--format", "{{.State.Health.Status}}", name - ).stdout.strip() - if status == "healthy": - break - if status == "unhealthy": - pytest.fail( - "Hardened extraction processor failed its isolation/health probe" - ) - time.sleep(1) - else: - pytest.fail("Hardened extraction processor did not become healthy") script = """ import asyncio +import errno +import os +import stat import sys sys.path.insert(0, "/") from client import ExtractionClient, ExtractionSettings async def check(): - client = ExtractionClient(ExtractionSettings(enabled=True)) + settings = ExtractionSettings.from_env() + assert settings.socket == "/socket/extraction.sock" + assert stat.S_IMODE(os.stat("/socket").st_mode) == 0o770 + assert stat.S_IMODE(os.stat(settings.socket).st_mode) == 0o660 + try: + open("/socket/not-writable", "wb") + except OSError as error: + assert error.errno == errno.EROFS + else: + raise AssertionError("caller socket mount is writable") + client = ExtractionClient(settings) pdf = await client.extract("/fixtures/pages.pdf", "pdf") assert pdf == [ {"kind": "page", "index": 1, "text": "First page text"}, @@ -106,7 +116,7 @@ async def check(): "none", "--read-only", "--user", - "10001:10001", + "65532:65532", "--group-add", "10001", "--cap-drop", @@ -140,5 +150,6 @@ async def check(): ) finally: docker("rm", "-f", caller, check=False) - docker("rm", "-f", name, check=False) - docker("volume", "rm", volume, check=False) + run_compose("down", "-v", "--remove-orphans", check=False) + if image: + docker("image", "rm", image, check=False) diff --git a/tests/extraction/test_routes.py b/tests/extraction/test_routes.py index f869a078..6bda7570 100644 --- a/tests/extraction/test_routes.py +++ b/tests/extraction/test_routes.py @@ -19,6 +19,7 @@ ROUTES = ["/text", "/embed", "/embed-upload", "/local/embed"] MIMES = { + "txt": "text/plain", "pdf": "application/pdf", "docx": "application/vnd.openxmlformats-officedocument.wordprocessingml.document", } @@ -83,10 +84,15 @@ async def api(processor, monkeypatch, tmp_path): monkeypatch.setattr(routes, "PARALLEL_EXECUTION", 1) monkeypatch.setenv("JWT_SECRET", "test-extraction-secret") # The migrated path must never construct either legacy parser. - monkeypatch.setattr( - "app.utils.document_loader.PyPDFLoader", - Mock(side_effect=AssertionError("local PDF parser")), - ) + local_loader = routes.get_loader + + def no_migrated_local_loader(filename, content_type, *args, **kwargs): + assert ( + routes.extraction_format(filename, content_type) is None + ), "local migrated parser" + return local_loader(filename, content_type, *args, **kwargs) + + monkeypatch.setattr(routes, "get_loader", no_migrated_local_loader) monkeypatch.setattr( "app.utils.document_loader.Docx2txtLoader", Mock(side_effect=AssertionError("local DOCX parser")), @@ -216,7 +222,7 @@ async def test_selected_format_failure_never_falls_back_or_persists( processor.error = (422, "UNSUPPORTED_CONTENT") elif failure == "outage": routes.extraction_client = ExtractionClient( - ExtractionSettings(enabled=True, socket="/socket/missing.sock") + ExtractionSettings(socket="/socket/missing.sock") ) elif failure == "incompatible": processor.capabilities["version"] = 2 @@ -329,40 +335,32 @@ async def test_requested_pdf_image_extraction_is_not_silently_downgraded( assert not processor.requests and not store.calls -async def test_format_policy_leaves_unselected_formats_on_existing_path( - api, processor, monkeypatch +@pytest.mark.parametrize("route", ROUTES) +@pytest.mark.parametrize("format", ["pdf", "docx"]) +async def test_no_extraction_configuration_still_requires_service( + api, processor, monkeypatch, route, format ): client, store, uploads = api - routes.extraction_client = ExtractionClient( - ExtractionSettings( - enabled=True, - socket=processor.socket, - formats=frozenset({"docx"}), - ) - ) - local = Mock() - local.lazy_load.return_value = [Document(page_content="existing PDF path")] - get_loader = Mock(return_value=(local, True, "pdf")) - monkeypatch.setattr(routes, "get_loader", get_loader) - response = await post(client, uploads, "/text") - assert response.status_code == 200 - assert response.json()["text"] == "existing PDF path" - get_loader.assert_called_once() - assert not processor.requests - - -async def test_disabled_by_default_keeps_existing_loader(api, processor, monkeypatch): - client, store, uploads = api - routes.extraction_client = ExtractionClient(ExtractionSettings()) - local = Mock() - local.lazy_load.return_value = [Document(page_content="legacy extraction")] - get_loader = Mock(return_value=(local, True, "pdf")) - monkeypatch.setattr(routes, "get_loader", get_loader) - response = await post(client, uploads, "/text") - assert response.status_code == 200 - assert response.json()["text"] == "legacy extraction" - get_loader.assert_called_once() - assert not processor.requests + for key in ( + "RAG_EXTRACTION_ENABLED", + "RAG_EXTRACTION_FORMATS", + "RAG_EXTRACTION_SOCKET", + "RAG_EXTRACTION_DEADLINE_SECONDS", + ): + monkeypatch.delenv(key, raising=False) + defaults = ExtractionSettings.from_env() + routes.extraction_client = ExtractionClient(defaults) + connect = asyncio.open_unix_connection + + async def disposable_socket(path, **kwargs): + assert path == "/socket/extraction.sock" + return await connect(processor.socket, **kwargs) + + monkeypatch.setattr(asyncio, "open_unix_connection", disposable_socket) + response = await post(client, uploads, route, format) + assert response.status_code == 200, response.text + assert [r[0] for r in processor.requests] == ["GET", "POST"] + assert response.json()["known_type"] is True class ASGIConnection: @@ -700,25 +698,15 @@ async def delayed_insert(*args, **kwargs): @pytest.mark.parametrize("route", ["/embed", "/embed-upload", "/local/embed"]) -@pytest.mark.parametrize("policy", ["disabled", "unselected"]) -async def test_legacy_request_keeps_post_extraction_disconnect_behavior( - api, monkeypatch, route, policy +async def test_unmigrated_text_keeps_post_extraction_disconnect_behavior( + api, monkeypatch, route ): import threading client, store, uploads = api - if policy == "disabled": - routes.extraction_client = ExtractionClient(ExtractionSettings()) - else: - routes.extraction_client = ExtractionClient( - ExtractionSettings( - enabled=True, - formats=frozenset({"docx"}), - ) - ) loader = Mock() loader.lazy_load.return_value = [Document(page_content="legacy text")] - monkeypatch.setattr(routes, "get_loader", Mock(return_value=(loader, True, "pdf"))) + monkeypatch.setattr(routes, "get_loader", Mock(return_value=(loader, True, "txt"))) entered = threading.Event() release = threading.Event() prepare = routes._prepare_documents_sync @@ -729,7 +717,7 @@ def delayed_prepare(*args): return prepare(*args) monkeypatch.setattr(routes, "_prepare_documents_sync", delayed_prepare) - request = disconnect_request(client, uploads, route, "pdf") + request = disconnect_request(client, uploads, route, "txt") connection = ASGIConnection(client._transport.app, request, await request.aread()) try: assert await asyncio.to_thread(entered.wait, 1) @@ -1256,3 +1244,34 @@ async def delayed_remove(*args, **kwargs): wrapper_release.set() unlink_release.set() await connection.close() + + +@pytest.mark.parametrize("route", ROUTES) +@pytest.mark.parametrize("format", ["pdf", "docx"]) +async def test_default_socket_unavailable_fails_closed_without_affecting_text( + api, processor, monkeypatch, route, format +): + client, store, uploads = api + for key in ( + "RAG_EXTRACTION_ENABLED", + "RAG_EXTRACTION_FORMATS", + "RAG_EXTRACTION_SOCKET", + "RAG_EXTRACTION_DEADLINE_SECONDS", + ): + monkeypatch.delenv(key, raising=False) + routes.extraction_client = ExtractionClient(ExtractionSettings.from_env()) + + async def unavailable(path, **kwargs): + assert path == "/socket/extraction.sock" + raise FileNotFoundError() + + monkeypatch.setattr(asyncio, "open_unix_connection", unavailable) + response = await post(client, uploads, route, format) + assert response.status_code == 503 + assert response.json()["detail"]["code"] == "UNAVAILABLE" + assert not store.calls and not store.persisted and not processor.requests + assert len(list(uploads.rglob(f"*.{format}"))) == ( + 1 if route == "/local/embed" else 0 + ) + plaintext = await post(client, uploads, "/text", "txt", b"plain text", "text/plain") + assert plaintext.status_code == 200 and plaintext.json()["text"] == "plain text" diff --git a/tests/utils/test_document_loader.py b/tests/utils/test_document_loader.py index 8baafa78..051fe60b 100644 --- a/tests/utils/test_document_loader.py +++ b/tests/utils/test_document_loader.py @@ -1,7 +1,3 @@ -import os -from collections.abc import Iterator -from unittest.mock import MagicMock, patch - import pytest from app.utils.document_loader import get_loader, clean_text, process_documents @@ -46,17 +42,6 @@ def test_process_documents(): assert "# PAGE 2" in processed -def test_safe_pdf_loader_class(): - """Test that SafePyPDFLoader class can be instantiated""" - from app.utils.document_loader import SafePyPDFLoader - - # Test instantiation - loader = SafePyPDFLoader("dummy.pdf", extract_images=True) - assert loader.filepath == "dummy.pdf" - assert loader.extract_images == True - assert loader._temp_filepath is None - - def test_get_loader_text_lazy_load(tmp_path): """Test that lazy_load returns an iterator yielding documents.""" file_path = tmp_path / "test.txt" @@ -69,111 +54,6 @@ def test_get_loader_text_lazy_load(tmp_path): assert hasattr(data[0], "page_content") -def test_get_loader_pdf(tmp_path): - """Test get_loader returns SafePyPDFLoader for PDF files""" - # Create a dummy PDF file path (doesn't need to be real for this test) - file_path = tmp_path / "test.pdf" - file_path.write_text("dummy content") # Not a real PDF, but that's OK for this test - - loader, known_type, file_ext = get_loader( - "test.pdf", "application/pdf", str(file_path) - ) - - # Check that we get our SafePyPDFLoader - from app.utils.document_loader import SafePyPDFLoader - - assert isinstance(loader, SafePyPDFLoader) - assert known_type is True - assert file_ext == "pdf" - - -def test_safe_pdf_loader_lazy_load(): - """Test that SafePyPDFLoader.lazy_load() returns an Iterator.""" - from app.utils.document_loader import SafePyPDFLoader - - loader = SafePyPDFLoader("dummy.pdf", extract_images=False) - assert hasattr(loader, "lazy_load") - result = loader.lazy_load() - assert isinstance(result, Iterator) - - -def test_safe_pdf_loader_fallback_no_duplicate_pages(): - """Fallback after mid-stream KeyError must not duplicate already-yielded pages.""" - from app.utils.document_loader import SafePyPDFLoader - - fallback_docs = [Document(page_content=f"fallback page {i}") for i in range(5)] - - def primary_gen(): - yield Document(page_content="partial page 0") - yield Document(page_content="partial page 1") - raise KeyError("/Filter") - - def fallback_gen(): - yield from fallback_docs - - loader = SafePyPDFLoader("dummy.pdf", extract_images=True) - - with patch("app.utils.document_loader.PyPDFLoader") as MockPDF: - primary_instance = MagicMock() - primary_instance.lazy_load.side_effect = primary_gen - fallback_instance = MagicMock() - fallback_instance.lazy_load.side_effect = fallback_gen - MockPDF.side_effect = [primary_instance, fallback_instance] - - result = list(loader.lazy_load()) - - # Must be exactly the 5 fallback pages, NOT 2 partial + 5 fallback = 7 - assert len(result) == 5 - assert result[0].page_content == "fallback page 0" - assert result[-1].page_content == "fallback page 4" - - -def test_safe_pdf_loader_fallback_via_load(): - """load() delegates to lazy_load(), so fallback must also be correct via load().""" - from app.utils.document_loader import SafePyPDFLoader - - fallback_docs = [Document(page_content=f"fb {i}") for i in range(3)] - - def primary_gen(): - yield Document(page_content="partial 0") - raise KeyError("/Filter") - - def fallback_gen(): - yield from fallback_docs - - loader = SafePyPDFLoader("dummy.pdf", extract_images=True) - - with patch("app.utils.document_loader.PyPDFLoader") as MockPDF: - primary_instance = MagicMock() - primary_instance.lazy_load.side_effect = primary_gen - fallback_instance = MagicMock() - fallback_instance.lazy_load.side_effect = fallback_gen - MockPDF.side_effect = [primary_instance, fallback_instance] - - result = loader.load() - - assert len(result) == 3 - assert result[0].page_content == "fb 0" - - -def test_safe_pdf_loader_non_filter_error_propagates(): - """KeyError that isn't /Filter should propagate, not silently fallback.""" - from app.utils.document_loader import SafePyPDFLoader - - def bad_gen(): - raise KeyError("SomeOtherKey") - - loader = SafePyPDFLoader("dummy.pdf", extract_images=True) - - with patch("app.utils.document_loader.PyPDFLoader") as MockPDF: - instance = MagicMock() - instance.lazy_load.side_effect = bad_gen - MockPDF.return_value = instance - - with pytest.raises(KeyError, match="SomeOtherKey"): - list(loader.lazy_load()) - - MARKDOWN_SAMPLE = ( "# Heading\n\n" "**bold** and *italic* text with a [link](https://example.com).\n\n" @@ -236,26 +116,9 @@ def test_get_loader_markdown_raw_text_by_extension_only(tmp_path): assert isinstance(loader, TextLoader) -def test_get_loader_raw_text_leaves_pdf_alone(tmp_path): - """raw_text must not disturb binary formats — PDF still uses the PDF loader.""" - from app.utils.document_loader import SafePyPDFLoader - - file_path = tmp_path / "doc.pdf" - file_path.write_text("not a real pdf") - - loader, _, file_ext = get_loader( - "doc.pdf", "application/pdf", str(file_path), raw_text=True - ) - - assert isinstance(loader, SafePyPDFLoader) - assert file_ext == "pdf" - - @pytest.mark.parametrize( "filename, expected_loader_name", [ - ("doc.pdf", "SafePyPDFLoader"), - ("report.docx", "Docx2txtLoader"), ("book.epub", "UnstructuredEPubLoader"), ("data.xlsx", "UnstructuredExcelLoader"), ("slides.pptx", "UnstructuredPowerPointLoader"), @@ -273,8 +136,30 @@ def test_get_loader_raw_text_respects_binary_extensions_over_markdown_mime( file_path = tmp_path / filename file_path.write_text("placeholder binary content") - loader, _, _ = get_loader( - filename, "text/markdown", str(file_path), raw_text=True - ) + loader, _, _ = get_loader(filename, "text/markdown", str(file_path), raw_text=True) assert type(loader).__name__ == expected_loader_name + + +@pytest.mark.parametrize( + "filename,mime", + [ + ("doc.pdf", "application/pdf"), + ("doc.pdf", "text/markdown"), + ("doc.docx", "text/xml"), + ("doc.docx", "text/markdown"), + ("opaque.bin", "application/pdf"), + ( + "opaque.bin", + "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + ), + ], +) +@pytest.mark.parametrize("raw_text", [False, True]) +def test_migrated_formats_cannot_construct_local_loader( + tmp_path, filename, mime, raw_text +): + path = tmp_path / filename + path.write_bytes(b"not parsed locally") + with pytest.raises(ValueError, match="asynchronous extraction service"): + get_loader(filename, mime, str(path), raw_text=raw_text) diff --git a/tests/utils/test_lazy_load.py b/tests/utils/test_lazy_load.py index 9506f95f..144dd9e0 100644 --- a/tests/utils/test_lazy_load.py +++ b/tests/utils/test_lazy_load.py @@ -4,7 +4,7 @@ Verifies that every loader returned by get_loader() supports lazy_load(), returns a generator/iterator, and yields valid Document objects with content. Also includes memory benchmarks comparing lazy_load() vs load() to document -which loaders benefit from streaming (CSV, PDF) vs which don't (Unstructured*). +which local loaders benefit from streaming (CSV) vs which don't (Unstructured*). """ import gc @@ -12,12 +12,11 @@ import shutil import tracemalloc import zipfile -from collections.abc import Iterator import pytest from langchain_core.documents import Document -from app.utils.document_loader import get_loader, SafePyPDFLoader +from app.utils.document_loader import get_loader # --------------------------------------------------------------------------- # Environment checks — these deps aren't guaranteed in every CI runner @@ -51,85 +50,6 @@ # --------------------------------------------------------------------------- -def _make_pdf(path, *, num_pages=1): - """Create a PDF with extractable text using pypdf. - - When *num_pages* > 1 each page gets unique text so tracemalloc can - observe the difference between holding 1 page vs N pages in memory. - """ - from pypdf import PdfWriter - from pypdf.generic import ( - DecodedStreamObject, - DictionaryObject, - NameObject, - ) - - writer = PdfWriter() - for i in range(num_pages): - writer.add_blank_page(width=612, height=792) - page = writer.pages[i] - - font_dict = DictionaryObject() - font_dict[NameObject("/Type")] = NameObject("/Font") - font_dict[NameObject("/Subtype")] = NameObject("/Type1") - font_dict[NameObject("/BaseFont")] = NameObject("/Helvetica") - - resources = DictionaryObject() - font_resources = DictionaryObject() - font_resources[NameObject("/F1")] = font_dict - resources[NameObject("/Font")] = font_resources - page[NameObject("/Resources")] = resources - - # Pad each page with unique filler so memory differences are measurable - filler = f"PAGE {i} " + ("X" * 2000) - content = f"BT /F1 12 Tf 100 700 Td ({filler}) Tj ET".encode() - stream = DecodedStreamObject() - stream.set_data(content) - page[NameObject("/Contents")] = stream - - with open(path, "wb") as f: - writer.write(f) - - -def _make_docx(path): - """Create a minimal DOCX (Office Open XML zip) with text content.""" - content_types = ( - '' - '' - ' ' - ' ' - ' ' - "" - ) - rels = ( - '' - '' - ' ' - "" - ) - document = ( - '' - '' - " " - " Hello from lazy_load DOCX test" - " " - "" - ) - word_rels = ( - '' - '' - "" - ) - with zipfile.ZipFile(path, "w", zipfile.ZIP_DEFLATED) as z: - z.writestr("[Content_Types].xml", content_types) - z.writestr("_rels/.rels", rels) - z.writestr("word/document.xml", document) - z.writestr("word/_rels/document.xml.rels", word_rels) - - def _make_xlsx(path): """Create a minimal XLSX with openpyxl.""" from openpyxl import Workbook @@ -283,14 +203,6 @@ def _make_large_csv(path, num_rows=500): "Hello from lazy_load PY test", id="py-source", ), - pytest.param("test.pdf", "application/pdf", _make_pdf, "PAGE 0", id="pdf"), - pytest.param( - "test.docx", - "application/vnd.openxmlformats-officedocument.wordprocessingml.document", - _make_docx, - "lazy_load DOCX test", - id="docx", - ), pytest.param( "test.xlsx", "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", @@ -383,41 +295,6 @@ def test_lazy_load_matches_load( ), f"{type(loader).__name__} doc[{i}]: content mismatch between load() and lazy_load()" -# --------------------------------------------------------------------------- -# SafePyPDFLoader-specific tests -# --------------------------------------------------------------------------- - - -def test_safe_pdf_loader_lazy_load_is_iterator(tmp_path): - """SafePyPDFLoader.lazy_load() should return an Iterator.""" - pdf_path = tmp_path / "gen_test.pdf" - _make_pdf(str(pdf_path)) - - loader = SafePyPDFLoader(str(pdf_path), extract_images=False) - result = loader.lazy_load() - assert isinstance(result, Iterator) - - # Consuming the iterator should yield documents - docs = list(result) - assert len(docs) > 0 - - -def test_safe_pdf_loader_load_delegates_to_lazy_load(tmp_path): - """SafePyPDFLoader.load() should produce the same results as list(lazy_load()).""" - pdf_path = tmp_path / "delegate_test.pdf" - _make_pdf(str(pdf_path)) - - loader1 = SafePyPDFLoader(str(pdf_path), extract_images=False) - loader2 = SafePyPDFLoader(str(pdf_path), extract_images=False) - - load_docs = loader1.load() - lazy_docs = list(loader2.lazy_load()) - - assert len(load_docs) == len(lazy_docs) - for ld, lz in zip(load_docs, lazy_docs): - assert ld.page_content == lz.page_content - - # --------------------------------------------------------------------------- # CSV with non-UTF-8 encoding # --------------------------------------------------------------------------- @@ -486,37 +363,6 @@ def _measure_lazy_load_streaming(loader_factory): return texts, peak -class TestMemoryBenchmarkPDF: - """Memory comparison for PyPDFLoader -- the loader most likely to benefit - from lazy_load() since it yields page-by-page.""" - - NUM_PAGES = 50 - - @pytest.fixture() - def pdf_path(self, tmp_path): - path = tmp_path / "bench.pdf" - _make_pdf(str(path), num_pages=self.NUM_PAGES) - return str(path) - - def test_pdf_streaming_lazy_load_peak_memory(self, pdf_path): - """Streaming lazy_load() should use <= peak memory vs load().""" - - def factory(): - return SafePyPDFLoader(pdf_path, extract_images=False) - - load_docs, peak_load = _measure_load(factory) - texts, peak_lazy_stream = _measure_lazy_load_streaming(factory) - - assert len(load_docs) == self.NUM_PAGES - assert len(texts) == self.NUM_PAGES - - # Streaming should not use MORE memory than eager (allow 10% noise) - assert peak_lazy_stream <= peak_load * 1.10, ( - f"streaming peak ({peak_lazy_stream:,} B) exceeded " - f"load() peak ({peak_load:,} B) by >10%" - ) - - class TestMemoryBenchmarkCSV: """Memory comparison for CSVLoader -- yields one Document per row.""" @@ -571,11 +417,6 @@ def factory(): "application/vnd.openxmlformats-officedocument.presentationml.presentation", id="pptx", ), - pytest.param( - "test.docx", - "application/vnd.openxmlformats-officedocument.wordprocessingml.document", - id="docx", - ), ] # Map extensions to their file creators @@ -595,7 +436,6 @@ def factory(): ".epub": lambda p: _make_epub(p), ".xlsx": lambda p: _make_xlsx(p), ".pptx": lambda p: _make_pptx(p), - ".docx": lambda p: _make_docx(p), } From 1eb1ccc17cd3ec5bd36415fbfe84812e9911c682 Mon Sep 17 00:00:00 2001 From: Lia Date: Fri, 23 Oct 2026 10:51:02 +0000 Subject: [PATCH 6/7] =?UTF-8?q?=F0=9F=94=8C=20fix:=20Keep=20Shared=20Proce?= =?UTF-8?q?ssor=20Hardening=20Atomic=20in=20Compose?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- api-compose.yaml | 2 +- docker-compose.yaml | 2 +- tests/extraction/test_deployment.py | 36 +++++++++++++++++++++++++---- 3 files changed, 33 insertions(+), 7 deletions(-) diff --git a/api-compose.yaml b/api-compose.yaml index fee0f1a0..22782927 100644 --- a/api-compose.yaml +++ b/api-compose.yaml @@ -1,7 +1,7 @@ version: '3.8' services: - extraction: + extraction: !override extends: file: extraction-compose.yaml service: extraction diff --git a/docker-compose.yaml b/docker-compose.yaml index e5bc27a9..1c4d8b2f 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -1,5 +1,5 @@ services: - extraction: + extraction: !override extends: file: extraction-compose.yaml service: extraction diff --git a/tests/extraction/test_deployment.py b/tests/extraction/test_deployment.py index 9bb8d35b..9dc50495 100644 --- a/tests/extraction/test_deployment.py +++ b/tests/extraction/test_deployment.py @@ -36,9 +36,12 @@ def render(directory, *files): [*command, "config", "--no-env-resolution", "--format", "json"], capture_output=True, text=True, - check=True, timeout=15, ) + if result.returncode != 0: + raise AssertionError( + f"Compose render failed ({result.returncode}):\n{result.stderr}" + ) return json.loads(result.stdout) @@ -97,10 +100,15 @@ def test_standard_deployment_requires_one_hardened_processor(tmp_path, recipe, c ) -def test_combined_recipes_share_one_processor_and_socket(tmp_path): - config = render( - stage_compose_files(tmp_path), "docker-compose.yaml", "api-compose.yaml" - ) +@pytest.mark.parametrize( + "recipes", + [ + ("docker-compose.yaml", "api-compose.yaml"), + ("api-compose.yaml", "docker-compose.yaml"), + ], +) +def test_combined_recipes_share_one_processor_and_socket(tmp_path, recipes): + config = render(stage_compose_files(tmp_path), *recipes) assert set(config["services"]) == {"db", "fastapi", "rag_api", "extraction"} assert_hardened_processor(config["services"]["extraction"]) for caller in ("fastapi", "rag_api"): @@ -110,3 +118,21 @@ def test_combined_recipes_share_one_processor_and_socket(tmp_path): if item["target"] == "/socket" ) assert mount["source"] == "extraction-socket" and mount["read_only"] is True + + +def test_render_failure_includes_stderr_without_dumping_model(monkeypatch, tmp_path): + diagnostic = "services.extraction.security_opt items at 0 and 1 are equal" + monkeypatch.setattr( + subprocess, + "run", + lambda command, **kwargs: subprocess.CompletedProcess( + command, + 1, + stdout="do not dump the rendered application model", + stderr=diagnostic, + ), + ) + with pytest.raises(AssertionError, match="security_opt") as caught: + render(tmp_path, "docker-compose.yaml", "api-compose.yaml") + assert diagnostic in str(caught.value) + assert "do not dump" not in str(caught.value) From 812264158c4c709670ffb6340926b1ed63215130 Mon Sep 17 00:00:00 2001 From: Lia Date: Fri, 23 Oct 2026 13:36:01 +0000 Subject: [PATCH 7/7] =?UTF-8?q?=F0=9F=94=8C=20feat:=20Provision=20Authenti?= =?UTF-8?q?cated=20HTTP=20Extraction?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- api-compose.yaml | 16 +- app/services/extraction_client.py | 153 +++++++-- docker-compose.yaml | 16 +- extraction-compose.yaml | 33 +- tests/extraction/conftest.py | 78 +++-- tests/extraction/test_client.py | 26 +- tests/extraction/test_deployment.py | 121 ++++--- tests/extraction/test_http.py | 364 ++++++++++++++++++++++ tests/extraction/test_processor.py | 163 ++++++++-- tests/extraction/test_routes.py | 67 +++- tests/extraction/test_upload_transport.py | 135 ++++++++ 11 files changed, 1045 insertions(+), 127 deletions(-) create mode 100644 tests/extraction/test_http.py create mode 100644 tests/extraction/test_upload_transport.py diff --git a/api-compose.yaml b/api-compose.yaml index 22782927..02b8f515 100644 --- a/api-compose.yaml +++ b/api-compose.yaml @@ -1,6 +1,11 @@ version: '3.8' services: + extraction-init: !override + extends: + file: extraction-compose.yaml + service: extraction-init + extraction: !override extends: file: extraction-compose.yaml @@ -8,8 +13,7 @@ services: rag_api: build: . - group_add: - - '10001' + networks: [default, extraction] depends_on: extraction: condition: service_healthy @@ -17,10 +21,14 @@ services: - "8000:8000" volumes: - ./uploads:/app/uploads - - extraction-socket:/socket:ro + - extraction-credentials:/run/extraction:ro env_file: - .env volumes: - extraction-socket: + extraction-credentials: pgdata2: + +networks: + extraction: + internal: true diff --git a/app/services/extraction_client.py b/app/services/extraction_client.py index 20b4841d..2bbf5b8f 100644 --- a/app/services/extraction_client.py +++ b/app/services/extraction_client.py @@ -1,4 +1,4 @@ -"""Bounded v1 extraction transport. Only caller-owned bytes cross the socket.""" +"""Bounded v1 extraction transport. Only caller-owned bytes and service auth cross HTTP.""" import asyncio import hashlib @@ -6,7 +6,10 @@ import math import os import re +import ssl +import stat from dataclasses import dataclass +from urllib.parse import urlsplit import aiofiles @@ -18,6 +21,7 @@ CHUNK_BYTES = 64 * 1024 OPERATION = "document.extract-text" ERROR_STATUS = { + "UNAUTHORIZED": 401, "INVALID_REQUEST": 400, "INPUT_LIMIT": 413, "OUTPUT_LIMIT": 413, @@ -48,20 +52,65 @@ def __init__(self, code: str, status_code: int = None): super().__init__(code) +def _endpoint(url): + try: + if ( + not isinstance(url, str) + or not 1 <= len(url) <= 2048 + or any(ord(char) <= 32 or ord(char) >= 127 for char in url) + or "?" in url + or "#" in url + or "\\" in url + ): + raise ValueError() + endpoint = urlsplit(url) + if ( + endpoint.scheme not in {"http", "https"} + or not endpoint.hostname + or endpoint.username is not None + or endpoint.password is not None + or endpoint.path not in {"", "/"} + or endpoint.netloc.endswith(":") + or not 1 + <= (endpoint.port or (443 if endpoint.scheme == "https" else 80)) + <= 65535 + or endpoint.port == 0 + or not re.fullmatch(r"[a-zA-Z0-9.:-]+", endpoint.hostname) + ): + raise ValueError() + authority = ( + f"[{endpoint.hostname}]" if ":" in endpoint.hostname else endpoint.hostname + ) + if endpoint.port is not None: + authority += f":{endpoint.port}" + if endpoint.netloc.lower() != authority: + raise ValueError() + if endpoint.scheme == "http" and not ( + endpoint.hostname in {"localhost", "127.0.0.1", "::1"} + or (endpoint.hostname == "extraction" and endpoint.port == 8000) + ): + raise ValueError() + return endpoint + except ValueError: + raise ValueError( + "RAG_EXTRACTION_URL must be an HTTPS origin or local HTTP extraction origin" + ) from None + + @dataclass(frozen=True) class ExtractionSettings: - socket: str = "/socket/extraction.sock" + url: str = "http://extraction:8000" + token_file: str = "/run/extraction/token" deadline: float = 10.0 def __post_init__(self): + _endpoint(self.url) if ( - not isinstance(self.socket, str) - or not os.path.isabs(self.socket) - or "\x00" in self.socket + not isinstance(self.token_file, str) + or not os.path.isabs(self.token_file) + or "\x00" in self.token_file ): - raise ValueError( - "RAG_EXTRACTION_SOCKET must be an absolute Unix socket path" - ) + raise ValueError("RAG_EXTRACTION_TOKEN_FILE must be an absolute file path") if ( type(self.deadline) not in (int, float) or not math.isfinite(self.deadline) @@ -71,17 +120,44 @@ def __post_init__(self): @classmethod def from_env(cls): - for key in ("RAG_EXTRACTION_ENABLED", "RAG_EXTRACTION_FORMATS"): + for key in ( + "RAG_EXTRACTION_ENABLED", + "RAG_EXTRACTION_FORMATS", + "RAG_EXTRACTION_SOCKET", + ): if key in os.environ: raise ValueError( - f"{key} is obsolete; PDF/DOCX service extraction is required" + f"{key} is obsolete; authenticated PDF/DOCX service extraction is required" ) + try: + deadline = float(os.getenv("RAG_EXTRACTION_DEADLINE_SECONDS", "10")) + except ValueError: + raise ValueError( + "RAG_EXTRACTION_DEADLINE_SECONDS must be in (0, 10]" + ) from None return cls( - socket=os.getenv("RAG_EXTRACTION_SOCKET", "/socket/extraction.sock"), - deadline=float(os.getenv("RAG_EXTRACTION_DEADLINE_SECONDS", "10")), + url=os.getenv("RAG_EXTRACTION_URL", "http://extraction:8000"), + token_file=os.getenv("RAG_EXTRACTION_TOKEN_FILE", "/run/extraction/token"), + deadline=deadline, ) +def _read_token(path): + descriptor = os.open( + path, os.O_RDONLY | os.O_NONBLOCK | os.O_NOFOLLOW | os.O_CLOEXEC + ) + try: + metadata = os.fstat(descriptor) + if not stat.S_ISREG(metadata.st_mode) or metadata.st_size != 64: + raise ExtractionError("UNAVAILABLE") + token = os.read(descriptor, 65) + if not re.fullmatch(b"[0-9a-f]{64}", token): + raise ExtractionError("UNAVAILABLE") + return token.decode("ascii") + finally: + os.close(descriptor) + + def _object(pairs): result = {} for key, value in pairs: @@ -207,12 +283,27 @@ class ExtractionClient: def __init__(self, settings: ExtractionSettings): self.settings = settings self._admitted = 0 + self._endpoint = _endpoint(settings.url) + self._ssl = ( + ssl.create_default_context() if self._endpoint.scheme == "https" else None + ) async def _request( - self, method: str, *, file=None, size=0, format=None, maximum=CAPABILITY_BYTES + self, + method: str, + token: str, + *, + file=None, + size=0, + format=None, + maximum=CAPABILITY_BYTES, ): - reader, writer = await asyncio.open_unix_connection( - self.settings.socket, limit=HEADER_BYTES + reader, writer = await asyncio.open_connection( + self._endpoint.hostname, + self._endpoint.port or (443 if self._ssl else 80), + ssl=self._ssl, + server_hostname=self._endpoint.hostname if self._ssl else None, + limit=HEADER_BYTES, ) # At most one chunk may be queued without drain, including on cancellation. writer.transport.set_write_buffer_limits(high=CHUNK_BYTES, low=CHUNK_BYTES // 2) @@ -220,7 +311,8 @@ async def _request( try: path = "/v1/extract-text" if file is not None else "/v1/capabilities" headers = ( - f"{method} {path} HTTP/1.1\r\nHost: extraction\r\nConnection: close\r\n" + f"{method} {path} HTTP/1.1\r\nHost: {self._endpoint.netloc}\r\n" + f"Authorization: Bearer {token}\r\nConnection: close\r\n" ) if file is not None: headers += ( @@ -283,12 +375,31 @@ async def _request( and type(value["error"]["code"]) is str and ERROR_STATUS.get(value["error"]["code"]) == status ): - raise ExtractionError(value["error"]["code"]) + code = value["error"]["code"] + # Service authentication is not the RAG caller's authentication. + raise ExtractionError( + "UNAVAILABLE" if code == "UNAUTHORIZED" else code + ) raise ExtractionError("INVALID_RESPONSE") return value, digest.hexdigest() finally: # Abort instead of flushing pending input on timeout/cancellation. writer.transport.abort() + closed = asyncio.create_task(writer.wait_closed()) + cancelled = False + while not closed.done(): + try: + await asyncio.shield(closed) + except asyncio.CancelledError: + cancelled = True + except OSError: + break + try: + closed.result() + except OSError: + pass + if cancelled: + raise asyncio.CancelledError() async def extract(self, file_path: str, format: str): if format not in {"pdf", "docx"}: @@ -302,11 +413,15 @@ async def extract(self, file_path: str, format: str): stat = await asyncio.to_thread(os.fstat, file.fileno()) if not 0 < stat.st_size <= INPUT_BYTES: raise ExtractionError("INPUT_LIMIT") - # No cache: each job checks the current socket's contract before upload. - capabilities, _ = await self._request("GET") + token = await asyncio.to_thread( + _read_token, self.settings.token_file + ) + # No cache: authenticate and check the current service before upload. + capabilities, _ = await self._request("GET", token) limits = validate_capabilities(capabilities, format, stat.st_size) result, digest = await self._request( "POST", + token, file=file, size=stat.st_size, format=format, diff --git a/docker-compose.yaml b/docker-compose.yaml index 1c4d8b2f..37700811 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -1,4 +1,9 @@ services: + extraction-init: !override + extends: + file: extraction-compose.yaml + service: extraction-init + extraction: !override extends: file: extraction-compose.yaml @@ -17,8 +22,7 @@ services: fastapi: build: . - group_add: - - '10001' + networks: [default, extraction] environment: - DB_HOST=db - DB_PORT=5432 @@ -27,7 +31,7 @@ services: - "8000:8000" volumes: - ./uploads:/app/uploads - - extraction-socket:/socket:ro + - extraction-credentials:/run/extraction:ro depends_on: db: condition: service_started @@ -37,5 +41,9 @@ services: - .env volumes: - extraction-socket: + extraction-credentials: pgdata2: + +networks: + extraction: + internal: true diff --git a/extraction-compose.yaml b/extraction-compose.yaml index bad0e713..d4cbf9ce 100644 --- a/extraction-compose.yaml +++ b/extraction-compose.yaml @@ -1,19 +1,42 @@ services: - extraction: - build: - context: https://github.com/LibreChat-AI/code-interpreter.git#77bcd2ec2e63be523bfab2dfa11159a2ceec0088 + extraction-init: + build: &extraction-build + context: https://github.com/LibreChat-AI/code-interpreter.git#c7bfe421243c66d4d5d9c6e3d08d276586ac47bf dockerfile: packages/extraction/Dockerfile target: runtime - init: true + command: ['node', 'dist/provision.js'] + healthcheck: + disable: true user: '10001:10001' read_only: true cap_drop: [ALL] security_opt: [no-new-privileges:true] network_mode: none + mem_limit: 128m + cpus: 0.25 + pids_limit: 16 + environment: + EXTRACTION_TOKEN_FILE: /run/extraction/token + volumes: + - extraction-credentials:/run/extraction + + extraction: + build: *extraction-build + init: true + user: '10001:10001' + read_only: true + cap_drop: [ALL] + security_opt: [no-new-privileges:true] + networks: [extraction] + depends_on: + extraction-init: + condition: service_completed_successfully + environment: + EXTRACTION_TOKEN_FILE: /run/extraction/token mem_limit: 768m cpus: 2 pids_limit: 32 tmpfs: - /jobs:rw,noexec,nosuid,nodev,size=96m,nr_inodes=1024,uid=10001,gid=10001,mode=0700 volumes: - - extraction-socket:/socket + - extraction-credentials:/run/extraction:ro diff --git a/tests/extraction/conftest.py b/tests/extraction/conftest.py index cb8b52f6..4d7b06cb 100644 --- a/tests/extraction/conftest.py +++ b/tests/extraction/conftest.py @@ -2,9 +2,7 @@ import copy import hashlib import json -import os -import tempfile -from pathlib import Path +import subprocess import pytest @@ -34,14 +32,19 @@ class Processor: - def __init__(self, socket): - self.socket = socket + def __init__(self, token_file): + self.url = None + self.token_file = str(token_file) + self.token = "ab" * 32 + token_file.write_text(self.token) self.capabilities = copy.deepcopy(CAPABILITIES) self.requests = [] self.mutate = lambda value: value self.raw = None self.error = None self.hold = False + self.hold_caps = False + self.caps_entered = asyncio.Event() self.before_response = None self.entered = asyncio.Event() self.closed = asyncio.Event() @@ -50,7 +53,9 @@ def __init__(self, socket): def client(self, deadline=10): return ExtractionClient( - ExtractionSettings(socket=self.socket, deadline=deadline) + ExtractionSettings( + url=self.url, token_file=self.token_file, deadline=deadline + ) ) async def handle(self, reader, writer): @@ -64,7 +69,15 @@ async def handle(self, reader, writer): length = int(fields.get("Content-Length", "0")) body = await reader.readexactly(length) self.requests.append((method, path, fields, body)) - if path == "/v1/capabilities": + if fields.get("Authorization") != f"Bearer {self.token}": + value = {"error": {"code": "UNAUTHORIZED"}} + status = 401 + elif path == "/v1/capabilities": + self.caps_entered.set() + if self.hold_caps: + await reader.read() + self.closed.set() + return value = self.capabilities status = 200 else: @@ -125,18 +138,15 @@ async def handle(self, reader, writer): @pytest.fixture -async def processor(): - # Keep AF_UNIX path below 108 bytes even in long worktree/pytest directories. - with tempfile.TemporaryDirectory( - prefix="rag-", dir=os.environ.get("TMPDIR") - ) as directory: - service = Processor(str(Path(directory, "s"))) - server = await asyncio.start_unix_server(service.handle, path=service.socket) - async with server: - yield service - for task in list(service.active): - task.cancel() - await asyncio.gather(*service.active, return_exceptions=True) +async def processor(tmp_path): + service = Processor(tmp_path / "token") + server = await asyncio.start_server(service.handle, host="127.0.0.1", port=0) + service.url = f"http://127.0.0.1:{server.sockets[0].getsockname()[1]}" + async with server: + yield service + for task in list(service.active): + task.cancel() + await asyncio.gather(*service.active, return_exceptions=True) @pytest.fixture @@ -144,3 +154,33 @@ def input_file(tmp_path): file = tmp_path / "input" file.write_bytes(b"caller-owned opaque document bytes") return str(file) + + +@pytest.fixture(scope="session") +def tls_certificate(tmp_path_factory): + directory = tmp_path_factory.mktemp("extraction-tls") + certificate, key = directory / "certificate.pem", directory / "key.pem" + subprocess.run( + [ + "openssl", + "req", + "-x509", + "-newkey", + "rsa:2048", + "-nodes", + "-days", + "1", + "-keyout", + str(key), + "-out", + str(certificate), + "-subj", + "/CN=localhost", + "-addext", + "subjectAltName=DNS:localhost", + ], + check=True, + capture_output=True, + timeout=15, + ) + return certificate, key diff --git a/tests/extraction/test_client.py b/tests/extraction/test_client.py index d5ff29fe..5fbd403b 100644 --- a/tests/extraction/test_client.py +++ b/tests/extraction/test_client.py @@ -25,7 +25,8 @@ async def test_raw_upload_and_valid_segments(processor, input_file, format): assert (method, path) == ("POST", "/v1/extract-text") assert body == Path(input_file).read_bytes() assert headers == { - "Host": "extraction", + "Host": processor.url.removeprefix("http://"), + "Authorization": f"Bearer {processor.token}", "Connection": "close", "Content-Length": str(len(body)), "Content-Type": "application/octet-stream", @@ -197,8 +198,12 @@ async def test_capabilities_rechecked_after_service_change(processor, input_file assert [r[0] for r in processor.requests] == ["GET", "POST", "GET"] -async def test_outage_is_safe(input_file, tmp_path): - client = ExtractionClient(ExtractionSettings(socket=str(tmp_path / "missing"))) +async def test_outage_is_safe(input_file, processor, monkeypatch): + async def unavailable(*args, **kwargs): + raise ConnectionRefusedError("private endpoint") + + monkeypatch.setattr(asyncio, "open_connection", unavailable) + client = processor.client() with pytest.raises(ExtractionError, match="UNAVAILABLE"): await client.extract(input_file, "pdf") @@ -208,17 +213,22 @@ def test_default_settings_require_service_without_environment(monkeypatch): "RAG_EXTRACTION_ENABLED", "RAG_EXTRACTION_FORMATS", "RAG_EXTRACTION_SOCKET", + "RAG_EXTRACTION_URL", + "RAG_EXTRACTION_TOKEN_FILE", "RAG_EXTRACTION_DEADLINE_SECONDS", ): monkeypatch.delenv(key, raising=False) settings = ExtractionSettings.from_env() - assert settings.socket == "/socket/extraction.sock" and settings.deadline == 10 + assert settings.url == "http://extraction:8000" and settings.deadline == 10 + assert settings.token_file == "/run/extraction/token" assert not hasattr(settings, "enabled") and not hasattr(settings, "formats") @pytest.mark.parametrize( "key,value", [ + ("RAG_EXTRACTION_SOCKET", "/socket/old.sock"), + ("RAG_EXTRACTION_SOCKET", ""), ("RAG_EXTRACTION_ENABLED", "false"), ("RAG_EXTRACTION_ENABLED", "true"), ("RAG_EXTRACTION_ENABLED", ""), @@ -236,8 +246,9 @@ def test_obsolete_opt_out_configuration_is_rejected(monkeypatch, key, value): @pytest.mark.parametrize( "kwargs", [ - {"socket": "relative"}, - {"socket": ""}, + {"token_file": "relative"}, + {"token_file": ""}, + {"token_file": "/bad\x00path"}, {"deadline": 0}, {"deadline": 11}, {"deadline": True}, @@ -253,7 +264,8 @@ def test_required_config_is_validated(kwargs): @pytest.mark.parametrize( "key,value", [ - ("RAG_EXTRACTION_SOCKET", "relative"), + ("RAG_EXTRACTION_URL", "relative"), + ("RAG_EXTRACTION_TOKEN_FILE", "relative"), ("RAG_EXTRACTION_DEADLINE_SECONDS", "stale"), ], ) diff --git a/tests/extraction/test_deployment.py b/tests/extraction/test_deployment.py index 9dc50495..bd93ebac 100644 --- a/tests/extraction/test_deployment.py +++ b/tests/extraction/test_deployment.py @@ -9,7 +9,7 @@ import pytest ROOT = Path(__file__).resolve().parents[2] -PROCESSOR_REF = "77bcd2ec2e63be523bfab2dfa11159a2ceec0088" +PROCESSOR_REF = "c7bfe421243c66d4d5d9c6e3d08d276586ac47bf" PROCESSOR_CONTEXT = ( f"https://github.com/LibreChat-AI/code-interpreter.git#{PROCESSOR_REF}" ) @@ -45,59 +45,102 @@ def render(directory, *files): return json.loads(result.stdout) -def assert_hardened_processor(service): +def assert_build(service): assert service["build"]["context"] == PROCESSOR_CONTEXT assert service["build"]["dockerfile"] == "packages/extraction/Dockerfile" assert service["build"]["target"] == "runtime" - assert service["init"] is True and service["user"] == "10001:10001" + + +def assert_credentials_mount(service, readonly): + mount = next( + item for item in service["volumes"] if item["target"] == "/run/extraction" + ) + assert mount["type"] == "volume" and mount["source"] == "extraction-credentials" + assert mount.get("read_only", False) is readonly + + +def assert_sandbox(service): assert service["read_only"] is True assert service["cap_drop"] == ["ALL"] assert service["security_opt"] == ["no-new-privileges:true"] - assert service["network_mode"] == "none" - assert int(service["mem_limit"]) == 768 * 1024 * 1024 - assert float(service["cpus"]) == 2 and int(service["pids_limit"]) == 32 - assert service["tmpfs"] == [ - "/jobs:rw,noexec,nosuid,nodev,size=96m,nr_inodes=1024,uid=10001,gid=10001,mode=0700" - ] - assert len(service["volumes"]) == 1 - mount = service["volumes"][0] - assert mount["type"] == "volume" and mount["source"] == "extraction-socket" - assert mount["target"] == "/socket" and not mount.get("read_only", False) for key in ( - "environment", "env_file", "ports", "expose", "profiles", "privileged", "cap_add", - "command", "entrypoint", + "container_name", + "group_add", ): assert not service.get(key), key +def assert_hardened_processor(service): + assert_build(service) + assert_sandbox(service) + assert service["init"] is True and service["user"] == "10001:10001" + assert set(service["networks"]) == {"extraction"} + assert "network_mode" not in service + assert ( + service["depends_on"]["extraction-init"]["condition"] + == "service_completed_successfully" + ) + assert not service.get("healthcheck") + assert service["environment"] == {"EXTRACTION_TOKEN_FILE": "/run/extraction/token"} + assert int(service["mem_limit"]) == 768 * 1024 * 1024 + assert float(service["cpus"]) == 2 and int(service["pids_limit"]) == 32 + assert service["tmpfs"] == [ + "/jobs:rw,noexec,nosuid,nodev,size=96m,nr_inodes=1024,uid=10001,gid=10001,mode=0700" + ] + assert len(service["volumes"]) == 1 + assert_credentials_mount(service, True) + assert not service.get("command") + + +def assert_provisioning(config): + service = config["services"]["extraction-init"] + assert_build(service) + assert_sandbox(service) + assert service["build"] == config["services"]["extraction"]["build"] + assert service["command"] == ["node", "dist/provision.js"] + assert service["user"] == "10001:10001" and service["network_mode"] == "none" + assert service["healthcheck"] == {"disable": True} + assert int(service["mem_limit"]) == 128 * 1024 * 1024 + assert float(service["cpus"]) == 0.25 and int(service["pids_limit"]) == 16 + assert service["environment"] == {"EXTRACTION_TOKEN_FILE": "/run/extraction/token"} + assert len(service["volumes"]) == 1 + assert_credentials_mount(service, False) + assert config["networks"]["extraction"]["internal"] is True + assert "extraction-credentials" in config["volumes"] + assert "extraction-socket" not in config["volumes"] + + +def assert_caller(app): + assert not app.get("group_add") and not app.get("profiles") + assert set(app["networks"]) == {"default", "extraction"} + assert app["depends_on"]["extraction"]["condition"] == "service_healthy" + assert_credentials_mount(app, True) + assert ( + not { + "RAG_EXTRACTION_ENABLED", + "RAG_EXTRACTION_FORMATS", + "RAG_EXTRACTION_SOCKET", + } + & app.get("environment", {}).keys() + ) + + @pytest.mark.parametrize( "recipe,caller", [("docker-compose.yaml", "fastapi"), ("api-compose.yaml", "rag_api")], ) -def test_standard_deployment_requires_one_hardened_processor(tmp_path, recipe, caller): +def test_standard_deployment_provisions_hardened_processor(tmp_path, recipe, caller): config = render(stage_compose_files(tmp_path), recipe) assert_hardened_processor(config["services"]["extraction"]) - app = config["services"][caller] - assert "10001" in app["group_add"] - assert app["depends_on"]["extraction"]["condition"] == "service_healthy" - mount = next(item for item in app["volumes"] if item["target"] == "/socket") - assert ( - mount["type"] == "volume" - and mount["source"] == "extraction-socket" - and mount["read_only"] is True - ) - assert not app.get("profiles") - assert ( - not {"RAG_EXTRACTION_ENABLED", "RAG_EXTRACTION_FORMATS"} - & app.get("environment", {}).keys() - ) + assert_provisioning(config) + assert_caller(config["services"][caller]) @pytest.mark.parametrize( @@ -107,17 +150,19 @@ def test_standard_deployment_requires_one_hardened_processor(tmp_path, recipe, c ("api-compose.yaml", "docker-compose.yaml"), ], ) -def test_combined_recipes_share_one_processor_and_socket(tmp_path, recipes): +def test_combined_recipes_share_provisioning_and_processor(tmp_path, recipes): config = render(stage_compose_files(tmp_path), *recipes) - assert set(config["services"]) == {"db", "fastapi", "rag_api", "extraction"} + assert set(config["services"]) == { + "db", + "fastapi", + "rag_api", + "extraction-init", + "extraction", + } assert_hardened_processor(config["services"]["extraction"]) + assert_provisioning(config) for caller in ("fastapi", "rag_api"): - mount = next( - item - for item in config["services"][caller]["volumes"] - if item["target"] == "/socket" - ) - assert mount["source"] == "extraction-socket" and mount["read_only"] is True + assert_caller(config["services"][caller]) def test_render_failure_includes_stderr_without_dumping_model(monkeypatch, tmp_path): diff --git a/tests/extraction/test_http.py b/tests/extraction/test_http.py new file mode 100644 index 00000000..4fa0164d --- /dev/null +++ b/tests/extraction/test_http.py @@ -0,0 +1,364 @@ +import asyncio +import os +import ssl +import threading +from pathlib import Path + +import pytest + +from app.services import extraction_client +from app.services.extraction_client import ExtractionError, ExtractionSettings + + +@pytest.mark.parametrize( + "url", + [ + "http://extraction:8000", + "http://extraction:8000/", + "http://localhost:1234", + "http://127.0.0.1:1234", + "http://[::1]:1234", + "https://processor.example", + "https://processor.example:8443/", + "https://[2001:db8::1]:8443", + ], +) +def test_valid_origins(url): + assert ExtractionSettings(url=url).url == url + + +@pytest.mark.parametrize( + "url", + [ + None, + "", + "extraction:8000", + "unix:///socket/extraction.sock", + "file:///run/extraction/token", + "ftp://extraction:8000", + "http://extraction", + "http://extraction:8001", + "http://remote.example:8000", + "http://10.0.0.1:8000", + "http://192.168.1.2:8000", + "http://127.0.0.2:8000", + "http://localhost.evil:8000", + "https://user:secret@processor.example", + "https://secret@processor.example", + "https://processor.example/path", + "https://processor.example//", + "https://processor.example?token=secret", + "https://processor.example?", + "https://processor.example#secret", + "https://processor.example#", + "https://processor.example:0", + "https://processor.example:65536", + "https://processor.example:secret", + "https://processor.example:", + "https://[broken", + "https://[::1]ignored", + "https://[::1]:0443", + "https://[v1.localhost]", + "https://processor.example:0443", + "https://processor.example\\@localhost", + " https://processor.example", + "https://processor.example\n", + "https://processor.example\x00", + "https://processor.example\t:443", + "https://procéssor.example", + ], +) +def test_invalid_origin_is_safe(url): + with pytest.raises(ValueError) as caught: + ExtractionSettings(url=url) + assert "secret" not in str(caught.value) + assert "processor.example" not in str(caught.value) + + +@pytest.mark.parametrize("phase", ["capabilities", "extract"]) +async def test_both_operations_require_service_auth(processor, input_file, phase): + if phase == "capabilities": + Path(processor.token_file).write_text("cd" * 32) + else: + processor.error = (401, "UNAUTHORIZED") + with pytest.raises(ExtractionError) as caught: + await processor.client().extract(input_file, "pdf") + assert (caught.value.code, caught.value.status_code) == ("UNAVAILABLE", 503) + assert len(processor.requests) == (1 if phase == "capabilities" else 2) + + +@pytest.mark.parametrize( + "token", + [b"", b"a" * 63, b"a" * 65, b"a" * 64 + b"\n", b"A" * 64, b"g" * 64, b"\xff" * 64], +) +async def test_invalid_credentials_never_connect(processor, input_file, token): + Path(processor.token_file).write_bytes(token) + with pytest.raises(ExtractionError) as caught: + await processor.client().extract(input_file, "pdf") + assert (caught.value.code, caught.value.status_code) == ("UNAVAILABLE", 503) + assert not processor.requests + + +@pytest.mark.parametrize( + "kind", ["missing", "directory", "fifo", "symlink", "oversized"] +) +async def test_nonregular_or_missing_credentials_fail_closed( + processor, input_file, kind +): + path = Path(processor.token_file) + path.unlink() + if kind == "directory": + path.mkdir() + elif kind == "fifo": + os.mkfifo(path) + elif kind == "symlink": + target = path.with_name("other-token") + target.write_text(processor.token) + path.symlink_to(target) + elif kind == "oversized": + with path.open("wb") as file: + file.truncate(1024 * 1024 * 1024) + with pytest.raises(ExtractionError, match="UNAVAILABLE"): + await asyncio.wait_for(processor.client().extract(input_file, "pdf"), 1) + assert not processor.requests + + +async def test_token_read_is_bounded_nonblocking_and_once_per_job( + processor, input_file, monkeypatch +): + original_open, original_read = os.open, os.read + opens, reads = [], [] + + def capture_open(path, flags, *args, **kwargs): + descriptor = original_open(path, flags, *args, **kwargs) + if str(path) == processor.token_file: + opens.append((descriptor, flags)) + return descriptor + + def capture_read(descriptor, size): + if opens and descriptor == opens[-1][0]: + reads.append(size) + return original_read(descriptor, size) + + monkeypatch.setattr(os, "open", capture_open) + monkeypatch.setattr(os, "read", capture_read) + await processor.client().extract(input_file, "pdf") + assert len(opens) == 1 and reads == [65] + assert opens[0][1] & os.O_NONBLOCK and opens[0][1] & os.O_NOFOLLOW + with pytest.raises(OSError): + os.fstat(opens[0][0]) + + +async def test_token_rotation_is_visible_on_next_job(processor, input_file): + client = processor.client() + await client.extract(input_file, "pdf") + replacement = Path(processor.token_file).with_name("rotated-token") + processor.token = "cd" * 32 + replacement.write_text(processor.token) + replacement.replace(processor.token_file) + await client.extract(input_file, "pdf") + assert [request[2]["Authorization"] for request in processor.requests] == [ + "Bearer " + "ab" * 32, + "Bearer " + "ab" * 32, + "Bearer " + "cd" * 32, + "Bearer " + "cd" * 32, + ] + + +async def test_proxy_environment_is_not_used(processor, input_file, monkeypatch): + for key in ( + "HTTP_PROXY", + "HTTPS_PROXY", + "ALL_PROXY", + "http_proxy", + "https_proxy", + "all_proxy", + ): + monkeypatch.setenv(key, "http://secret@127.0.0.1:1") + await processor.client().extract(input_file, "pdf") + assert len(processor.requests) == 2 + + +async def test_credential_read_is_inside_deadline(processor, input_file, monkeypatch): + entered, release, finished = threading.Event(), threading.Event(), threading.Event() + read = extraction_client._read_token + + def delayed_read(path): + entered.set() + try: + release.wait(2) + return read(path) + finally: + finished.set() + + monkeypatch.setattr(extraction_client, "_read_token", delayed_read) + client = processor.client(deadline=0.05) + try: + with pytest.raises(ExtractionError, match="DEADLINE"): + await client.extract(input_file, "pdf") + assert entered.is_set() and not processor.requests and client._admitted == 0 + finally: + release.set() + assert await asyncio.to_thread(finished.wait, 1) + + +async def test_connect_is_inside_deadline(processor, input_file, monkeypatch): + entered, closed = asyncio.Event(), asyncio.Event() + + async def delayed_connect(*args, **kwargs): + entered.set() + try: + await asyncio.Future() + finally: + closed.set() + + monkeypatch.setattr(asyncio, "open_connection", delayed_connect) + with pytest.raises(ExtractionError, match="DEADLINE"): + await processor.client(deadline=0.05).extract(input_file, "pdf") + assert entered.is_set() and closed.is_set() and not processor.requests + + +async def test_capabilities_are_inside_deadline(processor, input_file): + processor.hold_caps = True + with pytest.raises(ExtractionError, match="DEADLINE"): + await processor.client(deadline=0.05).extract(input_file, "pdf") + await asyncio.wait_for(processor.closed.wait(), 1) + assert processor.caps_entered.is_set() and len(processor.requests) == 1 + + +@pytest.fixture +async def tls_processor(processor, tls_certificate): + certificate, key = tls_certificate + context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + context.load_cert_chain(certificate, key) + server = await asyncio.start_server(processor.handle, "127.0.0.1", 0, ssl=context) + processor.url = f"https://localhost:{server.sockets[0].getsockname()[1]}" + async with server: + yield processor + + +async def test_https_uses_standard_trust_and_hostname( + tls_processor, tls_certificate, input_file, monkeypatch +): + monkeypatch.setenv("SSL_CERT_FILE", str(tls_certificate[0])) + client = tls_processor.client() + assert client._ssl.verify_mode == ssl.CERT_REQUIRED and client._ssl.check_hostname + await client.extract(input_file, "pdf") + assert len(tls_processor.requests) == 2 + + +@pytest.mark.parametrize("failure", ["untrusted", "hostname"]) +async def test_https_never_skips_verification( + tls_processor, tls_certificate, input_file, monkeypatch, failure +): + if failure == "hostname": + monkeypatch.setenv("SSL_CERT_FILE", str(tls_certificate[0])) + tls_processor.url = tls_processor.url.replace("localhost", "127.0.0.1") + with pytest.raises(ExtractionError) as caught: + await tls_processor.client().extract(input_file, "pdf") + assert (caught.value.code, caught.value.status_code) == ("UNAVAILABLE", 503) + assert not tls_processor.requests + + +@pytest.mark.parametrize("cancel", [False, True]) +async def test_https_abort_closes_real_transport( + tls_processor, tls_certificate, input_file, monkeypatch, cancel +): + monkeypatch.setenv("SSL_CERT_FILE", str(tls_certificate[0])) + tls_processor.hold = True + client = tls_processor.client(deadline=0.1 if not cancel else 10) + task = asyncio.create_task(client.extract(input_file, "pdf")) + await asyncio.wait_for(tls_processor.entered.wait(), 1) + if cancel: + task.cancel() + with pytest.raises(asyncio.CancelledError if cancel else ExtractionError): + await task + await asyncio.wait_for(tls_processor.closed.wait(), 1) + assert client._admitted == 0 + + +async def test_tls_handshake_deadline_aborts_connection(processor, input_file): + connected, closed = asyncio.Event(), asyncio.Event() + + async def never_tls(reader, writer): + connected.set() + try: + await reader.read() + closed.set() + finally: + writer.close() + await writer.wait_closed() + + server = await asyncio.start_server(never_tls, "127.0.0.1", 0) + async with server: + processor.url = f"https://localhost:{server.sockets[0].getsockname()[1]}" + with pytest.raises(ExtractionError, match="DEADLINE"): + await processor.client(deadline=0.05).extract(input_file, "pdf") + assert connected.is_set() + await asyncio.wait_for(closed.wait(), 1) + assert not processor.requests + + +async def test_capabilities_and_result_share_one_deadline( + processor, input_file, monkeypatch +): + client = processor.client(deadline=0.05) + request = client._request + remaining = [] + + async def delay(method, *args, **kwargs): + if method == "POST": + remaining.append(asyncio.current_task()) + await asyncio.sleep(0.05) + else: + await asyncio.sleep(0.03) + return await request(method, *args, **kwargs) + + monkeypatch.setattr(client, "_request", delay) + with pytest.raises(ExtractionError, match="DEADLINE"): + await client.extract(input_file, "pdf") + assert remaining and len(processor.requests) == 1 + assert client._admitted == 0 + + +async def test_remote_error_text_and_redirects_are_not_reflected(processor, input_file): + secret = b"private-service-details" + body = b'{"message":"private-service-details"}' + processor.raw = ( + b"HTTP/1.1 401 Unauthorized\r\nContent-Type: application/json; charset=utf-8\r\n" + + f"Content-Length: {len(body)}\r\nConnection: close\r\n\r\n".encode() + + body + ) + with pytest.raises(ExtractionError) as caught: + await processor.client().extract(input_file, "pdf") + assert secret.decode() not in str(caught.value) + processor.raw = ( + b"HTTP/1.1 302 Redirect\r\nLocation: https://secret@private.example/\r\n" + b"Content-Type: application/json; charset=utf-8\r\nContent-Length: 2\r\n" + b"Connection: close\r\n\r\n{}" + ) + with pytest.raises(ExtractionError, match="INVALID_RESPONSE"): + await processor.client().extract(input_file, "pdf") + assert len(processor.requests) == 4 + + +def test_invalid_deadline_environment_does_not_echo_value(monkeypatch): + monkeypatch.setenv("RAG_EXTRACTION_DEADLINE_SECONDS", "private-service-details") + with pytest.raises(ValueError) as caught: + ExtractionSettings.from_env() + assert "private-service-details" not in str(caught.value) + + +@pytest.mark.parametrize( + "url", ["https://processor.example:8443", "http://localhost:2345"] +) +def test_service_environment_uses_only_service_configuration(monkeypatch, url): + monkeypatch.setenv("RAG_EXTRACTION_URL", url) + monkeypatch.setenv("RAG_EXTRACTION_TOKEN_FILE", "/run/extraction/custom-token") + monkeypatch.setenv("RAG_EXTRACTION_DEADLINE_SECONDS", "4.5") + monkeypatch.setenv("OPENAI_API_KEY", "unrelated-app-credential") + monkeypatch.setenv("JWT_SECRET", "unrelated-user-auth-secret") + settings = ExtractionSettings.from_env() + assert settings.url == url + assert settings.token_file == "/run/extraction/custom-token" + assert settings.deadline == 4.5 diff --git a/tests/extraction/test_processor.py b/tests/extraction/test_processor.py index b9b83d47..ab76fbe5 100644 --- a/tests/extraction/test_processor.py +++ b/tests/extraction/test_processor.py @@ -16,6 +16,11 @@ ) pytestmark = pytest.mark.integration +CALLER_IMAGE = ( + "python:3.13-slim-bookworm@" + "sha256:5024f48ba9441d4b13a95d3945abc6365538e3a31109833367a1923523c6efed" +) +GATEWAY = "/opt/extraction/gateway" def docker(*args, check=True): @@ -34,10 +39,11 @@ def test_python_client_to_default_hardened_processor(tmp_path): ) run_id = uuid.uuid4().hex project = f"rag-extraction-{run_id}" - volume = f"{project}_extraction-socket" + volume = f"{project}_extraction-credentials" + network = f"{project}_extraction" caller = f"rag-extraction-caller-{run_id}" compose = compose_command(directory, project, "docker-compose.yaml") - image = f"{project}-extraction" + images = [f"{project}-extraction", f"{project}-extraction-init"] def run_compose(*args, check=True, timeout=40): result = subprocess.run( @@ -52,14 +58,18 @@ def run_compose(*args, check=True, timeout=40): fixtures = Path(__file__).parent / "fixtures" library = Path(aiofiles.__file__).parent try: - run_compose("build", "extraction", timeout=240) + run_compose("build", "extraction-init", "extraction", timeout=240) run_compose("up", "-d", "--wait", "--wait-timeout", "30", "extraction") name = run_compose("ps", "-q", "extraction").stdout.strip() image = docker("inspect", "--format", "{{.Config.Image}}", name).stdout.strip() + assert ( + json.loads(docker("network", "inspect", network).stdout)[0]["Internal"] + is True + ) runtime = json.loads(docker("inspect", name).stdout)[0] host = runtime["HostConfig"] assert ( - host["NetworkMode"] == "none" + host["NetworkMode"] == network and host["ReadonlyRootfs"] and not host["Privileged"] ) @@ -67,12 +77,38 @@ def run_compose(*args, check=True, timeout=40): host["CapDrop"] == ["ALL"] and "no-new-privileges:true" in host["SecurityOpt"] ) + assert "seccomp=unconfined" not in host["SecurityOpt"] assert runtime["Config"]["User"] == "10001:10001" + assert runtime["Config"]["Entrypoint"] == [GATEWAY] + assert runtime["Config"]["Healthcheck"]["Test"] == [ + "CMD", + GATEWAY, + "node", + "dist/health.js", + ] + init_name = run_compose("ps", "-a", "-q", "extraction-init").stdout.strip() + provisioner = json.loads(docker("inspect", init_name).stdout)[0] + assert provisioner["Config"]["User"] == "10001:10001" + assert provisioner["Config"]["Entrypoint"] == [GATEWAY] + assert provisioner["Config"]["Cmd"] == ["node", "dist/provision.js"] + assert provisioner["Config"]["Healthcheck"]["Test"] == ["NONE"] + # Compose adds service-specific image labels; the pinned payload must match. + provisioner_image = json.loads( + docker("image", "inspect", provisioner["Image"]).stdout + )[0] + runtime_image = json.loads(docker("image", "inspect", runtime["Image"]).stdout)[ + 0 + ] + assert provisioner_image["RootFS"] == runtime_image["RootFS"] + assert provisioner["State"]["ExitCode"] == 0 + assert not host["PortBindings"] + assert set(runtime["NetworkSettings"]["Networks"]) == {network} assert host["Memory"] == 768 * 1024 * 1024 and host["NanoCpus"] == 2 * 10**9 assert host["PidsLimit"] == 32 and set(host["Tmpfs"]) == {"/jobs"} assert ( len(runtime["Mounts"]) == 1 - and runtime["Mounts"][0]["Destination"] == "/socket" + and runtime["Mounts"][0]["Destination"] == "/run/extraction" + and runtime["Mounts"][0]["RW"] is False ) script = """ import asyncio @@ -81,18 +117,61 @@ def run_compose(*args, check=True, timeout=40): import stat import sys sys.path.insert(0, "/") -from client import ExtractionClient, ExtractionSettings +from dataclasses import replace +from pathlib import Path +from client import ExtractionClient, ExtractionError, ExtractionSettings async def check(): settings = ExtractionSettings.from_env() - assert settings.socket == "/socket/extraction.sock" - assert stat.S_IMODE(os.stat("/socket").st_mode) == 0o770 - assert stat.S_IMODE(os.stat(settings.socket).st_mode) == 0o660 + assert settings.url == "http://extraction:8000" + assert settings.token_file == "/run/extraction/token" + assert stat.S_IMODE(os.stat("/run/extraction").st_mode) == 0o755 + assert stat.S_IMODE(os.stat(settings.token_file).st_mode) == 0o444 try: - open("/socket/not-writable", "wb") + open("/run/extraction/not-writable", "wb") except OSError as error: assert error.errno == errno.EROFS else: - raise AssertionError("caller socket mount is writable") + raise AssertionError("caller credential mount is writable") + for token_file in ("/jobs/missing-token", "/jobs/wrong-token"): + if token_file.endswith("wrong-token"): + token = Path(settings.token_file).read_text() + Path(token_file).write_text(("a" if token[0] != "a" else "b") + token[1:]) + try: + await ExtractionClient(replace(settings, token_file=token_file)).extract("/fixtures/pages.pdf", "pdf") + except ExtractionError as error: + assert (error.code, error.status_code) == ("UNAVAILABLE", 503) + else: + raise AssertionError("missing/wrong service credentials accepted") + # Stop a real HTTP upload before the body completes, then reuse admission. + cancellation_input = Path("/jobs/cancel.pdf") + cancellation_input.write_bytes(b"%PDF-1.7\\n" + b"x" * 131072) + entered = asyncio.Event() + class SlowInput: + def __init__(self, file): + self.file = file + self.sent = False + async def read(self, size): + if not self.sent: + self.sent = True + return await self.file.read(1) + entered.set() + await asyncio.Future() + class CancellingClient(ExtractionClient): + async def _request(self, method, token, **kwargs): + if method == "POST": + kwargs["file"] = SlowInput(kwargs["file"]) + return await super()._request(method, token, **kwargs) + cancelled_client = CancellingClient(settings) + task = asyncio.create_task(cancelled_client.extract(str(cancellation_input), "pdf")) + await asyncio.wait_for(entered.wait(), 2) + task.cancel() + try: + await task + except asyncio.CancelledError: + pass + else: + raise AssertionError("extraction did not propagate cancellation") + assert cancelled_client._admitted == 0 client = ExtractionClient(settings) pdf = await client.extract("/fixtures/pages.pdf", "pdf") assert pdf == [ @@ -106,42 +185,88 @@ async def check(): asyncio.run(check()) print("Python client to hardened PDF/DOCX processor passed") """ - result = docker( + # The trusted RAG caller uses Python's normal asyncio self-pipe, not the worker gateway. + caller_command = [ "run", "--rm", "--name", caller, "--init", "--network", - "none", + network, "--read-only", "--user", "65532:65532", - "--group-add", - "10001", "--cap-drop", "ALL", "--security-opt", "no-new-privileges:true", "--mount", - f"type=volume,src={volume},dst=/socket,readonly", + f"type=volume,src={volume},dst=/run/extraction,readonly", + "--tmpfs", + "/jobs:rw,noexec,nosuid,nodev,size=1m,uid=65532,gid=65532,mode=0700", "--mount", f"type=bind,src={client},dst=/client.py,readonly", "--mount", f"type=bind,src={fixtures},dst=/fixtures,readonly", "--mount", f"type=bind,src={library},dst=/usr/local/lib/python3.13/site-packages/aiofiles,readonly", - image, + CALLER_IMAGE, "python", "-B", "-c", script, - ) + ] + result = docker(*caller_command) assert "processor passed" in result.stdout + # Re-provisioning preserves credentials already loaded by the running service. + run_compose("run", "--rm", "--no-deps", "extraction-init") + docker("exec", name, GATEWAY, "node", "dist/health.js") + run_compose( + "up", + "-d", + "--wait", + "--wait-timeout", + "30", + "--scale", + "extraction=2", + "extraction", + ) + replicas = run_compose("ps", "-q", "extraction").stdout.split() + assert len(replicas) == 2 + for replica in replicas: + docker("exec", replica, GATEWAY, "node", "dist/health.js") + assert "processor passed" in docker(*caller_command).stdout + run_compose("restart", "extraction") + run_compose( + "up", + "-d", + "--wait", + "--wait-timeout", + "30", + "--scale", + "extraction=2", + "extraction", + ) + assert "processor passed" in docker(*caller_command).stdout + # Exercise the shipped gateway rather than a test-only seccomp policy. + probe = """ +import errno +import socket +for operation in (lambda: socket.socket(socket.AF_UNIX), socket.socketpair): + try: + operation() + except OSError as error: + assert error.errno == errno.EPERM + else: + raise AssertionError("gateway permitted Unix sockets") +""" + docker("exec", name, GATEWAY, "python", "-c", probe) assert ( docker( "exec", name, + GATEWAY, "node", "-e", 'if(require("node:fs").readdirSync("/jobs").length)process.exit(1)', @@ -151,5 +276,5 @@ async def check(): finally: docker("rm", "-f", caller, check=False) run_compose("down", "-v", "--remove-orphans", check=False) - if image: + for image in images: docker("image", "rm", image, check=False) diff --git a/tests/extraction/test_routes.py b/tests/extraction/test_routes.py index 6bda7570..9fe87b3a 100644 --- a/tests/extraction/test_routes.py +++ b/tests/extraction/test_routes.py @@ -169,6 +169,11 @@ async def test_migrated_routes_parse_once_and_keep_caller_metadata( assert result["filename"] == f"report.{format}" assert result["known_type"] is True assert [r[0] for r in processor.requests] == ["GET", "POST"] + assert all( + request[2]["Authorization"] == f"Bearer {processor.token}" + and request[2]["Authorization"] != client.headers["Authorization"] + for request in processor.requests + ) if route == "/text": assert result["text"] == ( "First page café\nSecond page" @@ -222,7 +227,7 @@ async def test_selected_format_failure_never_falls_back_or_persists( processor.error = (422, "UNSUPPORTED_CONTENT") elif failure == "outage": routes.extraction_client = ExtractionClient( - ExtractionSettings(socket="/socket/missing.sock") + ExtractionSettings(token_file="/run/extraction/missing-token") ) elif failure == "incompatible": processor.capabilities["version"] = 2 @@ -345,18 +350,23 @@ async def test_no_extraction_configuration_still_requires_service( "RAG_EXTRACTION_ENABLED", "RAG_EXTRACTION_FORMATS", "RAG_EXTRACTION_SOCKET", + "RAG_EXTRACTION_URL", + "RAG_EXTRACTION_TOKEN_FILE", "RAG_EXTRACTION_DEADLINE_SECONDS", ): monkeypatch.delenv(key, raising=False) + monkeypatch.setenv("RAG_EXTRACTION_TOKEN_FILE", processor.token_file) defaults = ExtractionSettings.from_env() routes.extraction_client = ExtractionClient(defaults) - connect = asyncio.open_unix_connection + connect = asyncio.open_connection - async def disposable_socket(path, **kwargs): - assert path == "/socket/extraction.sock" - return await connect(processor.socket, **kwargs) + async def disposable_endpoint(host, port, **kwargs): + assert (host, port) == ("extraction", 8000) + return await connect( + "127.0.0.1", int(processor.url.rsplit(":", 1)[1]), **kwargs + ) - monkeypatch.setattr(asyncio, "open_unix_connection", disposable_socket) + monkeypatch.setattr(asyncio, "open_connection", disposable_endpoint) response = await post(client, uploads, route, format) assert response.status_code == 200, response.text assert [r[0] for r in processor.requests] == ["GET", "POST"] @@ -852,7 +862,7 @@ async def observe_submission(*args): connection = ASGIConnection(app, request, await request.aread()) try: await asyncio.wait_for(submitted.wait(), 1) - assert blocker_entered.is_set() + assert await asyncio.to_thread(blocker_entered.wait, 1) connection.disconnect() await asyncio.sleep(0.05) assert "http.disconnect" in connection.received @@ -1248,7 +1258,7 @@ async def delayed_remove(*args, **kwargs): @pytest.mark.parametrize("route", ROUTES) @pytest.mark.parametrize("format", ["pdf", "docx"]) -async def test_default_socket_unavailable_fails_closed_without_affecting_text( +async def test_default_service_unavailable_fails_closed_without_affecting_text( api, processor, monkeypatch, route, format ): client, store, uploads = api @@ -1256,16 +1266,19 @@ async def test_default_socket_unavailable_fails_closed_without_affecting_text( "RAG_EXTRACTION_ENABLED", "RAG_EXTRACTION_FORMATS", "RAG_EXTRACTION_SOCKET", + "RAG_EXTRACTION_URL", + "RAG_EXTRACTION_TOKEN_FILE", "RAG_EXTRACTION_DEADLINE_SECONDS", ): monkeypatch.delenv(key, raising=False) + monkeypatch.setenv("RAG_EXTRACTION_TOKEN_FILE", processor.token_file) routes.extraction_client = ExtractionClient(ExtractionSettings.from_env()) - async def unavailable(path, **kwargs): - assert path == "/socket/extraction.sock" - raise FileNotFoundError() + async def unavailable(host, port, **kwargs): + assert (host, port) == ("extraction", 8000) + raise ConnectionRefusedError() - monkeypatch.setattr(asyncio, "open_unix_connection", unavailable) + monkeypatch.setattr(asyncio, "open_connection", unavailable) response = await post(client, uploads, route, format) assert response.status_code == 503 assert response.json()["detail"]["code"] == "UNAVAILABLE" @@ -1275,3 +1288,33 @@ async def unavailable(path, **kwargs): ) plaintext = await post(client, uploads, "/text", "txt", b"plain text", "text/plain") assert plaintext.status_code == 200 and plaintext.json()["text"] == "plain text" + + +@pytest.mark.parametrize("route", ROUTES) +@pytest.mark.parametrize("failure", ["missing", "wrong", "extract-unauthorized"]) +@pytest.mark.parametrize("format", ["pdf", "docx"]) +async def test_service_auth_failure_is_unavailable_not_user_unauthorized( + api, processor, route, failure, format, caplog +): + client, store, uploads = api + if failure == "missing": + Path(processor.token_file).unlink() + elif failure == "wrong": + Path(processor.token_file).write_text("cd" * 32) + else: + processor.error = (401, "UNAUTHORIZED") + response = await post(client, uploads, route, format) + assert response.status_code == 503 + assert response.json()["detail"] == {"code": "UNAVAILABLE"} + assert not store.calls and not store.persisted and not store.rollbacks + assert ( + len(processor.requests) + == {"missing": 0, "wrong": 1, "extract-unauthorized": 2}[failure] + ) + assert "Bearer" not in caplog.text and processor.token not in caplog.text + assert "cd" * 32 not in caplog.text + assert len(list(uploads.rglob(f"*.{format}"))) == ( + 1 if route == "/local/embed" else 0 + ) + plaintext = await post(client, uploads, "/text", "txt", b"plain text", "text/plain") + assert plaintext.status_code == 200 diff --git a/tests/extraction/test_upload_transport.py b/tests/extraction/test_upload_transport.py new file mode 100644 index 00000000..c5e574cf --- /dev/null +++ b/tests/extraction/test_upload_transport.py @@ -0,0 +1,135 @@ +import asyncio +import socket +import ssl + +import pytest + +from app.services.extraction_client import ( + CHUNK_BYTES, + INPUT_BYTES, + ExtractionSettings, + ExtractionClient, + ExtractionError, +) + + +@pytest.mark.parametrize("https", [False, True]) +@pytest.mark.parametrize("cancel", [False, True]) +async def test_backpressured_upload_abort_stops_input( + processor, input_file, tls_certificate, monkeypatch, https, cancel +): + with open(input_file, "wb") as file: + file.truncate(INPUT_BYTES) + entered, release, closed = asyncio.Event(), asyncio.Event(), asyncio.Event() + received = 0 + tasks = set() + + async def handle(reader, writer): + nonlocal received + tasks.add(asyncio.current_task()) + try: + await reader.readuntil(b"\r\n\r\n") + entered.set() + await release.wait() + while chunk := await reader.read(CHUNK_BYTES): + received += len(chunk) + except OSError: + pass + finally: + writer.close() + try: + await writer.wait_closed() + except OSError: + pass + if entered.is_set(): + closed.set() + tasks.discard(asyncio.current_task()) + + context = None + if https: + monkeypatch.setenv("SSL_CERT_FILE", str(tls_certificate[0])) + context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + context.load_cert_chain(*tls_certificate) + server = await asyncio.start_server(handle, "127.0.0.1", 0, ssl=context) + connect = asyncio.open_connection + + async def bounded_socket(*args, **kwargs): + reader, writer = await connect(*args, **kwargs) + writer.get_extra_info("socket").setsockopt( + socket.SOL_SOCKET, socket.SO_SNDBUF, 8192 + ) + return reader, writer + + monkeypatch.setattr(asyncio, "open_connection", bounded_socket) + async with server: + client = ExtractionClient( + ExtractionSettings( + url=f"{'https://localhost' if https else 'http://127.0.0.1'}:{server.sockets[0].getsockname()[1]}", + token_file=processor.token_file, + deadline=10 if cancel else 0.2, + ) + ) + request = client._request + + async def extraction_only(method, *args, **kwargs): + if method == "GET": + return processor.capabilities, "" + return await request(method, *args, **kwargs) + + monkeypatch.setattr(client, "_request", extraction_only) + task = asyncio.create_task(client.extract(input_file, "pdf")) + try: + await asyncio.wait_for(entered.wait(), 1) + if cancel: + task.cancel() + with pytest.raises( + asyncio.CancelledError if cancel else ExtractionError + ) as caught: + await asyncio.wait_for(asyncio.shield(task), 1) + if not cancel: + assert caught.value.code == "DEADLINE" + release.set() + await asyncio.wait_for(closed.wait(), 1) + assert received < INPUT_BYTES and client._admitted == 0 + finally: + release.set() + task.cancel() + await asyncio.gather(task, return_exceptions=True) + for active in list(tasks): + active.cancel() + await asyncio.gather(*tasks, return_exceptions=True) + + +async def test_repeated_cancellation_drains_aborted_writer( + processor, input_file, monkeypatch +): + processor.hold = True + entered, release = asyncio.Event(), asyncio.Event() + wait_closed = asyncio.StreamWriter.wait_closed + + async def delayed_close(writer): + await wait_closed(writer) + if processor.entered.is_set(): + entered.set() + await release.wait() + + monkeypatch.setattr(asyncio.StreamWriter, "wait_closed", delayed_close) + client = processor.client() + task = asyncio.create_task(client.extract(input_file, "pdf")) + try: + await asyncio.wait_for(processor.entered.wait(), 1) + task.cancel() + await asyncio.wait_for(entered.wait(), 1) + for _ in range(2): + task.cancel() + await asyncio.sleep(0) + assert not task.done() and client._admitted == 1 + release.set() + with pytest.raises(asyncio.CancelledError): + await asyncio.wait_for(task, 1) + await asyncio.wait_for(processor.closed.wait(), 1) + assert client._admitted == 0 + finally: + release.set() + task.cancel() + await asyncio.gather(task, return_exceptions=True)