From 86946ca14282d06c83061644d2cf3111a9adaed3 Mon Sep 17 00:00:00 2001 From: Dhirender Choudhary Date: Fri, 23 Oct 2026 19:59:28 +0530 Subject: [PATCH 1/4] Add speech endpoints to inference --- src/inference/app.py | 55 ++++++++ src/inference/core/handlers/__init__.py | 2 + src/inference/core/handlers/audio.py | 134 ++++++++++++++++++++ src/inference/core/orchestrator.py | 5 + src/inference/core/providers/base.py | 4 +- src/inference/core/providers/registry.py | 3 + src/inference/core/service.py | 77 +++++++++++ src/tests/inference/test_audio_endpoints.py | 110 ++++++++++++++++ 8 files changed, 389 insertions(+), 1 deletion(-) create mode 100644 src/inference/core/handlers/audio.py create mode 100644 src/tests/inference/test_audio_endpoints.py diff --git a/src/inference/app.py b/src/inference/app.py index af201f5d..6213e1e9 100644 --- a/src/inference/app.py +++ b/src/inference/app.py @@ -289,6 +289,61 @@ async def create_embeddings( ) +@app.post("/v1/audio/transcriptions") +async def create_transcription( + request: Request, + background_tasks: BackgroundTasks, + authorization: str = Header(None), + sandbox: str = Header(None, alias="x-sandbox"), +): + """Speech-to-text endpoint - OpenAI compatible (multipart upload).""" + is_sandbox = sandbox.lower() == "true" if sandbox else False + api_key = extract_api_key(authorization, is_sandbox) + try: + form = await request.form() + except Exception: + raise HTTPException(status_code=400, detail="Expected a multipart form") + upload = form.get("file") + if upload is None or isinstance(upload, str): + raise HTTPException(status_code=400, detail="Model and file are required") + fields = { + k: v for k, v in form.items() if isinstance(v, str) and k not in ("model", "file") + } + + return await OrchestrationService.handle_transcription( + api_key=api_key, + model=str(form.get("model") or ""), + audio=await upload.read(), + filename=upload.filename or "audio", + content_type=upload.content_type, + fields=fields, + background_tasks=background_tasks, + ip_address=extract_client_ip(request), + sandbox=is_sandbox, + ) + + +@app.post("/v1/audio/speech") +async def create_speech( + request: Request, + background_tasks: BackgroundTasks, + authorization: str = Header(None), + sandbox: str = Header(None, alias="x-sandbox"), +): + """Text-to-speech endpoint - OpenAI compatible. Returns audio bytes.""" + is_sandbox = sandbox.lower() == "true" if sandbox else False + api_key = extract_api_key(authorization, is_sandbox) + body = await parse_json_body(request) + + return await OrchestrationService.handle_speech( + api_key=api_key, + body=body, + background_tasks=background_tasks, + ip_address=extract_client_ip(request), + sandbox=is_sandbox, + ) + + @app.post("/v1/images/generations") async def create_image( request: Request, diff --git a/src/inference/core/handlers/__init__.py b/src/inference/core/handlers/__init__.py index ca031f50..2430038b 100644 --- a/src/inference/core/handlers/__init__.py +++ b/src/inference/core/handlers/__init__.py @@ -1,5 +1,6 @@ """Handler modules for each inference request type.""" +from .audio import AudioHandler from .completion import CompletionHandler from .embedding import EmbeddingHandler from .image import ImageHandler @@ -7,6 +8,7 @@ from .video import VideoHandler __all__ = [ + "AudioHandler", "CompletionHandler", "EmbeddingHandler", "ImageHandler", diff --git a/src/inference/core/handlers/audio.py b/src/inference/core/handlers/audio.py new file mode 100644 index 00000000..86f584f8 --- /dev/null +++ b/src/inference/core/handlers/audio.py @@ -0,0 +1,134 @@ +"""Audio request handlers: speech-to-text and text-to-speech.""" + +import logging +import time +from typing import Dict, Optional + +from fastapi import BackgroundTasks, HTTPException +from fastapi.responses import Response + +from ..pipeline import Pipeline, RequestContext +from ..request_logger import RequestLogger +from ..service import GatewayService + +logger = logging.getLogger(__name__) + +MODEL_TYPE = "audio_generation" +DEFAULT_ENGINE = "speaches" + + +class AudioHandler: + @staticmethod + async def handle_transcription( + api_key: str, + model: str, + audio: bytes, + filename: str, + content_type: Optional[str], + fields: Dict[str, str], + background_tasks: BackgroundTasks, + ip_address: Optional[str] = None, + sandbox: bool = False, + ): + if not model or not audio: + raise HTTPException(status_code=400, detail="Model and file are required") + + async def send(ctx: RequestContext): + return await GatewayService.call_upstream_raw( + ctx.endpoint_url, + ctx.adapter.get_endpoint_path("transcription"), + ctx.provider_headers, + files={"file": (filename, audio, content_type or "application/octet-stream")}, + data={**fields, "model": ctx.body["model"]}, + concurrency_key=ctx.concurrency_key, + ) + + resp = await AudioHandler._handle( + api_key, {"model": model}, "transcription", send, + log_payload={"model": model, "file": filename, "bytes": len(audio), **fields}, + background_tasks=background_tasks, ip_address=ip_address, sandbox=sandbox, + ) + if "json" in resp.headers.get("content-type", ""): + data = resp.json() + if isinstance(data, dict) and "model" in data: + data["model"] = model + return data + return Response(content=resp.content, media_type=resp.headers.get("content-type")) + + @staticmethod + async def handle_speech( + api_key: str, + body: Dict, + background_tasks: BackgroundTasks, + ip_address: Optional[str] = None, + sandbox: bool = False, + ): + if not body.get("model") or not body.get("input"): + raise HTTPException(status_code=400, detail="Model and input are required") + + async def send(ctx: RequestContext): + return await GatewayService.call_upstream_raw( + ctx.endpoint_url, + ctx.adapter.get_endpoint_path("speech"), + ctx.provider_headers, + json_body=ctx.body, + concurrency_key=ctx.concurrency_key, + ) + + resp = await AudioHandler._handle( + api_key, body, "speech", send, log_payload=body, + background_tasks=background_tasks, ip_address=ip_address, sandbox=sandbox, + ) + return Response( + content=resp.content, + media_type=resp.headers.get("content-type", "audio/mpeg"), + ) + + @staticmethod + async def _handle( + api_key, body, request_type, send, *, log_payload, + background_tasks, ip_address, sandbox, + ): + requested_model = body["model"] + ctx = RequestContext( + api_key=api_key, + body=dict(body), + model=requested_model, + sandbox=sandbox, + ip_address=ip_address, + background_tasks=background_tasks, + start_time=time.time(), + ) + await Pipeline.resolve_context(ctx, model_type=MODEL_TYPE) + await Pipeline.check_rate_limit(ctx) + await Pipeline.check_quota(ctx) + Pipeline.resolve_provider(ctx, default_engine=DEFAULT_ENGINE) + + status_code = 200 + error_message = None + try: + return await send(ctx) + except HTTPException as e: + status_code = e.status_code + error_message = str(e.detail) + raise + except Exception as e: + status_code = 500 + error_message = str(e) + raise + finally: + background_tasks.add_task( + RequestLogger.log, + deployment_id=ctx.deployment_id, + user_id=ctx.user_context_id, + org_id=ctx.org_id, + model=requested_model, + request_payload=log_payload, + start_time=ctx.start_time, + request_type=request_type, + applied_policies=ctx.applied_policies, + log_payloads=ctx.log_payloads, + ip_address=ip_address, + status_code=status_code, + error_message=error_message, + ) diff --git a/src/inference/core/orchestrator.py b/src/inference/core/orchestrator.py index 4e152a2f..067e8b71 100644 --- a/src/inference/core/orchestrator.py +++ b/src/inference/core/orchestrator.py @@ -9,6 +9,7 @@ from inference.client import api_gateway_client from .handlers import ( + AudioHandler, CompletionHandler, EmbeddingHandler, ImageHandler, @@ -51,6 +52,10 @@ async def list_models(api_key: str, sandbox: bool = False): handle_image_edit = ImageHandler.handle_edit handle_image_variations = ImageHandler.handle_variations + # --- Audio --- + handle_transcription = AudioHandler.handle_transcription + handle_speech = AudioHandler.handle_speech + # --- Video --- handle_video_generation = VideoHandler.handle_generation handle_video_edit = VideoHandler.handle_edit diff --git a/src/inference/core/providers/base.py b/src/inference/core/providers/base.py index 70ab5269..c7f813be 100644 --- a/src/inference/core/providers/base.py +++ b/src/inference/core/providers/base.py @@ -78,7 +78,7 @@ def get_endpoint_path(self, request_type: str) -> str: Args: request_type: One of 'chat', 'embedding', 'image_generation', 'image_edit', 'image_variations', 'video_generation', - 'video_edit', 'video_extension'. + 'video_edit', 'video_extension', 'transcription', 'speech'. """ paths = { "chat": self.get_chat_path(), @@ -89,6 +89,8 @@ def get_endpoint_path(self, request_type: str) -> str: "video_generation": "/v1/videos/generations", "video_edit": "/v1/videos/edits", "video_extension": "/v1/videos/extensions", + "transcription": "/v1/audio/transcriptions", + "speech": "/v1/audio/speech", } return paths.get(request_type, self.get_chat_path()) diff --git a/src/inference/core/providers/registry.py b/src/inference/core/providers/registry.py index ed6cebd9..fa7c0b16 100644 --- a/src/inference/core/providers/registry.py +++ b/src/inference/core/providers/registry.py @@ -34,6 +34,7 @@ EMBEDDING_ENGINES = {"infinity", "tei"} IMAGE_ENGINES = {"inferia-diffusion"} VIDEO_ENGINES = {"inferia-diffusion"} +AUDIO_ENGINES = {"speaches"} # Backward-compat aliases EXTERNAL_ENGINES = EXTERNAL_PROVIDERS @@ -71,6 +72,8 @@ def get_adapter(engine: str) -> ProviderAdapter: "vllm": ComputeAdapter(), "ollama": ComputeAdapter(), "generic": ComputeAdapter(), + # Speech engines (OpenAI-compatible /v1/audio/*) + "speaches": ComputeAdapter(), # Embedding engines (OpenAI-compatible for embeddings) "infinity": EmbeddingAdapter(), "tei": EmbeddingAdapter(), diff --git a/src/inference/core/service.py b/src/inference/core/service.py index 54b9f52b..9ed30b40 100644 --- a/src/inference/core/service.py +++ b/src/inference/core/service.py @@ -67,6 +67,8 @@ def _build_full_url(endpoint_url: str, chat_path: str) -> str: or endpoint.endswith("/videos/generations") or endpoint.endswith("/videos/edits") or endpoint.endswith("/videos/extensions") + or endpoint.endswith("/audio/transcriptions") + or endpoint.endswith("/audio/speech") ): return endpoint @@ -171,6 +173,81 @@ async def stream_upstream( logger.error(f"Streaming Exception: {e}") yield _upstream_error(error_sink, 502, "Streaming connection failed") + @staticmethod + async def call_upstream_raw( + endpoint_url: str, + path: str, + headers: Dict, + *, + json_body: Optional[Dict] = None, + files: Optional[Dict] = None, + data: Optional[Dict] = None, + concurrency_key: str = "default", + ) -> httpx.Response: + """POST to the upstream and return the read response without parsing + it. For multipart uploads and binary bodies, which call_upstream's + JSON-in/JSON-out shape can't carry.""" + full_url = GatewayService._build_full_url(endpoint_url, path) + try: + full_url = validate_upstream_url( + full_url, GatewayService._get_allowed_hosts() + ) + headers = sanitize_headers(headers) + except ValueError as e: + raise HTTPException(status_code=400, detail=str(e)) + if files: + # httpx sets the multipart boundary itself. + headers = {k: v for k, v in headers.items() if k.lower() != "content-type"} + + breaker = circuit_breaker_registry.get_or_create( + f"upstream:{concurrency_key}", + failure_threshold=5, + recovery_timeout=30.0, + expected_exception=(httpx.HTTPStatusError, httpx.RequestError), + ) + if not await breaker._can_execute(): + raise HTTPException( + status_code=503, + detail="Upstream temporarily unavailable (circuit breaker open)", + ) + + max_bytes = settings.upstream_max_response_bytes + client = http_client.get_client() + try: + async with upstream_concurrency_limiter.limit(concurrency_key): + resp = await client.post( + full_url, + json=json_body, + files=files, + data=data, + headers=headers, + timeout=settings.upstream_http_timeout_seconds, + ) + resp.raise_for_status() + if len(resp.content) > max_bytes: + raise HTTPException( + status_code=502, detail="Upstream response exceeded size limit" + ) + await breaker._record_success() + return resp + except HTTPException: + raise + except httpx.HTTPStatusError as e: + # A 4xx is about this request, not the upstream's health. + if e.response.status_code >= 500: + await breaker._record_failure() + logger.error(f"Provider error {e.response.status_code}: {e.response.text}") + raise HTTPException( + status_code=e.response.status_code, + detail="Upstream provider returned an error", + ) + except Exception as e: + await breaker._record_failure() + logger.error(f"Provider request failed: {e}") + raise HTTPException( + status_code=502, detail="Upstream provider is unavailable" + ) + @staticmethod async def call_upstream( endpoint_url: str, diff --git a/src/tests/inference/test_audio_endpoints.py b/src/tests/inference/test_audio_endpoints.py new file mode 100644 index 00000000..716240d6 --- /dev/null +++ b/src/tests/inference/test_audio_endpoints.py @@ -0,0 +1,110 @@ +import json +from unittest.mock import AsyncMock, patch + +import httpx +import pytest + +from inference.app import app + +UPSTREAM = "https://speech.example.com" + + +def _context(model_type_seen): + async def resolve(api_key, model, model_type="inference", sandbox=False): + model_type_seen.append(model_type) + return { + "deployment": { + "id": "dep-1", + "engine": "speaches", + "endpoint": UPSTREAM, + "inference_model": "Systran/faster-whisper-tiny" if model == "stt" else None, + "configuration": {}, + }, + "user_id_context": "u1", + "org_id": "o1", + } + return resolve + + +@pytest.fixture +def upstream(): + seen = [] + + def handler(request: httpx.Request): + seen.append(request) + if request.url.path == "/v1/audio/transcriptions": + return httpx.Response(200, json={"text": "what is my balance"}) + if request.url.path == "/v1/audio/speech": + return httpx.Response(200, content=b"RIFFfakewav", headers={"content-type": "audio/wav"}) + return httpx.Response(404) + + client = httpx.AsyncClient(transport=httpx.MockTransport(handler)) + model_types = [] + with patch("inference.core.service.GatewayService.resolve_context", side_effect=_context(model_types)), \ + patch("inference.core.pipeline.api_gateway_client.check_quota", AsyncMock()), \ + patch("inference.core.handlers.audio.RequestLogger.log", AsyncMock()), \ + patch("inference.core.service.http_client.get_client", return_value=client): + yield seen, model_types + + +async def _post(path, **kwargs): + transport = httpx.ASGITransport(app=app) + async with httpx.AsyncClient(transport=transport, base_url="http://inf") as c: + return await c.post(path, headers={"Authorization": "Bearer sk-test"}, **kwargs) + + +@pytest.mark.asyncio +async def test_transcription_forwards_file_and_returns_text(upstream): + seen, model_types = upstream + resp = await _post( + "/v1/audio/transcriptions", + files={"file": ("q.webm", b"\x1aEfakeaudio", "audio/webm")}, + data={"model": "stt", "language": "en"}, + ) + + assert resp.status_code == 200 + assert resp.json() == {"text": "what is my balance"} + assert model_types == ["audio_generation"] + sent = seen[0] + assert str(sent.url) == f"{UPSTREAM}/v1/audio/transcriptions" + assert sent.headers["content-type"].startswith("multipart/form-data; boundary=") + body = sent.content + assert b'name="model"\r\n\r\nSystran/faster-whisper-tiny' in body + assert b'name="language"\r\n\r\nen' in body + assert b'filename="q.webm"' in body and b"\x1aEfakeaudio" in body + + +@pytest.mark.asyncio +async def test_speech_returns_audio_bytes(upstream): + seen, model_types = upstream + resp = await _post("/v1/audio/speech", json={"model": "tts", "input": "Your balance is 500", "voice": "af_heart"}) + + assert resp.status_code == 200 + assert resp.content == b"RIFFfakewav" + assert resp.headers["content-type"] == "audio/wav" + assert model_types == ["audio_generation"] + assert str(seen[0].url) == f"{UPSTREAM}/v1/audio/speech" + assert json.loads(seen[0].content) == {"model": "tts", "input": "Your balance is 500", "voice": "af_heart"} + + +@pytest.mark.asyncio +async def test_transcription_without_file_is_400(upstream): + resp = await _post("/v1/audio/transcriptions", data={"model": "stt"}) + assert resp.status_code == 400 + + +@pytest.mark.asyncio +async def test_speech_without_input_is_400(upstream): + resp = await _post("/v1/audio/speech", json={"model": "tts"}) + assert resp.status_code == 400 + + +@pytest.mark.asyncio +async def test_upstream_4xx_is_passed_through(upstream): + def handler(request): + return httpx.Response(422, json={"detail": "bad voice"}) + + with patch("inference.core.service.http_client.get_client", + return_value=httpx.AsyncClient(transport=httpx.MockTransport(handler))): + resp = await _post("/v1/audio/speech", json={"model": "tts", "input": "hi", "voice": "nope"}) + assert resp.status_code == 422 From b0958403417a70ed01975980b252f918d3fa4912 Mon Sep 17 00:00:00 2001 From: Dhirender Choudhary Date: Fri, 23 Oct 2026 20:33:10 +0530 Subject: [PATCH 2/4] Return model type in deployment list --- src/orchestration/models/model_deployment/deployment_server.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/orchestration/models/model_deployment/deployment_server.py b/src/orchestration/models/model_deployment/deployment_server.py index 7b50d76d..7502e407 100644 --- a/src/orchestration/models/model_deployment/deployment_server.py +++ b/src/orchestration/models/model_deployment/deployment_server.py @@ -4150,6 +4150,7 @@ async def list_all_deployments( "engine": d.engine, "endpoint": d.endpoint, "org_id": d.org_id, + "model_type": d.model_type or "inference", "error_message": d.error_message or None, } for d in resp.deployments From 5fc22e84f273734a36371fe6a41f7322ad6fe6c0 Mon Sep 17 00:00:00 2001 From: Dhirender Choudhary Date: Fri, 23 Oct 2026 20:43:58 +0530 Subject: [PATCH 3/4] Test model type in deployment list --- .../test_list_deployments_inference_model.py | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/src/tests/orchestration/model_deployment/test_list_deployments_inference_model.py b/src/tests/orchestration/model_deployment/test_list_deployments_inference_model.py index 9a7d394c..ff37b563 100644 --- a/src/tests/orchestration/model_deployment/test_list_deployments_inference_model.py +++ b/src/tests/orchestration/model_deployment/test_list_deployments_inference_model.py @@ -51,6 +51,7 @@ def _fake_deployment(**over): engine="ollama", endpoint="http://x:11434", org_id="org-1", + model_type="inference", error_message="", ) base.update(over) @@ -68,9 +69,8 @@ async def _get(path): return await c.get(path, headers={"X-Organization-ID": "org-1"}) -async def test_listing_includes_inference_model_distinct_from_model_name(): - dep = _fake_deployment() - resp = SimpleNamespace(deployments=[dep]) +async def _list(*deps): + resp = SimpleNamespace(deployments=list(deps)) stub = MagicMock() stub.ListDeployments = AsyncMock(return_value=resp) @@ -84,7 +84,11 @@ async def test_listing_includes_inference_model_distinct_from_model_name(): deployment_server.asyncpg, "connect", AsyncMock(return_value=_FakeConn()), ): - r = await _get("/deployment/deployments?org_id=org-1") + return await _get("/deployment/deployments?org_id=org-1") + + +async def test_listing_includes_inference_model_distinct_from_model_name(): + r = await _list(_fake_deployment()) assert r.status_code == 200, r.text body = r.json()["deployments"] @@ -94,3 +98,10 @@ async def test_listing_includes_inference_model_distinct_from_model_name(): assert row["model_name"] == "My Prod Bot" # The two must be distinct fields so the UI can show the real model. assert row["inference_model"] != row["model_name"] + + +async def test_listing_includes_model_type(): + r = await _list(_fake_deployment(model_type="audio_generation")) + + assert r.status_code == 200, r.text + assert r.json()["deployments"][0]["model_type"] == "audio_generation" From fbd39a0964f6a4a75dc5b3bcd525e1176f64a1f1 Mon Sep 17 00:00:00 2001 From: Priyanshu-u07 Date: Sat, 24 Oct 2026 13:54:04 +0530 Subject: [PATCH 4/4] Cap audio upload size and speech input, restrict response types The API key is only validated after the form is read, so uploads are now size-checked first (25 MB). TTS is slow, so input is capped at 4096 characters to stop long texts blocking the speech server. Upstream content types are limited to audio, JSON and text, with nosniff, since /inf shares the dashboard's origin. Signed-off-by: Priyanshu-u07 --- src/inference/app.py | 17 ++++++++++- src/inference/core/handlers/audio.py | 24 +++++++++++---- src/tests/inference/test_audio_endpoints.py | 33 +++++++++++++++++++++ 3 files changed, 68 insertions(+), 6 deletions(-) diff --git a/src/inference/app.py b/src/inference/app.py index 6213e1e9..6b72aeb8 100644 --- a/src/inference/app.py +++ b/src/inference/app.py @@ -289,6 +289,11 @@ async def create_embeddings( ) +# OpenAI's limit. Checked before the form is read: the API key is only +# validated later, so an oversized upload must not reach memory first. +MAX_AUDIO_BYTES = 25 * 1024 * 1024 + + @app.post("/v1/audio/transcriptions") async def create_transcription( request: Request, @@ -299,6 +304,12 @@ async def create_transcription( """Speech-to-text endpoint - OpenAI compatible (multipart upload).""" is_sandbox = sandbox.lower() == "true" if sandbox else False api_key = extract_api_key(authorization, is_sandbox) + try: + declared = int(request.headers.get("content-length") or 0) + except ValueError: + declared = 0 + if declared > MAX_AUDIO_BYTES: + raise HTTPException(status_code=413, detail="Audio file too large (max 25 MB)") try: form = await request.form() except Exception: @@ -310,10 +321,14 @@ async def create_transcription( k: v for k, v in form.items() if isinstance(v, str) and k not in ("model", "file") } + audio = await upload.read(MAX_AUDIO_BYTES + 1) + if len(audio) > MAX_AUDIO_BYTES: + raise HTTPException(status_code=413, detail="Audio file too large (max 25 MB)") + return await OrchestrationService.handle_transcription( api_key=api_key, model=str(form.get("model") or ""), - audio=await upload.read(), + audio=audio, filename=upload.filename or "audio", content_type=upload.content_type, fields=fields, diff --git a/src/inference/core/handlers/audio.py b/src/inference/core/handlers/audio.py index 86f584f8..d93e95cc 100644 --- a/src/inference/core/handlers/audio.py +++ b/src/inference/core/handlers/audio.py @@ -15,6 +15,18 @@ MODEL_TYPE = "audio_generation" DEFAULT_ENGINE = "speaches" +MAX_SPEECH_CHARS = 4096 # OpenAI's limit. + +# /inf shares the dashboard's origin, so never pass through e.g. text/html. +_SAFE_TYPES = ("audio/", "application/json", "text/plain", "text/vtt", "application/x-subrip") + + +def _safe_response(resp, default_type: str) -> Response: + ctype = resp.headers.get("content-type") or default_type + if not ctype.lower().startswith(_SAFE_TYPES): + ctype = "application/octet-stream" + return Response(content=resp.content, media_type=ctype, + headers={"X-Content-Type-Options": "nosniff"}) class AudioHandler: @@ -53,7 +65,7 @@ async def send(ctx: RequestContext): if isinstance(data, dict) and "model" in data: data["model"] = model return data - return Response(content=resp.content, media_type=resp.headers.get("content-type")) + return _safe_response(resp, "text/plain") @staticmethod async def handle_speech( @@ -65,6 +77,11 @@ async def handle_speech( ): if not body.get("model") or not body.get("input"): raise HTTPException(status_code=400, detail="Model and input are required") + if len(str(body["input"])) > MAX_SPEECH_CHARS: + raise HTTPException( + status_code=400, + detail=f"Input too long (max {MAX_SPEECH_CHARS} characters)", + ) async def send(ctx: RequestContext): return await GatewayService.call_upstream_raw( @@ -79,10 +96,7 @@ async def send(ctx: RequestContext): api_key, body, "speech", send, log_payload=body, background_tasks=background_tasks, ip_address=ip_address, sandbox=sandbox, ) - return Response( - content=resp.content, - media_type=resp.headers.get("content-type", "audio/mpeg"), - ) + return _safe_response(resp, "audio/mpeg") @staticmethod async def _handle( diff --git a/src/tests/inference/test_audio_endpoints.py b/src/tests/inference/test_audio_endpoints.py index 716240d6..89dd67b7 100644 --- a/src/tests/inference/test_audio_endpoints.py +++ b/src/tests/inference/test_audio_endpoints.py @@ -108,3 +108,36 @@ def handler(request): return_value=httpx.AsyncClient(transport=httpx.MockTransport(handler))): resp = await _post("/v1/audio/speech", json={"model": "tts", "input": "hi", "voice": "nope"}) assert resp.status_code == 422 + + +@pytest.mark.asyncio +async def test_oversized_upload_is_refused_before_the_key_is_checked(upstream): + seen, model_types = upstream + with patch("inference.app.MAX_AUDIO_BYTES", 10): + resp = await _post( + "/v1/audio/transcriptions", + files={"file": ("big.wav", b"x" * 100, "audio/wav")}, + data={"model": "stt"}, + ) + assert resp.status_code == 413 + assert model_types == [] and seen == [] + + +@pytest.mark.asyncio +async def test_speech_input_over_the_limit_is_400(upstream): + seen, _ = upstream + resp = await _post("/v1/audio/speech", json={"model": "tts", "input": "a" * 4097, "voice": "af_heart"}) + assert resp.status_code == 400 + assert seen == [] + + +@pytest.mark.asyncio +async def test_unexpected_upstream_content_type_is_not_passed_through(upstream): + def handler(request): + return httpx.Response(200, content=b"", headers={"content-type": "text/html"}) + + with patch("inference.core.service.http_client.get_client", + return_value=httpx.AsyncClient(transport=httpx.MockTransport(handler))): + resp = await _post("/v1/audio/speech", json={"model": "tts", "input": "hi", "voice": "af_heart"}) + assert resp.headers["content-type"] == "application/octet-stream" + assert resp.headers["x-content-type-options"] == "nosniff"