Add speech-to-text and text-to-speech endpoints

src/inference/app.py

@@ -289,6 +289,76 @@ async def create_embeddings(
     )
 
 
+# OpenAI's limit. Checked before the form is read: the API key is only
+# validated later, so an oversized upload must not reach memory first.
+MAX_AUDIO_BYTES = 25 * 1024 * 1024
+
+
+@app.post("/v1/audio/transcriptions")
+async def create_transcription(
+    request: Request,
+    background_tasks: BackgroundTasks,
+    authorization: str = Header(None),
+    sandbox: str = Header(None, alias="x-sandbox"),
+):
+    """Speech-to-text endpoint - OpenAI compatible (multipart upload)."""
+    is_sandbox = sandbox.lower() == "true" if sandbox else False
+    api_key = extract_api_key(authorization, is_sandbox)
+    try:
+        declared = int(request.headers.get("content-length") or 0)
+    except ValueError:
+        declared = 0
+    if declared > MAX_AUDIO_BYTES:
+        raise HTTPException(status_code=413, detail="Audio file too large (max 25 MB)")
+    try:
+        form = await request.form()
+    except Exception:
+        raise HTTPException(status_code=400, detail="Expected a multipart form")
+    upload = form.get("file")
+    if upload is None or isinstance(upload, str):
+        raise HTTPException(status_code=400, detail="Model and file are required")
+    fields = {
+        k: v for k, v in form.items() if isinstance(v, str) and k not in ("model", "file")
+    }
+
+    audio = await upload.read(MAX_AUDIO_BYTES + 1)
+    if len(audio) > MAX_AUDIO_BYTES:
+        raise HTTPException(status_code=413, detail="Audio file too large (max 25 MB)")
+
+    return await OrchestrationService.handle_transcription(
+        api_key=api_key,
+        model=str(form.get("model") or ""),
+        audio=audio,
+        filename=upload.filename or "audio",
+        content_type=upload.content_type,
+        fields=fields,
+        background_tasks=background_tasks,
+        ip_address=extract_client_ip(request),
+        sandbox=is_sandbox,
+    )
+
+
+@app.post("/v1/audio/speech")
+async def create_speech(
+    request: Request,
+    background_tasks: BackgroundTasks,
+    authorization: str = Header(None),
+    sandbox: str = Header(None, alias="x-sandbox"),
+):
+    """Text-to-speech endpoint - OpenAI compatible. Returns audio bytes."""
+    is_sandbox = sandbox.lower() == "true" if sandbox else False
+    api_key = extract_api_key(authorization, is_sandbox)
+    body = await parse_json_body(request)
+
+    return await OrchestrationService.handle_speech(
+        api_key=api_key,
+        body=body,
+        background_tasks=background_tasks,
+        ip_address=extract_client_ip(request),
+        sandbox=is_sandbox,
+    )
+
+
 @app.post("/v1/images/generations")
 async def create_image(
     request: Request,

src/inference/core/handlers/__init__.py

@@ -1,12 +1,14 @@
 """Handler modules for each inference request type."""
 
+from .audio import AudioHandler
 from .completion import CompletionHandler
 from .embedding import EmbeddingHandler
 from .image import ImageHandler
 from .messages import MessagesHandler
 from .video import VideoHandler
 
 __all__ = [
+    "AudioHandler",
     "CompletionHandler",
     "EmbeddingHandler",
     "ImageHandler",

src/inference/core/handlers/audio.py

@@ -0,0 +1,148 @@
+"""Audio request handlers: speech-to-text and text-to-speech."""
+
+import logging
+import time
+from typing import Dict, Optional
+
+from fastapi import BackgroundTasks, HTTPException
+from fastapi.responses import Response
+
+from ..pipeline import Pipeline, RequestContext
+from ..request_logger import RequestLogger
+from ..service import GatewayService
+
+logger = logging.getLogger(__name__)
+
+MODEL_TYPE = "audio_generation"
+DEFAULT_ENGINE = "speaches"
+MAX_SPEECH_CHARS = 4096  # OpenAI's limit.
+
+# /inf shares the dashboard's origin, so never pass through e.g. text/html.
+_SAFE_TYPES = ("audio/", "application/json", "text/plain", "text/vtt", "application/x-subrip")
+
+
+def _safe_response(resp, default_type: str) -> Response:
+    ctype = resp.headers.get("content-type") or default_type
+    if not ctype.lower().startswith(_SAFE_TYPES):
+        ctype = "application/octet-stream"
+    return Response(content=resp.content, media_type=ctype,
+                    headers={"X-Content-Type-Options": "nosniff"})
+
+
+class AudioHandler:
+    @staticmethod
+    async def handle_transcription(
+        api_key: str,
+        model: str,
+        audio: bytes,
+        filename: str,
+        content_type: Optional[str],
+        fields: Dict[str, str],
+        background_tasks: BackgroundTasks,
+        ip_address: Optional[str] = None,
+        sandbox: bool = False,
+    ):
+        if not model or not audio:
+            raise HTTPException(status_code=400, detail="Model and file are required")
+
+        async def send(ctx: RequestContext):
+            return await GatewayService.call_upstream_raw(
+                ctx.endpoint_url,
+                ctx.adapter.get_endpoint_path("transcription"),
+                ctx.provider_headers,
+                files={"file": (filename, audio, content_type or "application/octet-stream")},
+                data={**fields, "model": ctx.body["model"]},
+                concurrency_key=ctx.concurrency_key,
+            )
+
+        resp = await AudioHandler._handle(
+            api_key, {"model": model}, "transcription", send,
+            log_payload={"model": model, "file": filename, "bytes": len(audio), **fields},
+            background_tasks=background_tasks, ip_address=ip_address, sandbox=sandbox,
+        )
+        if "json" in resp.headers.get("content-type", ""):
+            data = resp.json()
+            if isinstance(data, dict) and "model" in data:
+                data["model"] = model
+            return data
+        return _safe_response(resp, "text/plain")
+
+    @staticmethod
+    async def handle_speech(
+        api_key: str,
+        body: Dict,
+        background_tasks: BackgroundTasks,
+        ip_address: Optional[str] = None,
+        sandbox: bool = False,
+    ):
+        if not body.get("model") or not body.get("input"):
+            raise HTTPException(status_code=400, detail="Model and input are required")
+        if len(str(body["input"])) > MAX_SPEECH_CHARS:
+            raise HTTPException(
+                status_code=400,
+                detail=f"Input too long (max {MAX_SPEECH_CHARS} characters)",
+            )
+
+        async def send(ctx: RequestContext):
+            return await GatewayService.call_upstream_raw(
+                ctx.endpoint_url,
+                ctx.adapter.get_endpoint_path("speech"),
+                ctx.provider_headers,
+                json_body=ctx.body,
+                concurrency_key=ctx.concurrency_key,
+            )
+
+        resp = await AudioHandler._handle(
+            api_key, body, "speech", send, log_payload=body,
+            background_tasks=background_tasks, ip_address=ip_address, sandbox=sandbox,
+        )
+        return _safe_response(resp, "audio/mpeg")
+
+    @staticmethod
+    async def _handle(
+        api_key, body, request_type, send, *, log_payload,
+        background_tasks, ip_address, sandbox,
+    ):
+        requested_model = body["model"]
+        ctx = RequestContext(
+            api_key=api_key,
+            body=dict(body),
+            model=requested_model,
+            sandbox=sandbox,
+            ip_address=ip_address,
+            background_tasks=background_tasks,
+            start_time=time.time(),
+        )
+        await Pipeline.resolve_context(ctx, model_type=MODEL_TYPE)
+        await Pipeline.check_rate_limit(ctx)
+        await Pipeline.check_quota(ctx)
+        Pipeline.resolve_provider(ctx, default_engine=DEFAULT_ENGINE)
+
+        status_code = 200
+        error_message = None
+        try:
+            return await send(ctx)
+        except HTTPException as e:
+            status_code = e.status_code
+            error_message = str(e.detail)
+            raise
+        except Exception as e:
+            status_code = 500
+            error_message = str(e)
+            raise
+        finally:
+            background_tasks.add_task(
+                RequestLogger.log,
+                deployment_id=ctx.deployment_id,
+                user_id=ctx.user_context_id,
+                org_id=ctx.org_id,
+                model=requested_model,
+                request_payload=log_payload,
+                start_time=ctx.start_time,
+                request_type=request_type,
+                applied_policies=ctx.applied_policies,
+                log_payloads=ctx.log_payloads,
+                ip_address=ip_address,
+                status_code=status_code,
+                error_message=error_message,
+            )

src/inference/core/orchestrator.py

@@ -9,6 +9,7 @@
 
 from inference.client import api_gateway_client
 from .handlers import (
+    AudioHandler,
     CompletionHandler,
     EmbeddingHandler,
     ImageHandler,
@@ -51,6 +52,10 @@ async def list_models(api_key: str, sandbox: bool = False):
     handle_image_edit = ImageHandler.handle_edit
     handle_image_variations = ImageHandler.handle_variations
 
+    # --- Audio ---
+    handle_transcription = AudioHandler.handle_transcription
+    handle_speech = AudioHandler.handle_speech
+
     # --- Video ---
     handle_video_generation = VideoHandler.handle_generation
     handle_video_edit = VideoHandler.handle_edit

src/inference/core/providers/base.py

@@ -78,7 +78,7 @@ def get_endpoint_path(self, request_type: str) -> str:
         Args:
             request_type: One of 'chat', 'embedding', 'image_generation',
                 'image_edit', 'image_variations', 'video_generation',
-                'video_edit', 'video_extension'.
+                'video_edit', 'video_extension', 'transcription', 'speech'.
         """
         paths = {
             "chat": self.get_chat_path(),
@@ -89,6 +89,8 @@ def get_endpoint_path(self, request_type: str) -> str:
             "video_generation": "/v1/videos/generations",
             "video_edit": "/v1/videos/edits",
             "video_extension": "/v1/videos/extensions",
+            "transcription": "/v1/audio/transcriptions",
+            "speech": "/v1/audio/speech",
         }
         return paths.get(request_type, self.get_chat_path())
 

src/inference/core/providers/registry.py

@@ -34,6 +34,7 @@
 EMBEDDING_ENGINES = {"infinity", "tei"}
 IMAGE_ENGINES = {"inferia-diffusion"}
 VIDEO_ENGINES = {"inferia-diffusion"}
+AUDIO_ENGINES = {"speaches"}
 
 # Backward-compat aliases
 EXTERNAL_ENGINES = EXTERNAL_PROVIDERS
@@ -71,6 +72,8 @@ def get_adapter(engine: str) -> ProviderAdapter:
         "vllm": ComputeAdapter(),
         "ollama": ComputeAdapter(),
         "generic": ComputeAdapter(),
+        # Speech engines (OpenAI-compatible /v1/audio/*)
+        "speaches": ComputeAdapter(),
         # Embedding engines (OpenAI-compatible for embeddings)
         "infinity": EmbeddingAdapter(),
         "tei": EmbeddingAdapter(),

src/inference/core/service.py

@@ -67,6 +67,8 @@ def _build_full_url(endpoint_url: str, chat_path: str) -> str:
             or endpoint.endswith("/videos/generations")
             or endpoint.endswith("/videos/edits")
             or endpoint.endswith("/videos/extensions")
+            or endpoint.endswith("/audio/transcriptions")
+            or endpoint.endswith("/audio/speech")
         ):
             return endpoint
 
@@ -171,6 +173,81 @@ async def stream_upstream(
             logger.error(f"Streaming Exception: {e}")
             yield _upstream_error(error_sink, 502, "Streaming connection failed")
 
+    @staticmethod
+    async def call_upstream_raw(
+        endpoint_url: str,
+        path: str,
+        headers: Dict,
+        *,
+        json_body: Optional[Dict] = None,
+        files: Optional[Dict] = None,
+        data: Optional[Dict] = None,
+        concurrency_key: str = "default",
+    ) -> httpx.Response:
+        """POST to the upstream and return the read response without parsing
+        it. For multipart uploads and binary bodies, which call_upstream's
+        JSON-in/JSON-out shape can't carry."""
+        full_url = GatewayService._build_full_url(endpoint_url, path)
+        try:
+            full_url = validate_upstream_url(
+                full_url, GatewayService._get_allowed_hosts()
+            )
+            headers = sanitize_headers(headers)
+        except ValueError as e:
+            raise HTTPException(status_code=400, detail=str(e))
+        if files:
+            # httpx sets the multipart boundary itself.
+            headers = {k: v for k, v in headers.items() if k.lower() != "content-type"}
+
+        breaker = circuit_breaker_registry.get_or_create(
+            f"upstream:{concurrency_key}",
+            failure_threshold=5,
+            recovery_timeout=30.0,
+            expected_exception=(httpx.HTTPStatusError, httpx.RequestError),
+        )
+        if not await breaker._can_execute():
+            raise HTTPException(
+                status_code=503,
+                detail="Upstream temporarily unavailable (circuit breaker open)",
+            )
+
+        max_bytes = settings.upstream_max_response_bytes
+        client = http_client.get_client()
+        try:
+            async with upstream_concurrency_limiter.limit(concurrency_key):
+                resp = await client.post(
+                    full_url,
+                    json=json_body,
+                    files=files,
+                    data=data,
+                    headers=headers,
+                    timeout=settings.upstream_http_timeout_seconds,
+                )
+                resp.raise_for_status()
+            if len(resp.content) > max_bytes:
+                raise HTTPException(
+                    status_code=502, detail="Upstream response exceeded size limit"
+                )
+            await breaker._record_success()
+            return resp
+        except HTTPException:
+            raise
+        except httpx.HTTPStatusError as e:
+            # A 4xx is about this request, not the upstream's health.
+            if e.response.status_code >= 500:
+                await breaker._record_failure()
+            logger.error(f"Provider error {e.response.status_code}: {e.response.text}")
+            raise HTTPException(
+                status_code=e.response.status_code,
+                detail="Upstream provider returned an error",
+            )
+        except Exception as e:
+            await breaker._record_failure()
+            logger.error(f"Provider request failed: {e}")
+            raise HTTPException(
+                status_code=502, detail="Upstream provider is unavailable"
+            )
+
     @staticmethod
     async def call_upstream(
         endpoint_url: str,

src/orchestration/models/model_deployment/deployment_server.py

@@ -4150,6 +4150,7 @@ async def list_all_deployments(
             "engine": d.engine,
             "endpoint": d.endpoint,
             "org_id": d.org_id,
+            "model_type": d.model_type or "inference",
             "error_message": d.error_message or None,
         }
         for d in resp.deployments