diff --git a/apps/api/src/opendirector_api/localization.py b/apps/api/src/opendirector_api/localization.py index 5018e414..b652b70b 100644 --- a/apps/api/src/opendirector_api/localization.py +++ b/apps/api/src/opendirector_api/localization.py @@ -88,6 +88,7 @@ "bigmodel_image_size": "GLM-Image requires a preset size, or dimensions of 1024–2048 in multiples of 32.", "moss_layout_reference": "MOSS layout JSON does not accept additional text references. Put the requirements in the JSON or remove the text references.", "agent_dependency_unavailable": "Bundled Agent is unavailable. Reinstall with pip install --force-reinstall opendirector; for a source checkout, use pip install -e .", + "model_read_timeout": "The model did not respond in time, so this turn has ended. Try again later, or check that the model service is available.", "invalid_input": "The request is invalid. Check the required fields and their formats.", "invalid_project": "The project or asset identifier is invalid.", "missing_object": "The requested item no longer exists.", diff --git a/packages/core/src/opendirector/agent/model_gateway.py b/packages/core/src/opendirector/agent/model_gateway.py index a87c89f6..009dc05b 100644 --- a/packages/core/src/opendirector/agent/model_gateway.py +++ b/packages/core/src/opendirector/agent/model_gateway.py @@ -21,16 +21,42 @@ logger = logging.getLogger(__name__) +DEFAULT_MODEL_READ_TIMEOUT = 120.0 +UNANSWERED_MODEL_ERROR = "模型在限定时间内没有返回数据,本轮已结束。请稍后重试,或检查模型服务是否可用。" + + +def model_read_timeout(raw: str | None = None) -> float: + """Seconds allowed for the whole upstream read. Default 120; mid-stream stalls count.""" + value = os.environ.get("OPENDIRECTOR_MODEL_READ_TIMEOUT") if raw is None else raw + if value is None: + return DEFAULT_MODEL_READ_TIMEOUT + try: + seconds = float(value) + except (TypeError, ValueError): + return DEFAULT_MODEL_READ_TIMEOUT + if seconds <= 0 or seconds != seconds or seconds == float("inf"): + return DEFAULT_MODEL_READ_TIMEOUT + return seconds + + +def unanswered_model_failure(error_name: str) -> ProjectError: + logger.info("Ending the turn after an unanswered model request (%s)", error_name) + return ProjectError(UNANSWERED_MODEL_ERROR, 504, code="model_read_timeout") + class ModelGateway: """Loopback-only OpenAI transport for the bundled runtime's configured model.""" def __init__(self, config: dict, thinking: str, token: str, project_id: str, turn_id: str, transport: httpx.AsyncBaseTransport | None = None, - *, trace_path: Path | None = None): + *, trace_path: Path | None = None, + read_timeout: float | None = None, + on_unanswered_failure: object | None = None): self.config, self.thinking, self.token = config, thinking, token self.project_id, self.turn_id, self.transport = project_id, turn_id, transport self.trace_path = trace_path + self.read_timeout = model_read_timeout() if read_timeout is None else read_timeout + self.on_unanswered_failure = on_unanswered_failure self.active: set[asyncio.Task] = set() self.closed = False @@ -120,12 +146,12 @@ async def request( url = self.config["url"].rstrip("/") + "/chat/completions" request_id = secrets.token_hex(8) self.trace("model.request", request_id, payload=payload) - async with httpx.AsyncClient( - transport=self.transport, - event_hooks=MODEL_HTTP_HOOKS, - trust_env=trust_environment(url, config=self.config), - timeout=httpx.Timeout(120, connect=15), - ) as client: + async with httpx.AsyncClient( + transport=self.transport, + event_hooks=MODEL_HTTP_HOOKS, + trust_env=trust_environment(url, config=self.config), + timeout=httpx.Timeout(self.read_timeout, connect=15), + ) as client: async with client.stream( "POST", url, @@ -168,12 +194,12 @@ async def request( event_lines.clear() if data != "[DONE]": self.trace_reasoning(request_id, data) - # Only successful exhaustion and cleanup may terminate this response. - # Sending this in finally would disguise a truncated body as complete. - writer.write(b"0\r\n\r\n") - await writer.drain() - self.trace("model.completed", request_id, forwardedBytes=forwarded_bytes, - elapsedMs=round((monotonic() - started) * 1000)) + # Only successful exhaustion and cleanup may terminate this response. + # Sending this in finally would disguise a truncated body as complete. + writer.write(b"0\r\n\r\n") + await writer.drain() + self.trace("model.completed", request_id, forwardedBytes=forwarded_bytes, + elapsedMs=round((monotonic() - started) * 1000)) except asyncio.CancelledError: if request_id is not None: self.trace("model.cancelled", request_id, forwardedBytes=forwarded_bytes) @@ -192,6 +218,9 @@ async def request( if request_id is not None: self.trace("model.error", request_id, error=type(failure).__name__, forwardedBytes=forwarded_bytes) + if forwarded_bytes == 0 and self.on_unanswered_failure is not None: + with suppress(Exception): + self.on_unanswered_failure(type(failure).__name__) logger.error( f"agent.model.error turn_id={self.turn_id} failure={type(failure).__name__}" ) diff --git a/packages/core/src/opendirector/agent/native.py b/packages/core/src/opendirector/agent/native.py index e9f6beab..680edb96 100644 --- a/packages/core/src/opendirector/agent/native.py +++ b/packages/core/src/opendirector/agent/native.py @@ -12,7 +12,7 @@ from urllib.request import getproxies from uuid import UUID -from opendirector.agent.model_gateway import ModelGateway +from opendirector.agent.model_gateway import ModelGateway, unanswered_model_failure from opendirector.agent import cli_transport, deepseek from opendirector.agent.cli_discovery import cli_executable from opendirector.agent.context import RunContext @@ -223,10 +223,21 @@ async def stream(self, run: RunContext, gateway: ToolGateway): gateway.roots = roots endpoint = str(Path(temporary) / 'cli.sock') server, bridge_env = await cli_transport.start_server(gateway, queue, endpoint, pending) + unanswered: list[str] = [] + def report_unanswered(error_name: str) -> None: + if unanswered: + return + unanswered.append(error_name) + if model is not None: + model.closed = True + current = process + if current is not None: + pending.add(asyncio.create_task(terminate(current, group=True))) if run.runtime == 'default': model = ModelGateway( config, thinking, token, str(project_id), str(turn_id), self.transport, trace_path=work.parent.with_suffix('.model.jsonl'), + on_unanswered_failure=report_unanswered, ) model_server = await asyncio.start_server(model.connect, '127.0.0.1', 0) if model else None model_url = f'http://127.0.0.1:{model_server.sockets[0].getsockname()[1]}' if model_server else '' @@ -323,7 +334,11 @@ async def observe(raw): await queue.put({'native': raw}) if await process.wait(): await record_exit() + if unanswered: + raise unanswered_model_failure(unanswered[0]) raise ProjectError(f'{runtime} 进程退出({process.returncode}),请检查 CLI 登录和配置。', 502) + if unanswered: + raise unanswered_model_failure(unanswered[0]) if reported_session: run.save_native_session(reported_session, selection, engine_directory=native_work, diff --git a/tests/test_model_gateway.py b/tests/test_model_gateway.py index 14d5131e..8b566c42 100644 --- a/tests/test_model_gateway.py +++ b/tests/test_model_gateway.py @@ -1,5 +1,6 @@ """Exercise HTTP framing and interrupted model transport with real loopback clients.""" import asyncio +import contextlib from contextlib import asynccontextmanager import json @@ -185,3 +186,93 @@ async def run(): assert await response.aread() == b'' assert records(tmp_path)[-1]['forwardedBytes'] == 0 asyncio.run(run()) + +def test_model_read_timeout_uses_default_and_rejects_invalid(monkeypatch): + from opendirector.agent.model_gateway import DEFAULT_MODEL_READ_TIMEOUT, model_read_timeout + monkeypatch.delenv('OPENDIRECTOR_MODEL_READ_TIMEOUT', raising=False) + assert model_read_timeout() == DEFAULT_MODEL_READ_TIMEOUT + assert model_read_timeout('0.25') == 0.25 + for raw in ('', '0', '-1', 'abc', 'nan', 'inf', '-inf'): + assert model_read_timeout(raw) == DEFAULT_MODEL_READ_TIMEOUT + monkeypatch.setenv('OPENDIRECTOR_MODEL_READ_TIMEOUT', '7.5') + assert model_read_timeout() == 7.5 + monkeypatch.setenv('OPENDIRECTOR_MODEL_READ_TIMEOUT', 'nope') + assert model_read_timeout() == DEFAULT_MODEL_READ_TIMEOUT + + +def test_read_timeout_before_first_byte_is_a_bounded_error(tmp_path): + notified = [] + + async def run(): + release = asyncio.Event() + + async def silent(reader, writer): + try: + await release.wait() + finally: + writer.close() + with contextlib.suppress(Exception): + await writer.wait_closed() + + upstream = await asyncio.start_server(silent, '127.0.0.1', 0) + port = upstream.sockets[0].getsockname()[1] + started = asyncio.get_running_loop().time() + try: + gateway = ModelGateway( + {'modelId': 'fixture', 'url': f'http://127.0.0.1:{port}', 'authMode': 'none'}, + 'auto', 'local-token', 'project', 'turn', None, + trace_path=tmp_path / 'model.jsonl', + read_timeout=0.2, + on_unanswered_failure=notified.append, + ) + def connect(reader, writer): + gateway.connect(reader, writer) + server = await asyncio.start_server(connect, '127.0.0.1', 0) + url = f'http://127.0.0.1:{server.sockets[0].getsockname()[1]}/chat/completions' + try: + async with httpx.AsyncClient(trust_env=False, timeout=3) as client: + async with request(client, url) as response: + assert response.status_code == 502 + assert await response.aread() == b'' + finally: + server.close() + await server.wait_closed() + tasks = list(gateway.active) + for task in tasks: + task.cancel() + await asyncio.gather(*tasks, return_exceptions=True) + finally: + release.set() + upstream.close() + await upstream.wait_closed() + elapsed = asyncio.get_running_loop().time() - started + rows = records(tmp_path) + assert rows[-1]['event'] == 'model.error' + assert rows[-1]['error'] == 'ReadTimeout' + assert rows[-1]['forwardedBytes'] == 0 + assert not any(row['event'] == 'model.completed' for row in rows) + assert notified == ['ReadTimeout'] + assert 0.15 <= elapsed < 2 + + asyncio.run(run()) + + +def test_mid_stream_timeout_does_not_report_an_unanswered_failure(tmp_path): + notified = [] + + async def run(): + stream = Stream([b'partial'], failure=httpx.ReadTimeout('private')) + async with gateway_server(tmp_path, lambda _: httpx.Response( + 200, headers={'content-type': 'text/event-stream'}, stream=stream)) as (gateway, url): + gateway.on_unanswered_failure = notified.append + async with httpx.AsyncClient(trust_env=False, timeout=3) as client: + async with request(client, url) as response: + received = [] + with pytest.raises(httpx.RemoteProtocolError): + async for chunk in response.aiter_bytes(): + received.append(chunk) + assert b''.join(received) == b'partial' + assert records(tmp_path)[-1]['event'] == 'model.error' + assert notified == [] + + asyncio.run(run()) diff --git a/tests/test_native_runtime.py b/tests/test_native_runtime.py index 280491a3..4df2dde8 100644 --- a/tests/test_native_runtime.py +++ b/tests/test_native_runtime.py @@ -660,3 +660,104 @@ def test_codex_resume_requires_a_persisted_rollout(tmp_path, monkeypatch): rollout.parent.mkdir(parents=True) rollout.write_text('{}\n') assert native.codex_session_available(session) + +RETRYING_CLI = r""" +import json, os, sys, time, urllib.error, urllib.request +from pathlib import Path +sys.stdin.read() +config = json.loads(os.environ['OPENCODE_CONFIG_CONTENT']) +base = config['provider']['opendirector']['options']['baseURL'] +key = os.environ['OPENDIRECTOR_MODEL_KEY'] +count = Path(os.environ['REQUEST_COUNT']) +payload = json.dumps({'messages': [{'role': 'user', 'content': 'hello'}], 'stream': True}).encode() +for attempt in range(6): + count.write_text(str(attempt + 1)) + request = urllib.request.Request( + base + '/chat/completions', data=payload, + headers={'Authorization': f'Bearer {key}', 'Content-Type': 'application/json'}, + method='POST') + try: + with urllib.request.urlopen(request, timeout=30) as response: + response.read() + print(json.dumps({'type': 'text', 'sessionID': 'timeout-session', 'part': {'text': 'should not appear'}})) + raise SystemExit(0) + except Exception: + if attempt < 5: + time.sleep(2) +time.sleep(300) +""" + + +def test_unanswered_model_timeout_ends_the_turn_without_silent_retries(tmp_path, monkeypatch): + """A first-byte timeout must end the live turn instead of letting the harness retry.""" + monkeypatch.setenv('OPENDIRECTOR_MODEL_READ_TIMEOUT', '0.3') + count = tmp_path / 'requests.txt' + count.write_text('0') + monkeypatch.setenv('REQUEST_COUNT', str(count)) + script = tmp_path / 'retrying_cli.py' + script.write_text(RETRYING_CLI) + real_command = native.command + + def command(kind, work, socket, token, model, model_url, catalog, **kwargs): + _, env = real_command(kind, work, socket, token, model, model_url, catalog, **kwargs) + return [sys.executable, str(script)], env + + monkeypatch.setattr(native, 'command', command) + monkeypatch.setattr(native, 'executable', lambda _: sys.executable) + async def no_skills(*args): + return [] + monkeypatch.setattr('opendirector_api.agent.skills', no_skills) + + server = socket.socket(); server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + server.bind(('127.0.0.1', 0)); server.listen() + stop = threading.Event() + # Accept connections but never reply, so the gateway read timeout can fire. + def serve(): + try: + while not stop.is_set(): + try: + server.settimeout(0.2) + connection, _ = server.accept() + except TimeoutError: + continue + except OSError: + break + try: + while not stop.is_set(): + time.sleep(0.05) + finally: + connection.close() + except Exception: + pass + thread = threading.Thread(target=serve, daemon=True) + thread.start() + try: + port = server.getsockname()[1] + local = LocalProjects(tmp_path) + config = ModelConfigs(tmp_path).save( + 'agent', f'http://127.0.0.1:{port}', 'slow-model', 'server-only-key', auth_mode='none') + started = time.monotonic() + with TestClient(create_app(local), headers={ + 'X-OpenDirector-Request': '1', 'X-OpenDirector-Session': str(uuid4()), + 'Accept-Language': 'en-US'}) as client: + path = project(client) + turn = {'id': str(uuid4()), 'runtime': 'default', 'modelId': config['id'], 'text': 'hello'} + response = client.post(path + '/assistant/turns', json=turn) + elapsed = time.monotonic() - started + assert response.status_code == 200 + events = [json.loads(line[6:]) for line in response.text.splitlines() if line.startswith('data: ')] + assert events, response.text + ending = events[-1] + assert ending['type'] == 'end', events + assert ending['error'] + assert 'did not respond in time' in ending['error'] + conversation = ending['conversation'] + assert conversation['messages'][-1]['status'] == 'interrupted' + assert conversation['messages'][-1]['text'] == '' + saved = client.get(path + '/assistant').json() + assert saved['messages'][-1]['status'] == 'interrupted' + assert elapsed < 8 + assert int(count.read_text() or '0') == 1 + finally: + stop.set() + server.close()