Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions apps/api/src/opendirector_api/localization.py
Original file line number Diff line number Diff line change
Expand Up @@ -88,6 +88,7 @@
"bigmodel_image_size": "GLM-Image requires a preset size, or dimensions of 1024–2048 in multiples of 32.",
"moss_layout_reference": "MOSS layout JSON does not accept additional text references. Put the requirements in the JSON or remove the text references.",
"agent_dependency_unavailable": "Bundled Agent is unavailable. Reinstall with pip install --force-reinstall opendirector; for a source checkout, use pip install -e .",
"model_read_timeout": "The model did not respond in time, so this turn has ended. Try again later, or check that the model service is available.",
"invalid_input": "The request is invalid. Check the required fields and their formats.",
"invalid_project": "The project or asset identifier is invalid.",
"missing_object": "The requested item no longer exists.",
Expand Down
55 changes: 42 additions & 13 deletions packages/core/src/opendirector/agent/model_gateway.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,16 +21,42 @@

logger = logging.getLogger(__name__)

DEFAULT_MODEL_READ_TIMEOUT = 120.0
UNANSWERED_MODEL_ERROR = "模型在限定时间内没有返回数据,本轮已结束。请稍后重试,或检查模型服务是否可用。"


def model_read_timeout(raw: str | None = None) -> float:
"""Seconds allowed for the whole upstream read. Default 120; mid-stream stalls count."""
value = os.environ.get("OPENDIRECTOR_MODEL_READ_TIMEOUT") if raw is None else raw
if value is None:
return DEFAULT_MODEL_READ_TIMEOUT
try:
seconds = float(value)
except (TypeError, ValueError):
return DEFAULT_MODEL_READ_TIMEOUT
if seconds <= 0 or seconds != seconds or seconds == float("inf"):
return DEFAULT_MODEL_READ_TIMEOUT
return seconds


def unanswered_model_failure(error_name: str) -> ProjectError:
logger.info("Ending the turn after an unanswered model request (%s)", error_name)
return ProjectError(UNANSWERED_MODEL_ERROR, 504, code="model_read_timeout")


class ModelGateway:
"""Loopback-only OpenAI transport for the bundled runtime's configured model."""

def __init__(self, config: dict, thinking: str, token: str, project_id: str,
turn_id: str, transport: httpx.AsyncBaseTransport | None = None,
*, trace_path: Path | None = None):
*, trace_path: Path | None = None,
read_timeout: float | None = None,
on_unanswered_failure: object | None = None):
self.config, self.thinking, self.token = config, thinking, token
self.project_id, self.turn_id, self.transport = project_id, turn_id, transport
self.trace_path = trace_path
self.read_timeout = model_read_timeout() if read_timeout is None else read_timeout
self.on_unanswered_failure = on_unanswered_failure
self.active: set[asyncio.Task] = set()
self.closed = False

Expand Down Expand Up @@ -120,12 +146,12 @@ async def request(
url = self.config["url"].rstrip("/") + "/chat/completions"
request_id = secrets.token_hex(8)
self.trace("model.request", request_id, payload=payload)
async with httpx.AsyncClient(
transport=self.transport,
event_hooks=MODEL_HTTP_HOOKS,
trust_env=trust_environment(url, config=self.config),
timeout=httpx.Timeout(120, connect=15),
) as client:
async with httpx.AsyncClient(
transport=self.transport,
event_hooks=MODEL_HTTP_HOOKS,
trust_env=trust_environment(url, config=self.config),
timeout=httpx.Timeout(self.read_timeout, connect=15),
) as client:
async with client.stream(
"POST",
url,
Expand Down Expand Up @@ -168,12 +194,12 @@ async def request(
event_lines.clear()
if data != "[DONE]":
self.trace_reasoning(request_id, data)
# Only successful exhaustion and cleanup may terminate this response.
# Sending this in finally would disguise a truncated body as complete.
writer.write(b"0\r\n\r\n")
await writer.drain()
self.trace("model.completed", request_id, forwardedBytes=forwarded_bytes,
elapsedMs=round((monotonic() - started) * 1000))
# Only successful exhaustion and cleanup may terminate this response.
# Sending this in finally would disguise a truncated body as complete.
writer.write(b"0\r\n\r\n")
await writer.drain()
self.trace("model.completed", request_id, forwardedBytes=forwarded_bytes,
elapsedMs=round((monotonic() - started) * 1000))
except asyncio.CancelledError:
if request_id is not None:
self.trace("model.cancelled", request_id, forwardedBytes=forwarded_bytes)
Expand All @@ -192,6 +218,9 @@ async def request(
if request_id is not None:
self.trace("model.error", request_id, error=type(failure).__name__,
forwardedBytes=forwarded_bytes)
if forwarded_bytes == 0 and self.on_unanswered_failure is not None:
with suppress(Exception):
self.on_unanswered_failure(type(failure).__name__)
logger.error(
f"agent.model.error turn_id={self.turn_id} failure={type(failure).__name__}"
)
Expand Down
17 changes: 16 additions & 1 deletion packages/core/src/opendirector/agent/native.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
from urllib.request import getproxies
from uuid import UUID

from opendirector.agent.model_gateway import ModelGateway
from opendirector.agent.model_gateway import ModelGateway, unanswered_model_failure
from opendirector.agent import cli_transport, deepseek
from opendirector.agent.cli_discovery import cli_executable
from opendirector.agent.context import RunContext
Expand Down Expand Up @@ -223,10 +223,21 @@ async def stream(self, run: RunContext, gateway: ToolGateway):
gateway.roots = roots
endpoint = str(Path(temporary) / 'cli.sock')
server, bridge_env = await cli_transport.start_server(gateway, queue, endpoint, pending)
unanswered: list[str] = []
def report_unanswered(error_name: str) -> None:
if unanswered:
return
unanswered.append(error_name)
if model is not None:
model.closed = True
current = process
if current is not None:
pending.add(asyncio.create_task(terminate(current, group=True)))
if run.runtime == 'default':
model = ModelGateway(
config, thinking, token, str(project_id), str(turn_id), self.transport,
trace_path=work.parent.with_suffix('.model.jsonl'),
on_unanswered_failure=report_unanswered,
)
model_server = await asyncio.start_server(model.connect, '127.0.0.1', 0) if model else None
model_url = f'http://127.0.0.1:{model_server.sockets[0].getsockname()[1]}' if model_server else ''
Expand Down Expand Up @@ -323,7 +334,11 @@ async def observe(raw):
await queue.put({'native': raw})
if await process.wait():
await record_exit()
if unanswered:
raise unanswered_model_failure(unanswered[0])
raise ProjectError(f'{runtime} 进程退出({process.returncode}),请检查 CLI 登录和配置。', 502)
if unanswered:
raise unanswered_model_failure(unanswered[0])
if reported_session:
run.save_native_session(reported_session, selection,
engine_directory=native_work,
Expand Down
91 changes: 91 additions & 0 deletions tests/test_model_gateway.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
"""Exercise HTTP framing and interrupted model transport with real loopback clients."""
import asyncio
import contextlib
from contextlib import asynccontextmanager
import json

Expand Down Expand Up @@ -185,3 +186,93 @@ async def run():
assert await response.aread() == b''
assert records(tmp_path)[-1]['forwardedBytes'] == 0
asyncio.run(run())

def test_model_read_timeout_uses_default_and_rejects_invalid(monkeypatch):
from opendirector.agent.model_gateway import DEFAULT_MODEL_READ_TIMEOUT, model_read_timeout
monkeypatch.delenv('OPENDIRECTOR_MODEL_READ_TIMEOUT', raising=False)
assert model_read_timeout() == DEFAULT_MODEL_READ_TIMEOUT
assert model_read_timeout('0.25') == 0.25
for raw in ('', '0', '-1', 'abc', 'nan', 'inf', '-inf'):
assert model_read_timeout(raw) == DEFAULT_MODEL_READ_TIMEOUT
monkeypatch.setenv('OPENDIRECTOR_MODEL_READ_TIMEOUT', '7.5')
assert model_read_timeout() == 7.5
monkeypatch.setenv('OPENDIRECTOR_MODEL_READ_TIMEOUT', 'nope')
assert model_read_timeout() == DEFAULT_MODEL_READ_TIMEOUT


def test_read_timeout_before_first_byte_is_a_bounded_error(tmp_path):
notified = []

async def run():
release = asyncio.Event()

async def silent(reader, writer):
try:
await release.wait()
finally:
writer.close()
with contextlib.suppress(Exception):
await writer.wait_closed()

upstream = await asyncio.start_server(silent, '127.0.0.1', 0)
port = upstream.sockets[0].getsockname()[1]
started = asyncio.get_running_loop().time()
try:
gateway = ModelGateway(
{'modelId': 'fixture', 'url': f'http://127.0.0.1:{port}', 'authMode': 'none'},
'auto', 'local-token', 'project', 'turn', None,
trace_path=tmp_path / 'model.jsonl',
read_timeout=0.2,
on_unanswered_failure=notified.append,
)
def connect(reader, writer):
gateway.connect(reader, writer)
server = await asyncio.start_server(connect, '127.0.0.1', 0)
url = f'http://127.0.0.1:{server.sockets[0].getsockname()[1]}/chat/completions'
try:
async with httpx.AsyncClient(trust_env=False, timeout=3) as client:
async with request(client, url) as response:
assert response.status_code == 502
assert await response.aread() == b''
finally:
server.close()
await server.wait_closed()
tasks = list(gateway.active)
for task in tasks:
task.cancel()
await asyncio.gather(*tasks, return_exceptions=True)
finally:
release.set()
upstream.close()
await upstream.wait_closed()
elapsed = asyncio.get_running_loop().time() - started
rows = records(tmp_path)
assert rows[-1]['event'] == 'model.error'
assert rows[-1]['error'] == 'ReadTimeout'
assert rows[-1]['forwardedBytes'] == 0
assert not any(row['event'] == 'model.completed' for row in rows)
assert notified == ['ReadTimeout']
assert 0.15 <= elapsed < 2

asyncio.run(run())


def test_mid_stream_timeout_does_not_report_an_unanswered_failure(tmp_path):
notified = []

async def run():
stream = Stream([b'partial'], failure=httpx.ReadTimeout('private'))
async with gateway_server(tmp_path, lambda _: httpx.Response(
200, headers={'content-type': 'text/event-stream'}, stream=stream)) as (gateway, url):
gateway.on_unanswered_failure = notified.append
async with httpx.AsyncClient(trust_env=False, timeout=3) as client:
async with request(client, url) as response:
received = []
with pytest.raises(httpx.RemoteProtocolError):
async for chunk in response.aiter_bytes():
received.append(chunk)
assert b''.join(received) == b'partial'
assert records(tmp_path)[-1]['event'] == 'model.error'
assert notified == []

asyncio.run(run())
101 changes: 101 additions & 0 deletions tests/test_native_runtime.py
Original file line number Diff line number Diff line change
Expand Up @@ -660,3 +660,104 @@ def test_codex_resume_requires_a_persisted_rollout(tmp_path, monkeypatch):
rollout.parent.mkdir(parents=True)
rollout.write_text('{}\n')
assert native.codex_session_available(session)

RETRYING_CLI = r"""
import json, os, sys, time, urllib.error, urllib.request
from pathlib import Path
sys.stdin.read()
config = json.loads(os.environ['OPENCODE_CONFIG_CONTENT'])
base = config['provider']['opendirector']['options']['baseURL']
key = os.environ['OPENDIRECTOR_MODEL_KEY']
count = Path(os.environ['REQUEST_COUNT'])
payload = json.dumps({'messages': [{'role': 'user', 'content': 'hello'}], 'stream': True}).encode()
for attempt in range(6):
count.write_text(str(attempt + 1))
request = urllib.request.Request(
base + '/chat/completions', data=payload,
headers={'Authorization': f'Bearer {key}', 'Content-Type': 'application/json'},
method='POST')
try:
with urllib.request.urlopen(request, timeout=30) as response:
response.read()
print(json.dumps({'type': 'text', 'sessionID': 'timeout-session', 'part': {'text': 'should not appear'}}))
raise SystemExit(0)
except Exception:
if attempt < 5:
time.sleep(2)
time.sleep(300)
"""


def test_unanswered_model_timeout_ends_the_turn_without_silent_retries(tmp_path, monkeypatch):
"""A first-byte timeout must end the live turn instead of letting the harness retry."""
monkeypatch.setenv('OPENDIRECTOR_MODEL_READ_TIMEOUT', '0.3')
count = tmp_path / 'requests.txt'
count.write_text('0')
monkeypatch.setenv('REQUEST_COUNT', str(count))
script = tmp_path / 'retrying_cli.py'
script.write_text(RETRYING_CLI)
real_command = native.command

def command(kind, work, socket, token, model, model_url, catalog, **kwargs):
_, env = real_command(kind, work, socket, token, model, model_url, catalog, **kwargs)
return [sys.executable, str(script)], env

monkeypatch.setattr(native, 'command', command)
monkeypatch.setattr(native, 'executable', lambda _: sys.executable)
async def no_skills(*args):
return []
monkeypatch.setattr('opendirector_api.agent.skills', no_skills)

server = socket.socket(); server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
server.bind(('127.0.0.1', 0)); server.listen()
stop = threading.Event()
# Accept connections but never reply, so the gateway read timeout can fire.
def serve():
try:
while not stop.is_set():
try:
server.settimeout(0.2)
connection, _ = server.accept()
except TimeoutError:
continue
except OSError:
break
try:
while not stop.is_set():
time.sleep(0.05)
finally:
connection.close()
except Exception:
pass
thread = threading.Thread(target=serve, daemon=True)
thread.start()
try:
port = server.getsockname()[1]
local = LocalProjects(tmp_path)
config = ModelConfigs(tmp_path).save(
'agent', f'http://127.0.0.1:{port}', 'slow-model', 'server-only-key', auth_mode='none')
started = time.monotonic()
with TestClient(create_app(local), headers={
'X-OpenDirector-Request': '1', 'X-OpenDirector-Session': str(uuid4()),
'Accept-Language': 'en-US'}) as client:
path = project(client)
turn = {'id': str(uuid4()), 'runtime': 'default', 'modelId': config['id'], 'text': 'hello'}
response = client.post(path + '/assistant/turns', json=turn)
elapsed = time.monotonic() - started
assert response.status_code == 200
events = [json.loads(line[6:]) for line in response.text.splitlines() if line.startswith('data: ')]
assert events, response.text
ending = events[-1]
assert ending['type'] == 'end', events
assert ending['error']
assert 'did not respond in time' in ending['error']
conversation = ending['conversation']
assert conversation['messages'][-1]['status'] == 'interrupted'
assert conversation['messages'][-1]['text'] == ''
saved = client.get(path + '/assistant').json()
assert saved['messages'][-1]['status'] == 'interrupted'
assert elapsed < 8
assert int(count.read_text() or '0') == 1
finally:
stop.set()
server.close()
Loading