From 9b383424e21b5b5b98024e7dc759e7c065b70c10 Mon Sep 17 00:00:00 2001 From: HSYZhang <2908064130@qq.com> Date: Tue, 17 Nov 2026 10:12:57 +0800 Subject: [PATCH] [fix] Retry empty metadata fetches in client Signed-off-by: HSYZhang <2908064130@qq.com> --- scripts/test_controller_fetch_progress.sh | 21 ++++++ tests/test_controller_fetch_progress.py | 82 +++++++++++++++++++++++ transfer_queue/client.py | 54 ++++++++++----- 3 files changed, 141 insertions(+), 16 deletions(-) create mode 100755 scripts/test_controller_fetch_progress.sh create mode 100644 tests/test_controller_fetch_progress.py diff --git a/scripts/test_controller_fetch_progress.sh b/scripts/test_controller_fetch_progress.sh new file mode 100755 index 00000000..985e3d8a --- /dev/null +++ b/scripts/test_controller_fetch_progress.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +# Copyright 2025 The TransferQueue Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -euo pipefail + +cd "$(dirname "$0")/.." +python_bin="${PYTHON:-python3}" +"$python_bin" -m compileall -q transfer_queue tutorial tests +"$python_bin" -m pytest -q tests/test_controller_fetch_progress.py diff --git a/tests/test_controller_fetch_progress.py b/tests/test_controller_fetch_progress.py new file mode 100644 index 00000000..1a2bfada --- /dev/null +++ b/tests/test_controller_fetch_progress.py @@ -0,0 +1,82 @@ +# Copyright 2025 The TransferQueue Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import asyncio +import time +from uuid import uuid4 + +import pytest +import ray +import torch +from omegaconf import OmegaConf +from tensordict import TensorDict + +import transfer_queue as tq +import transfer_queue.client as client_module + + +@pytest.mark.asyncio +async def test_waiting_fetch_allows_put_and_notification(monkeypatch): + namespace = f"tq-fetch-progress-{uuid4().hex}" + monkeypatch.setenv("TQ_RAY_NAMESPACE", namespace) + ray.init(address="local", namespace=namespace, num_cpus=3, include_dashboard=False) + try: + tq.init(OmegaConf.create({"backend": {"SimpleStorage": {"num_data_storage_units": 1}}})) + client = tq.get_client() + client.controller_rpc_pool._maxsize = 1 + monkeypatch.setattr(client_module, "TQ_CONTROLLER_GET_METADATA_TIMEOUT", 4) + monkeypatch.setattr(client_module, "TQ_CONTROLLER_GET_METADATA_CHECK_INTERVAL", 1) + + reader = asyncio.create_task( + asyncio.to_thread( + client.get_meta, + data_fields=["input_ids"], + batch_size=1, + partition_id="waiting_fetch", + task_name="trainer", + ) + ) + await asyncio.sleep(0.2) + data = TensorDict({"input_ids": torch.tensor([[1, 2]])}, batch_size=[1]) + await asyncio.wait_for(asyncio.to_thread(client.put, data, partition_id="waiting_fetch"), timeout=3) + metadata = await asyncio.wait_for(reader, timeout=5) + result = await asyncio.to_thread(client.get_data, metadata) + assert metadata.is_ready + assert torch.equal(result["input_ids"][0], data["input_ids"][0]) + + monkeypatch.setattr(client_module, "TQ_CONTROLLER_GET_METADATA_TIMEOUT", 0.25) + monkeypatch.setattr(client_module, "TQ_CONTROLLER_GET_METADATA_CHECK_INTERVAL", 5) + request_count = 0 + original_request = client._request_controller + + async def count_requests(*args, **kwargs): + nonlocal request_count + request_count += 1 + return await original_request(*args, **kwargs) + + monkeypatch.setattr(client, "_request_controller", count_requests) + started = time.monotonic() + with pytest.raises(TimeoutError, match="Timeout while waiting for sufficient data"): + await asyncio.to_thread( + client.get_meta, + data_fields=["input_ids"], + batch_size=1, + partition_id="never_produced", + task_name="trainer", + ) + assert time.monotonic() - started < 2 + assert request_count == 1 + finally: + tq.close() + ray.shutdown() diff --git a/transfer_queue/client.py b/transfer_queue/client.py index d44a0544..0cebcb6a 100644 --- a/transfer_queue/client.py +++ b/transfer_queue/client.py @@ -16,6 +16,7 @@ import asyncio import os import threading +import time import weakref from typing import Any, Callable @@ -53,6 +54,10 @@ # A DEALER drops a reply it can no longer route, so without a bound a lost response blocks # the caller forever. Generous: a busy controller is normal, an unanswered request is not. TQ_CONTROLLER_SEND_RECV_TIMEOUT = int(os.environ.get("TQ_CONTROLLER_SEND_RECV_TIMEOUT", 200)) # seconds +# Fetch requests retry empty metadata responses on the client so the controller's +# request loop remains free to process producer notifications. +TQ_CONTROLLER_GET_METADATA_TIMEOUT = int(os.environ.get("TQ_CONTROLLER_GET_METADATA_TIMEOUT", 1)) +TQ_CONTROLLER_GET_METADATA_CHECK_INTERVAL = int(os.environ.get("TQ_CONTROLLER_GET_METADATA_CHECK_INTERVAL", 5)) # Pre-bound decorator for controller socket operations. with_controller_socket = with_zmq_socket( @@ -239,7 +244,6 @@ async def _request_controller( return response_msg # ==================== Basic API ==================== - @with_controller_socket async def async_get_meta( self, data_fields: list[str], @@ -248,7 +252,6 @@ async def async_get_meta( mode: str = "fetch", task_name: str | None = None, sampling_config: dict[str, Any] | None = None, - socket: zmq.asyncio.Socket | None = None, ) -> BatchMeta: """Asynchronously fetch data metadata from the controller via ZMQ. @@ -262,7 +265,6 @@ async def async_get_meta( - 'insert': Internal usage - should not be used by users task_name: Optional task name associated with the request sampling_config: Optional sampling configuration for custom samplers. - socket: ZMQ async socket for message transmission (injected by decorator) Returns: BatchMeta: Metadata object containing data structure, sample information, and readiness status @@ -301,20 +303,40 @@ async def async_get_meta( ... ) ... print(batch_meta.is_ready) # May be False if some samples not ready """ - response_msg = await self._request_controller( - socket=socket, - request_type=ZMQRequestType.GET_META, - response_type=ZMQRequestType.GET_META_RESPONSE, - body={ - "data_fields": data_fields, - "batch_size": batch_size, - "partition_id": partition_id, - "mode": mode, - "task_name": task_name, - "sampling_config": sampling_config, - }, + request = { + "data_fields": data_fields, + "batch_size": batch_size, + "partition_id": partition_id, + "mode": mode, + "task_name": task_name, + "sampling_config": sampling_config, + } + deadline = time.monotonic() + TQ_CONTROLLER_GET_METADATA_TIMEOUT + first_attempt = True + + while first_attempt or time.monotonic() < deadline: + first_attempt = False + async with self.controller_rpc_pool.alease(self._controller) as socket: + response_msg = await self._request_controller( + socket=socket, + request_type=ZMQRequestType.GET_META, + response_type=ZMQRequestType.GET_META_RESPONSE, + body=request, + ) + + metadata = response_msg.body["metadata"] + if mode != "fetch" or metadata.size != 0: + return metadata + + remaining = deadline - time.monotonic() + if remaining <= 0: + break + await asyncio.sleep(min(TQ_CONTROLLER_GET_METADATA_CHECK_INTERVAL, remaining)) + + raise TimeoutError( + f"Timeout while waiting for sufficient data for task {task_name}. " + f"Required: {batch_size}, partition: {partition_id}, fields: {data_fields}" ) - return response_msg.body["metadata"] @with_controller_socket async def async_set_custom_meta(