Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 21 additions & 0 deletions scripts/test_controller_fetch_progress.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
#!/usr/bin/env bash
# Copyright 2025 The TransferQueue Team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

set -euo pipefail

cd "$(dirname "$0")/.."
python_bin="${PYTHON:-python3}"
"$python_bin" -m compileall -q transfer_queue tutorial tests
"$python_bin" -m pytest -q tests/test_controller_fetch_progress.py
82 changes: 82 additions & 0 deletions tests/test_controller_fetch_progress.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,82 @@
# Copyright 2025 The TransferQueue Team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import asyncio
import time
from uuid import uuid4

import pytest
import ray
import torch
from omegaconf import OmegaConf
from tensordict import TensorDict

import transfer_queue as tq
import transfer_queue.client as client_module


@pytest.mark.asyncio
async def test_waiting_fetch_allows_put_and_notification(monkeypatch):
namespace = f"tq-fetch-progress-{uuid4().hex}"
monkeypatch.setenv("TQ_RAY_NAMESPACE", namespace)
ray.init(address="local", namespace=namespace, num_cpus=3, include_dashboard=False)
try:
tq.init(OmegaConf.create({"backend": {"SimpleStorage": {"num_data_storage_units": 1}}}))
client = tq.get_client()
client.controller_rpc_pool._maxsize = 1
monkeypatch.setattr(client_module, "TQ_CONTROLLER_GET_METADATA_TIMEOUT", 4)
monkeypatch.setattr(client_module, "TQ_CONTROLLER_GET_METADATA_CHECK_INTERVAL", 1)

reader = asyncio.create_task(
asyncio.to_thread(
client.get_meta,
data_fields=["input_ids"],
batch_size=1,
partition_id="waiting_fetch",
task_name="trainer",
)
)
await asyncio.sleep(0.2)
data = TensorDict({"input_ids": torch.tensor([[1, 2]])}, batch_size=[1])
await asyncio.wait_for(asyncio.to_thread(client.put, data, partition_id="waiting_fetch"), timeout=3)
metadata = await asyncio.wait_for(reader, timeout=5)
result = await asyncio.to_thread(client.get_data, metadata)
assert metadata.is_ready
assert torch.equal(result["input_ids"][0], data["input_ids"][0])

monkeypatch.setattr(client_module, "TQ_CONTROLLER_GET_METADATA_TIMEOUT", 0.25)
monkeypatch.setattr(client_module, "TQ_CONTROLLER_GET_METADATA_CHECK_INTERVAL", 5)
request_count = 0
original_request = client._request_controller

async def count_requests(*args, **kwargs):
nonlocal request_count
request_count += 1
return await original_request(*args, **kwargs)

monkeypatch.setattr(client, "_request_controller", count_requests)
started = time.monotonic()
with pytest.raises(TimeoutError, match="Timeout while waiting for sufficient data"):
await asyncio.to_thread(
client.get_meta,
data_fields=["input_ids"],
batch_size=1,
partition_id="never_produced",
task_name="trainer",
)
assert time.monotonic() - started < 2
assert request_count == 1
finally:
tq.close()
ray.shutdown()
54 changes: 38 additions & 16 deletions transfer_queue/client.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
import asyncio
import os
import threading
import time
import weakref
from typing import Any, Callable

Expand Down Expand Up @@ -53,6 +54,10 @@
# A DEALER drops a reply it can no longer route, so without a bound a lost response blocks
# the caller forever. Generous: a busy controller is normal, an unanswered request is not.
TQ_CONTROLLER_SEND_RECV_TIMEOUT = int(os.environ.get("TQ_CONTROLLER_SEND_RECV_TIMEOUT", 200)) # seconds
# Fetch requests retry empty metadata responses on the client so the controller's
# request loop remains free to process producer notifications.
TQ_CONTROLLER_GET_METADATA_TIMEOUT = int(os.environ.get("TQ_CONTROLLER_GET_METADATA_TIMEOUT", 1))
TQ_CONTROLLER_GET_METADATA_CHECK_INTERVAL = int(os.environ.get("TQ_CONTROLLER_GET_METADATA_CHECK_INTERVAL", 5))

# Pre-bound decorator for controller socket operations.
with_controller_socket = with_zmq_socket(
Expand Down Expand Up @@ -239,7 +244,6 @@ async def _request_controller(
return response_msg

# ==================== Basic API ====================
@with_controller_socket
async def async_get_meta(
self,
data_fields: list[str],
Expand All @@ -248,7 +252,6 @@ async def async_get_meta(
mode: str = "fetch",
task_name: str | None = None,
sampling_config: dict[str, Any] | None = None,
socket: zmq.asyncio.Socket | None = None,
) -> BatchMeta:
"""Asynchronously fetch data metadata from the controller via ZMQ.

Expand All @@ -262,7 +265,6 @@ async def async_get_meta(
- 'insert': Internal usage - should not be used by users
task_name: Optional task name associated with the request
sampling_config: Optional sampling configuration for custom samplers.
socket: ZMQ async socket for message transmission (injected by decorator)

Returns:
BatchMeta: Metadata object containing data structure, sample information, and readiness status
Expand Down Expand Up @@ -301,20 +303,40 @@ async def async_get_meta(
... )
... print(batch_meta.is_ready) # May be False if some samples not ready
"""
response_msg = await self._request_controller(
socket=socket,
request_type=ZMQRequestType.GET_META,
response_type=ZMQRequestType.GET_META_RESPONSE,
body={
"data_fields": data_fields,
"batch_size": batch_size,
"partition_id": partition_id,
"mode": mode,
"task_name": task_name,
"sampling_config": sampling_config,
},
request = {
"data_fields": data_fields,
"batch_size": batch_size,
"partition_id": partition_id,
"mode": mode,
"task_name": task_name,
"sampling_config": sampling_config,
}
deadline = time.monotonic() + TQ_CONTROLLER_GET_METADATA_TIMEOUT
first_attempt = True

while first_attempt or time.monotonic() < deadline:
first_attempt = False
async with self.controller_rpc_pool.alease(self._controller) as socket:
response_msg = await self._request_controller(
socket=socket,
request_type=ZMQRequestType.GET_META,
response_type=ZMQRequestType.GET_META_RESPONSE,
body=request,
)

metadata = response_msg.body["metadata"]
if mode != "fetch" or metadata.size != 0:
return metadata

remaining = deadline - time.monotonic()
if remaining <= 0:
break
await asyncio.sleep(min(TQ_CONTROLLER_GET_METADATA_CHECK_INTERVAL, remaining))

raise TimeoutError(
f"Timeout while waiting for sufficient data for task {task_name}. "
f"Required: {batch_size}, partition: {partition_id}, fields: {data_fields}"
)
return response_msg.body["metadata"]

@with_controller_socket
async def async_set_custom_meta(
Expand Down