From 866b6f6eb0e23b0f5c42283d6154619258199888 Mon Sep 17 00:00:00 2001 From: lanzeshun <962034936@qq.com> Date: Tue, 17 Nov 2026 22:22:06 +0800 Subject: [PATCH 1/3] Add a ZMQ PayloadTransfer adapter for SimpleStorage Signed-off-by: lanzeshun <962034936@qq.com> --- tests/test_storage_key_race.py | 5 +- tests/test_zmq_payload_transfer.py | 68 ++++++ .../managers/simple_storage_manager.py | 198 +++++------------- .../storage/payload_transfer/__init__.py | 1 + .../storage/payload_transfer/base.py | 80 +++++++ .../storage/payload_transfer/zmq.py | 111 ++++++++++ transfer_queue/storage/simple_storage.py | 181 +++++----------- 7 files changed, 374 insertions(+), 270 deletions(-) create mode 100644 tests/test_zmq_payload_transfer.py create mode 100644 transfer_queue/storage/payload_transfer/__init__.py create mode 100644 transfer_queue/storage/payload_transfer/base.py create mode 100644 transfer_queue/storage/payload_transfer/zmq.py diff --git a/tests/test_storage_key_race.py b/tests/test_storage_key_race.py index 6d5670bf..995563a7 100644 --- a/tests/test_storage_key_race.py +++ b/tests/test_storage_key_race.py @@ -20,6 +20,7 @@ import pytest import torch +from transfer_queue.storage.payload_transfer.zmq import ZmqPayloadTransfer from transfer_queue.storage.simple_storage import ( KEY_NOT_FOUND_MARKER, SimpleStorageUnit, @@ -76,8 +77,10 @@ def test_get_error_reply_is_marked_and_logged_at_debug(storage_data, caplog): body={"fields": ["log_probs"], "global_indexes": [1]}, ) + transport = ZmqPayloadTransfer(owner_id=unit.storage_unit_id) + transport.listen(handle=unit._handle_payload) with caplog.at_level(logging.DEBUG, logger="transfer_queue.storage.simple_storage"): - reply = unit_class._handle_get(unit, request) + reply = transport.handle_legacy(request) assert reply.request_type == ZMQRequestType.GET_ERROR assert KEY_NOT_FOUND_MARKER in reply.body["message"] diff --git a/tests/test_zmq_payload_transfer.py b/tests/test_zmq_payload_transfer.py new file mode 100644 index 00000000..3a54a7a7 --- /dev/null +++ b/tests/test_zmq_payload_transfer.py @@ -0,0 +1,68 @@ +# Copyright 2025 Huawei Technologies Co., Ltd. All Rights Reserved. +# Copyright 2025 The TransferQueue Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +"""Exercise the field-data contract over the existing ZMQ multipart path.""" + +from uuid import uuid4 + +import pytest +import torch +import zmq.asyncio + +from transfer_queue.storage.payload_transfer.base import Operation, RequestHeader +from transfer_queue.storage.payload_transfer.zmq import ZmqPayloadTransfer +from transfer_queue.storage.simple_storage import SimpleStorageUnit +from transfer_queue.utils.zmq_utils import STORAGE_MANAGER_IDENTITY_PREFIX, ZMQSocketPool + + +@pytest.mark.asyncio +async def test_zmq_field_data_roundtrip(monkeypatch): + monkeypatch.setattr("transfer_queue.storage.simple_storage.get_node_ip_address", lambda: "127.0.0.1") + unit = SimpleStorageUnit.__ray_metadata__.modified_class({}) + context = zmq.asyncio.Context() + pool = ZMQSocketPool(context, f"{STORAGE_MANAGER_IDENTITY_PREFIX}{uuid4().hex[:8]}", "put_get_socket", timeout=3) + client = ZmqPayloadTransfer(owner_id="test-manager", pool=pool) + try: + connection = await client.connect({"backend": "zmq", "address": unit.get_zmq_server_info()}) + put = RequestHeader(Operation.PUT, {"global_indexes": [0]}) + _, ack = await client.request(connection, put, send={"value": [torch.tensor([1, 2])], "text": ["first"]}) + assert ack is None + + update = RequestHeader( + Operation.UPDATE, + { + "global_indexes": [0], + "merge_fn": lambda old, new: torch.cat([old, new]), + "merge_plan": {"value": {"held": [True], "dtype": torch.int64, "is_non_tensor": False}}, + }, + ) + reply, ack = await client.request(connection, update, send={"value": [torch.tensor([3])]}) + assert ack is None + assert tuple(reply.body["stored_shapes"]["value"]["shapes"][0]) == (3,) + + get = RequestHeader(Operation.GET, {"global_indexes": [0], "fields": ["value", "text"]}) + _, data = await client.request(connection, get, recv=None) + torch.testing.assert_close(data["value"][0], torch.tensor([1, 2, 3])) + assert data["text"] == ["first"] + + with pytest.raises(NotImplementedError, match="receive destinations"): + await client.request(connection, get, recv=(memoryview(bytearray(24)),)) + assert unit._arrivals_by_op["GET_DATA"] == 1 + finally: + await client.close() + pool.close() + context.term() + unit.shutdown() diff --git a/transfer_queue/storage/managers/simple_storage_manager.py b/transfer_queue/storage/managers/simple_storage_manager.py index d14e67c3..fed6f826 100644 --- a/transfer_queue/storage/managers/simple_storage_manager.py +++ b/transfer_queue/storage/managers/simple_storage_manager.py @@ -31,10 +31,14 @@ from transfer_queue.metadata import BatchMeta, extract_field_schema from transfer_queue.storage.managers.base import StorageManager, StorageManagerFactory -from transfer_queue.storage.simple_storage import ( - KEY_NOT_FOUND_MARKER, - StorageKeyNotFoundError, +from transfer_queue.storage.payload_transfer.base import ( + Operation, + RequestHeader, + StorageRejected, + TransferTimeout, ) +from transfer_queue.storage.payload_transfer.zmq import ZmqPayloadTransfer +from transfer_queue.storage.simple_storage import StorageKeyNotFoundError from transfer_queue.utils.common import log_heavy_operation from transfer_queue.utils.logging_utils import get_logger from transfer_queue.utils.zmq_utils import ( @@ -43,7 +47,6 @@ ZMQRequestType, ZMQServerInfo, ZMQSocketPool, - frame_nbytes, with_zmq_socket, ) @@ -238,6 +241,10 @@ def __init__( raise ValueError("AsyncSimpleStorageManager requires non-empty 'zmq_info' in config.") self.storage_unit_infos = self._register_servers(server_infos) + self._payload_peers = { + unit_id: {"backend": "zmq", "address": info} for unit_id, info in self.storage_unit_infos.items() + } + self.payload_transfer = ZmqPayloadTransfer(owner_id=self.storage_manager_id, pool=self.storage_rpc_pool) self._warn_if_pool_can_exhaust_context(len(self.storage_unit_infos)) def _warn_if_pool_can_exhaust_context(self, num_units: int) -> None: @@ -645,64 +652,44 @@ async def update_data( ) return field_schema - @with_storage_unit_socket + async def _request_payload(self, target, operation, body, *, data=None, recv=()): + started = time.perf_counter() + try: + connection = await self.payload_transfer.connect(self._payload_peers[target]) + reply, response_data = await self.payload_transfer.request( + connection, RequestHeader(operation, body), send=data, recv=recv, + ) + except TransferTimeout as exc: + verb = operation.name.lower() + size = f"; serialized_mb={exc.wire_bytes / 2**20:.1f}" if operation != Operation.GET else "" + raise StorageUnitTimeout( + f"no answer in {TQ_SIMPLE_STORAGE_SEND_RECV_TIMEOUT}s during {verb} at storage unit " + f"{target} ({self._describe_storage_unit(target)}){size}" + ) from exc + except StorageRejected as exc: + error = StorageKeyNotFoundError if exc.missing_key else RuntimeError + raise error(f"Failed to {operation.name.lower()} data on storage unit {target}: {exc}") from exc + + if data is not None: + log_heavy_operation( + self.storage_manager_id, operation.name.lower(), time.perf_counter() - started, reply.wire_bytes, + f"to {target} at {self._describe_storage_unit(target)} " + f"samples={len(body['global_indexes'])} fields={list(data)}", + ) + return reply, response_data + async def _put_to_single_storage_unit( self, global_indexes: list[int], storage_data: dict[str, Any], target_storage_unit: str, data_parser: Callable[[Any], Any] | None = None, - socket: zmq.Socket = None, ): - """ - Send data to a specific storage unit. - """ - - request_msg = ZMQMessage.create( - request_type=ZMQRequestType.PUT_DATA, # type: ignore[arg-type] - sender_id=self.storage_manager_id, - receiver_id=target_storage_unit, - body={"global_indexes": global_indexes, "data": storage_data, "data_parser": data_parser}, + await self._request_payload( + target_storage_unit, Operation.PUT, + {"global_indexes": global_indexes, "data_parser": data_parser}, data=storage_data, ) - serialized_bytes = 0 - started = time.perf_counter() - try: - frames = request_msg.serialize() - serialized_bytes = sum(frame_nbytes(frame) or 0 for frame in frames) - await socket.send_multipart(frames, copy=False) - messages = await socket.recv_multipart(copy=False) - response_msg = ZMQMessage.deserialize(messages) - - if response_msg.request_type != ZMQRequestType.PUT_DATA_RESPONSE: - raise RuntimeError( - f"Failed to put data to storage unit {target_storage_unit}: " - f"{response_msg.body.get('message', 'Unknown error')}" - ) - # This end serializes the put payload and waits out the round trip, so it holds both - # the true wire size and the end-to-end latency. - log_heavy_operation( - self.storage_manager_id, - "put", - time.perf_counter() - started, - serialized_bytes, - f"to {target_storage_unit} at {self._describe_storage_unit(target_storage_unit)} " - f"samples={len(global_indexes)} fields={list(storage_data.keys())}", - ) - except zmq.error.Again as e: - raise StorageUnitTimeout( - f"no answer in {TQ_SIMPLE_STORAGE_SEND_RECV_TIMEOUT}s during put to storage unit " - f"{target_storage_unit} at {self._describe_storage_unit(target_storage_unit)}; " - f"serialized_mb={serialized_bytes / 2**20:.1f}" - ) from e - except Exception as e: - logger.error( - f"[{self.storage_manager_id}]: Unexpected error during put to storage unit " - f"{target_storage_unit}: {type(e).__name__}: {e}" - ) - raise RuntimeError(f"Error in put to storage unit {target_storage_unit}: {type(e).__name__}: {e}") from e - - @with_storage_unit_socket async def _update_to_single_storage_unit( self, global_indexes: list[int], @@ -710,58 +697,13 @@ async def _update_to_single_storage_unit( merge_fn: Callable[[Any, Any], Any], merge_plan: dict[str, dict[str, Any]], target_storage_unit: str, - socket: zmq.Socket = None, ) -> dict[str, dict[str, Any]]: - """Send an update to one storage unit and return what it stored, per field.""" - body: dict[str, Any] = { - "global_indexes": global_indexes, - "data": storage_data, - "merge_fn": merge_fn, - "merge_plan": merge_plan, - } - - request_msg = ZMQMessage.create( - request_type=ZMQRequestType.UPDATE_DATA, # type: ignore[arg-type] - sender_id=self.storage_manager_id, - receiver_id=target_storage_unit, - body=body, + reply, _ = await self._request_payload( + target_storage_unit, Operation.UPDATE, + {"global_indexes": global_indexes, "merge_fn": merge_fn, "merge_plan": merge_plan}, + data=storage_data, ) - - serialized_bytes = 0 - started = time.perf_counter() - try: - frames = request_msg.serialize() - serialized_bytes = sum(frame_nbytes(frame) or 0 for frame in frames) - await socket.send_multipart(frames, copy=False) - messages = await socket.recv_multipart(copy=False) - response_msg = ZMQMessage.deserialize(messages) - - if response_msg.request_type != ZMQRequestType.UPDATE_DATA_RESPONSE: - raise RuntimeError( - f"Failed to update data on storage unit {target_storage_unit}: " - f"{response_msg.body.get('message', 'Unknown error')}" - ) - log_heavy_operation( - self.storage_manager_id, - "update", - time.perf_counter() - started, - serialized_bytes, - f"to {target_storage_unit} at {self._describe_storage_unit(target_storage_unit)} " - f"samples={len(global_indexes)} fields={list(storage_data)}", - ) - return response_msg.body.get("stored_shapes", {}) - except zmq.error.Again as e: - raise StorageUnitTimeout( - f"no answer in {TQ_SIMPLE_STORAGE_SEND_RECV_TIMEOUT}s during update to storage unit " - f"{target_storage_unit} at {self._describe_storage_unit(target_storage_unit)}; " - f"serialized_mb={serialized_bytes / 2**20:.1f}" - ) from e - except Exception as e: - logger.error( - f"[{self.storage_manager_id}]: Unexpected error during update to storage unit " - f"{target_storage_unit}: {type(e).__name__}: {e}" - ) - raise RuntimeError(f"Error in update to storage unit {target_storage_unit}: {type(e).__name__}: {e}") from e + return reply.body.get("stored_shapes", {}) @staticmethod def _pack_field_values(values: list) -> torch.Tensor | NonTensorStack: @@ -926,7 +868,6 @@ async def get_data( return TensorDict(tensor_data, batch_size=len(metadata)) - @with_storage_unit_socket async def _get_from_single_storage_unit( self, global_indexes: list[int], @@ -934,48 +875,16 @@ async def _get_from_single_storage_unit( target_storage_unit: str, data_parser: Callable[[Any, Any], Any] | None = None, parser_args: list | None = None, - socket: zmq.Socket = None, ): """Get data from a single SU by global index keys.""" - request_msg = ZMQMessage.create( - request_type=ZMQRequestType.GET_DATA, # type: ignore[arg-type] - sender_id=self.storage_manager_id, - receiver_id=target_storage_unit, - body={ - "global_indexes": global_indexes, - "fields": fields, - "data_parser": data_parser, - "parser_args": parser_args, - }, + _, response_data = await self._request_payload( + target_storage_unit, + Operation.GET, + {"global_indexes": global_indexes, "fields": fields, + "data_parser": data_parser, "parser_args": parser_args}, + recv=None, ) - try: - await socket.send_multipart(request_msg.serialize()) - messages = await socket.recv_multipart(copy=False) - response_msg = ZMQMessage.deserialize(messages) - - if response_msg.request_type == ZMQRequestType.GET_DATA_RESPONSE: - storage_unit_data = response_msg.body["data"] - return fields, storage_unit_data - else: - message = response_msg.body.get("message", "Unknown error") - error_type = StorageKeyNotFoundError if KEY_NOT_FOUND_MARKER in message else RuntimeError - raise error_type(f"Failed to get data from storage unit {target_storage_unit}: {message}") - except zmq.error.Again as e: - raise StorageUnitTimeout( - f"no answer in {TQ_SIMPLE_STORAGE_SEND_RECV_TIMEOUT}s during get from storage unit " - f"{target_storage_unit} at {self._describe_storage_unit(target_storage_unit)}" - ) from e - except StorageKeyNotFoundError: - # Already logged at debug by the storage unit; propagate for the caller to classify. - raise - except Exception as e: - logger.error( - f"[{self.storage_manager_id}]: Unexpected error from storage unit " - f"{target_storage_unit}: {type(e).__name__}: {e}" - ) - raise RuntimeError( - f"Error getting data from storage unit {target_storage_unit}: {type(e).__name__}: {e}" - ) from e + return fields, response_data async def clear_data(self, metadata: BatchMeta, fields: list[str] | None = None) -> None: """Clear data in remote StorageUnit. @@ -1325,6 +1234,9 @@ async def load_checkpoint(self, checkpoint_dir: str) -> None: def close(self) -> None: """Close all ZMQ sockets and context to prevent resource leaks.""" + transfer = getattr(self, "payload_transfer", None) + if transfer is not None: + transfer.shutdown() # Before super(), which may destroy the context these sockets live on. Absent when # the base constructor raised (a failed handshake does), and __del__ still calls this. pool = getattr(self, "storage_rpc_pool", None) diff --git a/transfer_queue/storage/payload_transfer/__init__.py b/transfer_queue/storage/payload_transfer/__init__.py new file mode 100644 index 00000000..92100f6c --- /dev/null +++ b/transfer_queue/storage/payload_transfer/__init__.py @@ -0,0 +1 @@ +"""SimpleStorage payload transport implementations.""" diff --git a/transfer_queue/storage/payload_transfer/base.py b/transfer_queue/storage/payload_transfer/base.py new file mode 100644 index 00000000..f2781e38 --- /dev/null +++ b/transfer_queue/storage/payload_transfer/base.py @@ -0,0 +1,80 @@ +"""Transport contract for one SimpleStorage payload request.""" + +from __future__ import annotations + +import asyncio +from abc import ABC, abstractmethod +from dataclasses import dataclass +from enum import Enum +from typing import Any, Callable + + +class Operation(str, Enum): + PUT = "PUT_DATA" + GET = "GET_DATA" + UPDATE = "UPDATE_DATA" + + +class TransferError(RuntimeError): + """Transport or protocol failure.""" + + +class TransferTimeout(TransferError): + def __init__(self, message: str, *, wire_bytes: int = 0): + super().__init__(message) + self.wire_bytes = wire_bytes + + +class StorageRejected(RuntimeError): + def __init__(self, message: str, *, missing_key: bool = False): + super().__init__(message) + self.missing_key = missing_key + + +@dataclass(frozen=True) +class RequestHeader: + operation: Operation + body: dict[str, Any] + + +@dataclass(frozen=True) +class ReplyHeader: + body: dict[str, Any] + wire_bytes: int = 0 + + +# Reserved destinations describe logical output ranges and retain their owners. +Spans = tuple[memoryview, ...] +PeerInfo = dict[str, Any] +StorageHandler = Callable[[RequestHeader, dict[str, Any] | None], tuple[dict[str, Any], dict[str, Any] | None]] + + +class PayloadTransfer(ABC): + """The selected backend owns the logical request through storage acknowledgement.""" + + @abstractmethod + def listen(self, *, handle: StorageHandler, address: Any = None) -> PeerInfo: + """Bind a local storage handler and return the bootstrap information.""" + + @abstractmethod + async def connect(self, peer_info: PeerInfo) -> Any: + """Resolve an opaque connection usable by this backend.""" + + @abstractmethod + async def request( + self, connection: Any, header: RequestHeader, *, send: dict[str, Any] | None = None, recv: Spans | None = () + ) -> tuple[ReplyHeader, dict[str, Any] | None]: + """Return once response bytes are readable and request sources are reusable. + + Send carries unencoded field values. With recv=None, return decoded fields + owning their backing memory. With recv=(), return only an acknowledgement. + A nonempty recv is reserved for writing into caller-owned output ranges. + On failure, retain any memory still accessible to the transport. + """ + + async def close(self) -> None: + await asyncio.to_thread(self.shutdown) + + @abstractmethod + def shutdown(self) -> None: + """Drain local work and close transport resources for synchronous TQ owners.""" diff --git a/transfer_queue/storage/payload_transfer/zmq.py b/transfer_queue/storage/payload_transfer/zmq.py new file mode 100644 index 00000000..ea8e57a5 --- /dev/null +++ b/transfer_queue/storage/payload_transfer/zmq.py @@ -0,0 +1,111 @@ +"""Adapter for the existing SimpleStorage multipart protocol.""" + +from __future__ import annotations + +import zmq + +from transfer_queue.storage.payload_transfer.base import ( + Operation, + PayloadTransfer, + ReplyHeader, + RequestHeader, + StorageRejected, + TransferError, + TransferTimeout, +) +from transfer_queue.utils.zmq_utils import ZMQMessage, ZMQRequestType, ZMQServerInfo, ZMQSocketPool, frame_nbytes + +KEY_NOT_FOUND_MARKER = "TQKeyNotFound" + +REQUEST_TYPES = { + Operation.PUT: ZMQRequestType.PUT_DATA, + Operation.GET: ZMQRequestType.GET_DATA, + Operation.UPDATE: ZMQRequestType.UPDATE_DATA, +} +RESPONSE_TYPES = { + Operation.PUT: ZMQRequestType.PUT_DATA_RESPONSE, + Operation.GET: ZMQRequestType.GET_DATA_RESPONSE, + Operation.UPDATE: ZMQRequestType.UPDATE_DATA_RESPONSE, +} +ERROR_TYPES = { + Operation.PUT: ZMQRequestType.PUT_ERROR, + Operation.GET: ZMQRequestType.GET_ERROR, + Operation.UPDATE: ZMQRequestType.UPDATE_ERROR, +} + + +class ZmqPayloadTransfer(PayloadTransfer): + def __init__(self, *, owner_id: str, pool: ZMQSocketPool | None = None): + self.owner_id = owner_id + self.pool = pool + self._closed = False + + def listen(self, *, handle, address=None): + self._handle = handle + return {"backend": "zmq", "address": address} + + async def connect(self, peer_info): + if peer_info.get("backend") != "zmq" or not isinstance(peer_info.get("address"), ZMQServerInfo): + raise TransferError("invalid ZMQ payload peer") + return peer_info["address"] + + async def request(self, connection, header, *, send=None, recv=()): + if self._closed or self.pool is None: + raise TransferError("ZMQ payload client is unavailable") + if recv: + raise NotImplementedError("ZMQ receive destinations are not implemented") + if header.operation == Operation.GET: + if send is not None or recv is not None: + raise TransferError("GET requires send=None and recv=None") + elif send is None or recv != (): + raise TransferError("PUT and UPDATE require input data and recv=()") + body = dict(header.body) + if send is not None: + body["data"] = send + request = ZMQMessage.create( + request_type=REQUEST_TYPES[header.operation], + sender_id=self.owner_id, + receiver_id=connection.id, + body=body, + ) + wire_bytes = 0 + try: + async with self.pool.alease(connection) as socket: + frames = request.serialize() + if send is not None: + wire_bytes = sum(frame_nbytes(frame) or 0 for frame in frames) + await socket.send_multipart(frames, copy=False) + reply = ZMQMessage.deserialize(await socket.recv_multipart(copy=False)) + except zmq.error.Again as exc: + raise TransferTimeout("ZMQ storage request timed out", wire_bytes=wire_bytes) from exc + + if reply.request_type != RESPONSE_TYPES[header.operation]: + message = reply.body.get("message", f"unexpected response {reply.request_type}") + raise StorageRejected(message, missing_key=KEY_NOT_FOUND_MARKER in message) + body = dict(reply.body) + if header.operation == Operation.GET: + data = body.pop("data", None) + if not isinstance(data, dict): + raise TransferError("GET response must contain field data") + return ReplyHeader(body, wire_bytes), data + return ReplyHeader(body, wire_bytes), None + + def handle_legacy(self, message): + """Translate the legacy wire message at the storage boundary.""" + operation = Operation(message.request_type.name) + body = dict(message.body) + data = body.pop("data", None) + try: + result, payload = self._handle(RequestHeader(operation, body), data) + if payload is not None: + result = {**result, "data": payload} + response_type = RESPONSE_TYPES[operation] + except StorageRejected as exc: + marker = f"[{KEY_NOT_FOUND_MARKER}] " if exc.missing_key else "" + result = {"message": f"{marker}{exc}"} + response_type = ERROR_TYPES[operation] + return ZMQMessage.create(request_type=response_type, sender_id=self.owner_id, body=result) + + def shutdown(self): + # The manager also uses this pool for administrative requests. + self._closed = True diff --git a/transfer_queue/storage/simple_storage.py b/transfer_queue/storage/simple_storage.py index 8dfa8242..65ec1565 100644 --- a/transfer_queue/storage/simple_storage.py +++ b/transfer_queue/storage/simple_storage.py @@ -51,6 +51,14 @@ import torch import zmq +from transfer_queue.storage.payload_transfer.base import ( + Operation, + PayloadTransfer, + RequestHeader, + StorageRejected, +) +from transfer_queue.storage.payload_transfer.zmq import KEY_NOT_FOUND_MARKER as KEY_NOT_FOUND_MARKER +from transfer_queue.storage.payload_transfer.zmq import REQUEST_TYPES, ZmqPayloadTransfer from transfer_queue.utils import compact_pickle from transfer_queue.utils.common import get_env_bool, limit_pytorch_auto_parallel_threads, log_heavy_operation from transfer_queue.utils.enum_utils import Role @@ -105,9 +113,6 @@ class _SSDValueRef: shape: tuple[int, ...] | None = None -# Marks a GET_ERROR reply as "the key is gone" so the caller can tell it apart from a real fault. -KEY_NOT_FOUND_MARKER = "TQKeyNotFound" - # Accept-queue depth for the client-facing ROUTER. A full queue loses connections silently. TQ_STORAGE_ZMQ_BACKLOG = int(os.environ.get("TQ_STORAGE_ZMQ_BACKLOG", 4096)) @@ -972,7 +977,12 @@ def __init__(self, config: dict[str, Any]): self._metrics: TQMetricsExporter | None = None + self.payload_transfer = ZmqPayloadTransfer(owner_id=self.storage_unit_id) self._init_zmq_socket() + unit = weakref.proxy(self) + self.payload_transfer.listen( + address=self.zmq_server_info, handle=lambda header, data: unit._handle_payload(header, data), + ) self._start_process_put_get() # Register finalizer for graceful cleanup when garbage collected @@ -987,6 +997,7 @@ def __init__(self, config: dict[str, Any]): self._accept_probe, self.worker_socket, self.storage_data, + self.payload_transfer, ) def shutdown(self) -> None: @@ -1213,18 +1224,11 @@ def _process_one_worker_request(self, worker_socket: zmq.Socket, monitor: Any) - try: self._requests_arrived += 1 self._arrivals_by_op[operation.name] = self._arrivals_by_op.get(operation.name, 0) + 1 - logger.debug(f"[{self.storage_unit_id}]: worker received operation: {operation}") - if operation == ZMQRequestType.PUT_DATA: # type: ignore[arg-type] - with monitor.measure(op_type="PUT_DATA"): - response_msg = self._handle_put(request_msg) - elif operation == ZMQRequestType.UPDATE_DATA: # type: ignore[arg-type] - with monitor.measure(op_type="UPDATE_DATA"): - response_msg = self._handle_update(request_msg) - elif operation == ZMQRequestType.GET_DATA: # type: ignore[arg-type] - with monitor.measure(op_type="GET_DATA"): - response_msg = self._handle_get(request_msg) + if operation in REQUEST_TYPES.values(): + with monitor.measure(op_type=operation.name): + response_msg = self.payload_transfer.handle_legacy(request_msg) elif operation == ZMQRequestType.CLEAR_DATA: # type: ignore[arg-type] with monitor.measure(op_type="CLEAR_DATA"): response_msg = self._handle_clear(request_msg) @@ -1290,126 +1294,48 @@ def _process_one_worker_request(self, worker_socket: zmq.Socket, monitor: Any) - ) worker_socket.send_multipart([identity] + response_frames, copy=False) - def _handle_put(self, data_parts: ZMQMessage) -> ZMQMessage: - """ - Handle put request, add or update data into storage unit. - - Args: - data_parts: ZMQMessage from client. - - Returns: - Put data success response ZMQMessage. - """ - try: - global_indexes = data_parts.body["global_indexes"] - field_data = data_parts.body["data"] # field_data should be a dict. - data_parser = data_parts.body.get("data_parser", None) - - with limit_pytorch_auto_parallel_threads( - target_num_threads=TQ_NUM_THREADS, info=f"[{self.storage_unit_id}] _handle_put" - ): - if data_parser is not None: - field_data = _apply_data_parser(field_data, data_parser) - self.storage_data.put_data(field_data, global_indexes) - - # After put operation finish, send a message to the client - response_msg = ZMQMessage.create( - request_type=ZMQRequestType.PUT_DATA_RESPONSE, # type: ignore[arg-type] - sender_id=self.storage_unit_id, - body={}, - ) - - return response_msg - except Exception as e: - return ZMQMessage.create( - request_type=ZMQRequestType.PUT_ERROR, # type: ignore[arg-type] - sender_id=self.storage_unit_id, - body={ - "message": f"Failed to put data into storage unit id " - f"#{self.storage_unit_id}, detail error message: {str(e)}" - }, - ) - - def _handle_update(self, data_parts: ZMQMessage) -> ZMQMessage: - """Merge or add fields, write once, and describe what was stored.""" + def _handle_payload(self, header: RequestHeader, data: dict[str, Any] | None): + """Apply storage semantics on the unit worker for every payload backend.""" + body = header.body + operation = header.operation try: - global_indexes = data_parts.body["global_indexes"] - merge_fn = data_parts.body.get("merge_fn") - new_data = data_parts.body.get("data") - merge_plan = data_parts.body.get("merge_plan") - + indexes = body["global_indexes"] with limit_pytorch_auto_parallel_threads( - target_num_threads=TQ_NUM_THREADS, info=f"[{self.storage_unit_id}] _handle_update" + target_num_threads=TQ_NUM_THREADS, info=f"[{self.storage_unit_id}] {operation.value}" ): + if operation == Operation.GET: + result = self.storage_data.get_data(body["fields"], indexes) + if (parser := body.get("data_parser")) is not None: + result = _apply_data_parser(result, parser, body.get("parser_args")) + return {}, result + if operation == Operation.PUT: + if (parser := body.get("data_parser")) is not None: + data = _apply_data_parser(data, parser) + self.storage_data.put_data(data, indexes) + return {}, None + + merge_fn = body.get("merge_fn") + merge_plan = body.get("merge_plan") if not callable(merge_fn): raise TypeError(f"merge_fn must be callable, got {type(merge_fn).__name__}") - if not isinstance(new_data, dict) or not new_data: + if not isinstance(data, dict) or not data: raise TypeError("update data must be a non-empty dict") - if not isinstance(merge_plan, dict) or set(merge_plan) != set(new_data): + if not isinstance(merge_plan, dict) or set(merge_plan) != set(data): raise ValueError("merge_plan must describe every update field") - stored_shapes = self.storage_data.apply_update(global_indexes, new_data, merge_fn, merge_plan) - - return ZMQMessage.create( - request_type=ZMQRequestType.UPDATE_DATA_RESPONSE, # type: ignore[arg-type] - sender_id=self.storage_unit_id, - body={"stored_shapes": stored_shapes}, - ) - except Exception as e: - return ZMQMessage.create( - request_type=ZMQRequestType.UPDATE_ERROR, # type: ignore[arg-type] - sender_id=self.storage_unit_id, - body={ - "message": f"Failed to update data in storage unit id " - f"#{self.storage_unit_id}, detail error message: {str(e)}" - }, - ) - - def _handle_get(self, data_parts: ZMQMessage) -> ZMQMessage: - """ - Handle get request, return data from storage unit. - - Args: - data_parts: ZMQMessage from client. - - Returns: - Get data success response ZMQMessage, containing target data. - """ - try: - fields = data_parts.body["fields"] - global_indexes = data_parts.body["global_indexes"] - data_parser = data_parts.body.get("data_parser") - - with limit_pytorch_auto_parallel_threads( - target_num_threads=TQ_NUM_THREADS, info=f"[{self.storage_unit_id}] _handle_get" - ): - result_data = self.storage_data.get_data(fields, global_indexes) - if data_parser is not None: - result_data = _apply_data_parser(result_data, data_parser, data_parts.body.get("parser_args")) - - response_msg = ZMQMessage.create( - request_type=ZMQRequestType.GET_DATA_RESPONSE, # type: ignore[arg-type] - sender_id=self.storage_unit_id, - body={ - "data": result_data, - }, - ) - except Exception as e: - key_not_found = isinstance(e, StorageKeyNotFoundError) - log = logger.debug if key_not_found else logger.error - log( - f"[{self.storage_unit_id}]: _handle_get error, " - f"fields={fields}, global_indexes={global_indexes}: {type(e).__name__}: {e}" - ) - marker = f"[{KEY_NOT_FOUND_MARKER}] " if key_not_found else "" - response_msg = ZMQMessage.create( - request_type=ZMQRequestType.GET_ERROR, # type: ignore[arg-type] - sender_id=self.storage_unit_id, - body={ - "message": f"{marker}Failed to get data from storage unit id #{self.storage_unit_id}, " - f"detail error message: {str(e)}" - }, - ) - return response_msg + stored_shapes = self.storage_data.apply_update(indexes, data, merge_fn, merge_plan) + return {"stored_shapes": stored_shapes}, None + except Exception as exc: + missing_key = operation == Operation.GET and isinstance(exc, StorageKeyNotFoundError) + if operation == Operation.GET: + log = logger.debug if missing_key else logger.error + log( + f"[{self.storage_unit_id}]: GET error, fields={body.get('fields')}, " + f"global_indexes={body.get('global_indexes')}: {type(exc).__name__}: {exc}" + ) + raise StorageRejected( + f"Failed to {operation.name.lower()} data on storage unit {self.storage_unit_id}: {exc}", + missing_key=missing_key, + ) from exc def _handle_clear(self, data_parts: ZMQMessage) -> ZMQMessage: """ @@ -1730,6 +1656,7 @@ def _shutdown_resources( accept_probe: "AcceptQueueProbe | None" = None, worker_socket: zmq.Socket | None = None, storage_data: StorageUnitData | None = None, + payload_transfer: PayloadTransfer | None = None, ) -> None: """Clean up resources on garbage collection.""" logger.info("Shutting down SimpleStorageUnit resources...") @@ -1751,6 +1678,8 @@ def _shutdown_resources( worker_socket.close(linger=0) if zmq_context: zmq_context.term() + if payload_transfer is not None: + payload_transfer.shutdown() if storage_data is not None: try: storage_data.close() From 2b235b6a265c9406d87043b677ca6cc4691f167d Mon Sep 17 00:00:00 2001 From: lanzeshun <962034936@qq.com> Date: Thu, 19 Nov 2026 20:55:20 +0800 Subject: [PATCH 2/3] Add ZMQ PayloadTransfer UT Signed-off-by: lanzeshun <962034936@qq.com> --- tests/test_zmq_payload_transfer.py | 104 ++++++++++++++++++++++++++++- 1 file changed, 102 insertions(+), 2 deletions(-) diff --git a/tests/test_zmq_payload_transfer.py b/tests/test_zmq_payload_transfer.py index 3a54a7a7..8e7c039e 100644 --- a/tests/test_zmq_payload_transfer.py +++ b/tests/test_zmq_payload_transfer.py @@ -16,16 +16,116 @@ """Exercise the field-data contract over the existing ZMQ multipart path.""" +import asyncio +from unittest.mock import AsyncMock, MagicMock from uuid import uuid4 import pytest import torch import zmq.asyncio -from transfer_queue.storage.payload_transfer.base import Operation, RequestHeader +from transfer_queue.storage.managers.simple_storage_manager import AsyncSimpleStorageManager, StorageUnitTimeout +from transfer_queue.storage.payload_transfer.base import Operation, RequestHeader, TransferTimeout from transfer_queue.storage.payload_transfer.zmq import ZmqPayloadTransfer from transfer_queue.storage.simple_storage import SimpleStorageUnit -from transfer_queue.utils.zmq_utils import STORAGE_MANAGER_IDENTITY_PREFIX, ZMQSocketPool +from transfer_queue.utils.enum_utils import Role +from transfer_queue.utils.zmq_utils import ( + STORAGE_MANAGER_IDENTITY_PREFIX, + ZMQMessage, + ZMQRequestType, + ZMQServerInfo, + ZMQSocketPool, +) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("failure", ["send_timeout", "receive_timeout", "disconnect", "cancel", "receive_error"]) +@pytest.mark.parametrize("operation", list(Operation)) +async def test_payload_failure_discards_socket_and_releases_permit(monkeypatch, failure, operation): + peer = ZMQServerInfo(role=Role.STORAGE, id="unit", ip="127.0.0.1", ports={"put_get_socket": 5555}) + failed = MagicMock(closed=False) + failed.send_multipart = AsyncMock() + failed.recv_multipart = AsyncMock() + error = { + "send_timeout": zmq.Again(), "receive_timeout": zmq.Again(), + "disconnect": zmq.ZMQError(zmq.ENOTSOCK), "cancel": asyncio.CancelledError(), + "receive_error": ValueError("invalid response"), + }[failure] + if failure == "send_timeout": + failed.send_multipart.side_effect = error + else: + failed.recv_multipart.side_effect = error + healthy = MagicMock(closed=False) + healthy.send_multipart = AsyncMock() + healthy.recv_multipart = AsyncMock(return_value=ZMQMessage.create( + request_type=ZMQRequestType.GET_DATA_RESPONSE, sender_id="unit", body={"data": {"text": ["next"]}}, + ).serialize()) + pool = ZMQSocketPool(MagicMock(), "manager", "put_get_socket", maxsize=1) + connect = MagicMock(side_effect=[failed, healthy]) + monkeypatch.setattr(pool, "_connect", connect) + client = ZmqPayloadTransfer(owner_id="manager", pool=pool) + header = RequestHeader(Operation.GET, {"global_indexes": [0], "fields": ["text"]}) + try: + expected = TransferTimeout if isinstance(error, zmq.Again) else type(error) + with pytest.raises(expected) as caught: + await client.request( + peer, RequestHeader(operation, header.body), + send=None if operation == Operation.GET else {"text": ["input"]}, + recv=None if operation == Operation.GET else (), + ) + if expected is TransferTimeout: + assert isinstance(caught.value.__cause__, zmq.Again) + assert (caught.value.wire_bytes > 0) == (operation != Operation.GET) + failed.close.assert_called_once_with(linger=0) + # A single permit forces recovery to exercise both socket eviction and permit return. + for _ in range(2): + _, data = await asyncio.wait_for(client.request(peer, header, recv=None), timeout=2) + assert data == {"text": ["next"]} + assert connect.call_count == 2 + healthy.close.assert_not_called() + finally: + client.shutdown() + pool.close() + + +@pytest.mark.asyncio +async def test_update_lost_reply_is_not_replayed_through_payload_adapter(monkeypatch): + monkeypatch.setattr("transfer_queue.storage.simple_storage.get_node_ip_address", lambda: "127.0.0.1") + unit = SimpleStorageUnit.__ray_metadata__.modified_class({}) + unit.storage_data.put_data({"value": [torch.tensor([1])]}, [0]) + peer = unit.get_zmq_server_info() + socket = MagicMock(closed=False) + applied = [] + + async def send(frames, **kwargs): + applied.append(unit.payload_transfer.handle_legacy(ZMQMessage.deserialize(frames))) + + socket.send_multipart = AsyncMock(side_effect=send) + socket.recv_multipart = AsyncMock(side_effect=zmq.Again()) + pool = ZMQSocketPool(MagicMock(), "manager", "put_get_socket", maxsize=1) + monkeypatch.setattr(pool, "_connect", MagicMock(return_value=socket)) + manager = AsyncSimpleStorageManager.__new__(AsyncSimpleStorageManager) + manager.storage_manager_id = "manager" + manager.storage_unit_infos = {peer.id: peer} + manager._payload_peers = {peer.id: {"backend": "zmq", "address": peer}} + manager.payload_transfer = ZmqPayloadTransfer(owner_id="manager", pool=pool) + manager.close = lambda: None + merge = MagicMock(side_effect=lambda old, new: torch.cat([old, new])) + try: + with pytest.raises(StorageUnitTimeout): + await manager._update_to_single_storage_unit( + [0], {"value": [torch.tensor([2])]}, merge, + {"value": {"held": [True], "dtype": torch.int64, "is_non_tensor": False}}, peer.id, + ) + assert len(applied) == 1 + assert applied[0].request_type == ZMQRequestType.UPDATE_DATA_RESPONSE + torch.testing.assert_close(unit.storage_data.get_data(["value"], [0])["value"][0], torch.tensor([1, 2])) + socket.send_multipart.assert_awaited_once() + socket.close.assert_called_once_with(linger=0) + finally: + manager.payload_transfer.shutdown() + pool.close() + unit.shutdown() @pytest.mark.asyncio From 40d9161a15afd68ee7b94d7a633b7ed817c769dc Mon Sep 17 00:00:00 2001 From: lanzeshun <962034936@qq.com> Date: Fri, 20 Nov 2026 19:46:18 +0800 Subject: [PATCH 3/3] Fix pre-commit Signed-off-by: lanzeshun <962034936@qq.com> --- tests/test_storage_key_race.py | 2 +- tests/test_zmq_payload_transfer.py | 28 +++++++++++++------ .../managers/simple_storage_manager.py | 27 +++++++++++++----- .../storage/payload_transfer/zmq.py | 4 +-- transfer_queue/storage/simple_storage.py | 5 ++-- 5 files changed, 45 insertions(+), 21 deletions(-) diff --git a/tests/test_storage_key_race.py b/tests/test_storage_key_race.py index 995563a7..7d8d3993 100644 --- a/tests/test_storage_key_race.py +++ b/tests/test_storage_key_race.py @@ -80,7 +80,7 @@ def test_get_error_reply_is_marked_and_logged_at_debug(storage_data, caplog): transport = ZmqPayloadTransfer(owner_id=unit.storage_unit_id) transport.listen(handle=unit._handle_payload) with caplog.at_level(logging.DEBUG, logger="transfer_queue.storage.simple_storage"): - reply = transport.handle_legacy(request) + reply = transport.handle_request(request) assert reply.request_type == ZMQRequestType.GET_ERROR assert KEY_NOT_FOUND_MARKER in reply.body["message"] diff --git a/tests/test_zmq_payload_transfer.py b/tests/test_zmq_payload_transfer.py index 8e7c039e..364eaec2 100644 --- a/tests/test_zmq_payload_transfer.py +++ b/tests/test_zmq_payload_transfer.py @@ -47,8 +47,10 @@ async def test_payload_failure_discards_socket_and_releases_permit(monkeypatch, failed.send_multipart = AsyncMock() failed.recv_multipart = AsyncMock() error = { - "send_timeout": zmq.Again(), "receive_timeout": zmq.Again(), - "disconnect": zmq.ZMQError(zmq.ENOTSOCK), "cancel": asyncio.CancelledError(), + "send_timeout": zmq.Again(), + "receive_timeout": zmq.Again(), + "disconnect": zmq.ZMQError(zmq.ENOTSOCK), + "cancel": asyncio.CancelledError(), "receive_error": ValueError("invalid response"), }[failure] if failure == "send_timeout": @@ -57,9 +59,13 @@ async def test_payload_failure_discards_socket_and_releases_permit(monkeypatch, failed.recv_multipart.side_effect = error healthy = MagicMock(closed=False) healthy.send_multipart = AsyncMock() - healthy.recv_multipart = AsyncMock(return_value=ZMQMessage.create( - request_type=ZMQRequestType.GET_DATA_RESPONSE, sender_id="unit", body={"data": {"text": ["next"]}}, - ).serialize()) + healthy.recv_multipart = AsyncMock( + return_value=ZMQMessage.create( + request_type=ZMQRequestType.GET_DATA_RESPONSE, + sender_id="unit", + body={"data": {"text": ["next"]}}, + ).serialize() + ) pool = ZMQSocketPool(MagicMock(), "manager", "put_get_socket", maxsize=1) connect = MagicMock(side_effect=[failed, healthy]) monkeypatch.setattr(pool, "_connect", connect) @@ -69,7 +75,8 @@ async def test_payload_failure_discards_socket_and_releases_permit(monkeypatch, expected = TransferTimeout if isinstance(error, zmq.Again) else type(error) with pytest.raises(expected) as caught: await client.request( - peer, RequestHeader(operation, header.body), + peer, + RequestHeader(operation, header.body), send=None if operation == Operation.GET else {"text": ["input"]}, recv=None if operation == Operation.GET else (), ) @@ -98,7 +105,7 @@ async def test_update_lost_reply_is_not_replayed_through_payload_adapter(monkeyp applied = [] async def send(frames, **kwargs): - applied.append(unit.payload_transfer.handle_legacy(ZMQMessage.deserialize(frames))) + applied.append(unit.payload_transfer.handle_request(ZMQMessage.deserialize(frames))) socket.send_multipart = AsyncMock(side_effect=send) socket.recv_multipart = AsyncMock(side_effect=zmq.Again()) @@ -114,8 +121,11 @@ async def send(frames, **kwargs): try: with pytest.raises(StorageUnitTimeout): await manager._update_to_single_storage_unit( - [0], {"value": [torch.tensor([2])]}, merge, - {"value": {"held": [True], "dtype": torch.int64, "is_non_tensor": False}}, peer.id, + [0], + {"value": [torch.tensor([2])]}, + merge, + {"value": {"held": [True], "dtype": torch.int64, "is_non_tensor": False}}, + peer.id, ) assert len(applied) == 1 assert applied[0].request_type == ZMQRequestType.UPDATE_DATA_RESPONSE diff --git a/transfer_queue/storage/managers/simple_storage_manager.py b/transfer_queue/storage/managers/simple_storage_manager.py index fed6f826..c691cc3b 100644 --- a/transfer_queue/storage/managers/simple_storage_manager.py +++ b/transfer_queue/storage/managers/simple_storage_manager.py @@ -657,7 +657,10 @@ async def _request_payload(self, target, operation, body, *, data=None, recv=()) try: connection = await self.payload_transfer.connect(self._payload_peers[target]) reply, response_data = await self.payload_transfer.request( - connection, RequestHeader(operation, body), send=data, recv=recv, + connection, + RequestHeader(operation, body), + send=data, + recv=recv, ) except TransferTimeout as exc: verb = operation.name.lower() @@ -672,7 +675,10 @@ async def _request_payload(self, target, operation, body, *, data=None, recv=()) if data is not None: log_heavy_operation( - self.storage_manager_id, operation.name.lower(), time.perf_counter() - started, reply.wire_bytes, + self.storage_manager_id, + operation.name.lower(), + time.perf_counter() - started, + reply.wire_bytes, f"to {target} at {self._describe_storage_unit(target)} " f"samples={len(body['global_indexes'])} fields={list(data)}", ) @@ -686,8 +692,10 @@ async def _put_to_single_storage_unit( data_parser: Callable[[Any], Any] | None = None, ): await self._request_payload( - target_storage_unit, Operation.PUT, - {"global_indexes": global_indexes, "data_parser": data_parser}, data=storage_data, + target_storage_unit, + Operation.PUT, + {"global_indexes": global_indexes, "data_parser": data_parser}, + data=storage_data, ) async def _update_to_single_storage_unit( @@ -699,7 +707,8 @@ async def _update_to_single_storage_unit( target_storage_unit: str, ) -> dict[str, dict[str, Any]]: reply, _ = await self._request_payload( - target_storage_unit, Operation.UPDATE, + target_storage_unit, + Operation.UPDATE, {"global_indexes": global_indexes, "merge_fn": merge_fn, "merge_plan": merge_plan}, data=storage_data, ) @@ -880,8 +889,12 @@ async def _get_from_single_storage_unit( _, response_data = await self._request_payload( target_storage_unit, Operation.GET, - {"global_indexes": global_indexes, "fields": fields, - "data_parser": data_parser, "parser_args": parser_args}, + { + "global_indexes": global_indexes, + "fields": fields, + "data_parser": data_parser, + "parser_args": parser_args + }, recv=None, ) return fields, response_data diff --git a/transfer_queue/storage/payload_transfer/zmq.py b/transfer_queue/storage/payload_transfer/zmq.py index ea8e57a5..1a477421 100644 --- a/transfer_queue/storage/payload_transfer/zmq.py +++ b/transfer_queue/storage/payload_transfer/zmq.py @@ -90,8 +90,8 @@ async def request(self, connection, header, *, send=None, recv=()): return ReplyHeader(body, wire_bytes), data return ReplyHeader(body, wire_bytes), None - def handle_legacy(self, message): - """Translate the legacy wire message at the storage boundary.""" + def handle_request(self, message): + """Dispatch a ZMQ payload request and build its response.""" operation = Operation(message.request_type.name) body = dict(message.body) data = body.pop("data", None) diff --git a/transfer_queue/storage/simple_storage.py b/transfer_queue/storage/simple_storage.py index 65ec1565..d64d0650 100644 --- a/transfer_queue/storage/simple_storage.py +++ b/transfer_queue/storage/simple_storage.py @@ -981,7 +981,8 @@ def __init__(self, config: dict[str, Any]): self._init_zmq_socket() unit = weakref.proxy(self) self.payload_transfer.listen( - address=self.zmq_server_info, handle=lambda header, data: unit._handle_payload(header, data), + address=self.zmq_server_info, + handle=lambda header, data: unit._handle_payload(header, data), ) self._start_process_put_get() @@ -1228,7 +1229,7 @@ def _process_one_worker_request(self, worker_socket: zmq.Socket, monitor: Any) - if operation in REQUEST_TYPES.values(): with monitor.measure(op_type=operation.name): - response_msg = self.payload_transfer.handle_legacy(request_msg) + response_msg = self.payload_transfer.handle_request(request_msg) elif operation == ZMQRequestType.CLEAR_DATA: # type: ignore[arg-type] with monitor.measure(op_type="CLEAR_DATA"): response_msg = self._handle_clear(request_msg)