From b7b2e28e3d22c2af34376f8c7d9b68a1c7db69af Mon Sep 17 00:00:00 2001 From: cc Date: Mon, 19 Oct 2026 21:45:18 -0700 Subject: [PATCH 1/2] refactor(perception): use generated CDR in temporal memory and entity viewer --- .github/workflows/message-codegen.yml | 5 + .../experimental/perceive_loop_skill.py | 35 +++- .../temporal_memory/clip_filter.py | 7 +- .../temporal_memory/entity_graph_db.py | 2 +- .../frame_window_accumulator.py | 9 +- .../temporal_memory/temporal_memory.py | 31 ++-- .../temporal_memory/temporal_utils/helpers.py | 8 +- .../test_temporal_memory_module.py | 160 ++++++++++++------ .../temporal_memory/window_analyzer.py | 2 +- .../experimental/test_perceive_loop_skill.py | 43 +++++ dimos/visualization/rerun/cdr.py | 24 ++- dimos/visualization/rerun/test_cdr.py | 31 +++- docs/capabilities/perception/index.md | 8 + 13 files changed, 278 insertions(+), 87 deletions(-) diff --git a/.github/workflows/message-codegen.yml b/.github/workflows/message-codegen.yml index 39f8f31277f..a50d298ec43 100644 --- a/.github/workflows/message-codegen.yml +++ b/.github/workflows/message-codegen.yml @@ -197,3 +197,8 @@ jobs: .native-review/bin/python -m pip install 'hydra-core==1.3.2' .native-review/bin/python -I -c 'import dimos; import pytest; import sys; raise SystemExit(pytest.main(sys.argv[1:]))' --noconftest --import-mode=importlib -o addopts='' -o 'markers=self_hosted: optional heavyweight integration' --strict-markers -m 'not self_hosted' -q "$PWD/dimos/perception/experimental/test_object_generated.py" "$PWD/dimos/perception/experimental/test_object_scene_registration_temporal.py" "$PWD/dimos/perception/experimental/test_generated_tracker.py" "$PWD/dimos/perception/experimental/test_spatial_memory.py" "$PWD/dimos/models/segmentation/test_generated_edge_tam_images.py" "$PWD/dimos/models/segmentation/test_yoloe.py" MYPYPATH="$PWD/build/message-source/python" .native-review/bin/python -m mypy --follow-imports=silent dimos/perception/experimental/object.py dimos/perception/experimental/objectDB.py dimos/perception/experimental/object_scene_registration.py dimos/perception/experimental/object_scene_registration_spec.py dimos/perception/experimental/object_tracker.py dimos/perception/experimental/object_tracker_2d.py dimos/perception/experimental/object_tracker_3d.py dimos/perception/experimental/spatial_perception.py dimos/models/segmentation/edge_tam.py dimos/models/segmentation/yoloe.py + - name: Verify generated temporal memory, PerceiveLoop and entity rendering + if: runner.os == 'Linux' + run: | + .native-review/bin/python -I -c 'import dimos; import pytest; import sys; raise SystemExit(pytest.main(sys.argv[1:]))' --noconftest --import-mode=importlib -o addopts='' -q "$PWD/dimos/perception/experimental/temporal_memory/test_temporal_memory_module.py" "$PWD/dimos/perception/experimental/test_perceive_loop_skill.py" "$PWD/dimos/visualization/rerun/test_cdr.py" + MYPYPATH="$PWD/build/message-source/python" .native-review/bin/python -m mypy --follow-imports=silent dimos/perception/experimental/perceive_loop_skill.py dimos/perception/experimental/temporal_memory/clip_filter.py dimos/perception/experimental/temporal_memory/entity_graph_db.py dimos/perception/experimental/temporal_memory/frame_window_accumulator.py dimos/perception/experimental/temporal_memory/temporal_memory.py dimos/perception/experimental/temporal_memory/temporal_utils/helpers.py dimos/perception/experimental/temporal_memory/window_analyzer.py dimos/visualization/rerun/cdr.py diff --git a/dimos/perception/experimental/perceive_loop_skill.py b/dimos/perception/experimental/perceive_loop_skill.py index 65b2a300e6f..502aa5ab59f 100644 --- a/dimos/perception/experimental/perceive_loop_skill.py +++ b/dimos/perception/experimental/perceive_loop_skill.py @@ -14,19 +14,27 @@ from __future__ import annotations +import base64 from datetime import datetime, timezone import json import os from threading import RLock from typing import TYPE_CHECKING, Any +import reactivex as rx +from reactivex import Observable, operators as ops +from reactivex.abc import SchedulerBase + from dimos.agents.agent_spec import AgentSpec from dimos.agents.annotation import skill from dimos.core.core import rpc from dimos.core.module import Module from dimos.core.stream import In +from dimos.messages.sensor_msgs.msg import Image from dimos.models.vl.create import create -from dimos.msgs.sensor_msgs.Image import Image, sharpness_window +from dimos.msgs.image import image_sharpness, image_to_bgr, image_to_jpeg +from dimos.msgs.time import to_seconds +from dimos.types.timestamped import TimestampedBufferCollection, TimestampedData from dimos.utils.logging_config import setup_logger from dimos.utils.reactive import backpressure from dimos.utils.threadpool import run_in_thread @@ -41,6 +49,25 @@ logger = setup_logger() +def _sharpness_window( + frequency: float, source: Observable[Image], *, scheduler: SchedulerBase | None = None +) -> Observable[Image]: + """Keep the existing source-time rolling selection and repeated timer emissions.""" + if frequency <= 0: + raise ValueError("frequency must be positive") + window: TimestampedBufferCollection[TimestampedData[Image]] = TimestampedBufferCollection( + 1 / frequency + ) + + def remember(image: Image) -> None: + window.add(TimestampedData(image, to_seconds(image.header.stamp))) + + return rx.interval(1 / frequency, scheduler=scheduler).pipe( + ops.with_latest_from(source.pipe(ops.do_action(remember))), + ops.map(lambda _: max(window, key=lambda item: image_sharpness(item.value)).value), + ) + + class PerceiveLoopSkill(Module): color_image: In[Image] @@ -114,7 +141,7 @@ def look_out_for( model = self._vl_model.result() sharpest = backpressure( - sharpness_window(1.0 / self._period, self.color_image.pure_observable()) + _sharpness_window(1.0 / self._period, self.color_image.pure_observable()) ) model.start() self._model_started = True @@ -181,7 +208,7 @@ def _on_image(self, image: Image) -> None: continuation_context: dict[str, Any] = { "bbox": list(best.bbox), "label": best.name, - "image": image.to_base64(quality=70), + "image": base64.b64encode(image_to_jpeg(image, quality=70)).decode("ascii"), } logger.info( "Lookout matched, dispatching continuation", @@ -208,7 +235,7 @@ def _write_debug_image(image: Image, detections: ImageDetections2D[Detection2DBB import cv2 try: - debug_img = image.to_opencv().copy() + debug_img = image_to_bgr(image).copy() for det in detections.detections: x1, y1, x2, y2 = (int(v) for v in det.bbox) cv2.rectangle(debug_img, (x1, y1), (x2, y2), (0, 255, 0), 2) diff --git a/dimos/perception/experimental/temporal_memory/clip_filter.py b/dimos/perception/experimental/temporal_memory/clip_filter.py index cb89c31c55d..148def62653 100644 --- a/dimos/perception/experimental/temporal_memory/clip_filter.py +++ b/dimos/perception/experimental/temporal_memory/clip_filter.py @@ -18,7 +18,8 @@ import numpy as np -from dimos.msgs.sensor_msgs.Image import Image +from dimos.messages.sensor_msgs.msg import Image +from dimos.msgs.image import image_to_rgb from dimos.utils.logging_config import setup_logger logger = setup_logger() @@ -36,9 +37,7 @@ def _get_image_data(image: Image) -> np.ndarray[Any, Any]: """Extract numpy array from Image.""" - if not hasattr(image, "data"): - raise AttributeError(f"Image missing .data attribute: {type(image)}") - return image.data + return image_to_rgb(image) if CLIP_AVAILABLE: diff --git a/dimos/perception/experimental/temporal_memory/entity_graph_db.py b/dimos/perception/experimental/temporal_memory/entity_graph_db.py index 54f188034a4..7dc8ab8890f 100644 --- a/dimos/perception/experimental/temporal_memory/entity_graph_db.py +++ b/dimos/perception/experimental/temporal_memory/entity_graph_db.py @@ -37,8 +37,8 @@ from dimos.utils.logging_config import setup_logger if TYPE_CHECKING: + from dimos.messages.sensor_msgs.msg import Image from dimos.models.vl.base import VlModel - from dimos.msgs.sensor_msgs.Image import Image logger = setup_logger() diff --git a/dimos/perception/experimental/temporal_memory/frame_window_accumulator.py b/dimos/perception/experimental/temporal_memory/frame_window_accumulator.py index 4c910a1b88a..dcb1a6e560a 100644 --- a/dimos/perception/experimental/temporal_memory/frame_window_accumulator.py +++ b/dimos/perception/experimental/temporal_memory/frame_window_accumulator.py @@ -24,8 +24,10 @@ import threading from typing import TYPE_CHECKING +from dimos.msgs.time import to_seconds + if TYPE_CHECKING: - from dimos.msgs.sensor_msgs.Image import Image + from dimos.messages.sensor_msgs.msg import Image @dataclass @@ -87,10 +89,7 @@ def add_frame(self, image: Image, wall_time: float) -> None: with self._lock: if self._video_start_wall_time is None: return - if image.ts is not None: - timestamp_s = image.ts - self._video_start_wall_time - else: - timestamp_s = wall_time - self._video_start_wall_time + timestamp_s = to_seconds(image.header.stamp) - self._video_start_wall_time frame = Frame( frame_index=self._frame_count, timestamp_s=timestamp_s, diff --git a/dimos/perception/experimental/temporal_memory/temporal_memory.py b/dimos/perception/experimental/temporal_memory/temporal_memory.py index 24054dc648a..ab9f878391a 100644 --- a/dimos/perception/experimental/temporal_memory/temporal_memory.py +++ b/dimos/perception/experimental/temporal_memory/temporal_memory.py @@ -37,10 +37,12 @@ from dimos.core.core import rpc from dimos.core.module import Module, ModuleConfig from dimos.core.stream import In, Out +from dimos.messages.dimos_msgs.msg import EntityMarker, EntityMarkers +from dimos.messages.geometry_msgs.msg import Point, PoseStamped +from dimos.messages.sensor_msgs.msg import Image from dimos.models.vl.base import VlModel -from dimos.msgs.geometry_msgs.PoseStamped import PoseStamped -from dimos.msgs.sensor_msgs.Image import Image, sharpness_barrier -from dimos.msgs.visualization_msgs.EntityMarkers import EntityMarkers, Marker +from dimos.msgs.image import image_sharpness +from dimos.msgs.time import header_now from dimos.perception.experimental.temporal_memory.clip_filter import ( CLIP_AVAILABLE, adaptive_keyframes, @@ -57,6 +59,7 @@ from dimos.perception.experimental.temporal_memory.temporal_utils.helpers import is_scene_stale from dimos.perception.experimental.temporal_memory.window_analyzer import WindowAnalyzer from dimos.utils.logging_config import get_run_log_dir, setup_logger +from dimos.utils.reactive import quality_barrier try: from dimos.perception.experimental.temporal_memory.clip_filter import CLIPFrameFilter @@ -255,7 +258,7 @@ def _publish_entity_markers(self) -> None: if not all_entities: return - markers: list[Marker] = [] + markers: list[EntityMarker] = [] for e in all_entities: meta = e.get("metadata") or {} x = meta.get("world_x") @@ -264,18 +267,18 @@ def _publish_entity_markers(self) -> None: if x is None or y is None: continue markers.append( - Marker( + EntityMarker( entity_id=e["entity_id"], label=(e.get("descriptor") or "")[:40], entity_type=e.get("entity_type", "object"), - x=x, - y=y, - z=(z or 0.0) + 0.3, # Offset up so labels float above ground + position=Point(x=x, y=y, z=(z or 0.0) + 0.3), # Offset labels above ground ) ) if markers: - self.entity_markers.publish(EntityMarkers(markers=markers)) + self.entity_markers.publish( + EntityMarkers(header=header_now("world"), markers=markers) + ) logger.info(f"[temporal-memory] published {len(markers)} entity markers to Rerun") except Exception as e: logger.debug(f"entity marker publish error: {e}") @@ -302,7 +305,9 @@ def _on_frame(img: Image) -> None: ) self.register_disposable( - frame_subject.pipe(sharpness_barrier(self.config.fps)).subscribe(_on_frame) + frame_subject.pipe(quality_barrier(image_sharpness, self.config.fps)).subscribe( + _on_frame + ) ) unsub_image = self.color_image.subscribe(frame_subject.on_next) self.register_disposable(Disposable(unsub_image)) @@ -310,9 +315,9 @@ def _on_frame(img: Image) -> None: # Odometry tracking for entity world positioning (optional — # module works without it, entities just won't have world positions) def _on_odom(msg: PoseStamped) -> None: - self._robot_x = msg.position.x - self._robot_y = msg.position.y - self._robot_z = msg.position.z + self._robot_x = msg.pose.position.x + self._robot_y = msg.pose.position.y + self._robot_z = msg.pose.position.z self._odom_count += 1 if self.odom.transport is not None: diff --git a/dimos/perception/experimental/temporal_memory/temporal_utils/helpers.py b/dimos/perception/experimental/temporal_memory/temporal_utils/helpers.py index 683c88376de..010845850bd 100644 --- a/dimos/perception/experimental/temporal_memory/temporal_utils/helpers.py +++ b/dimos/perception/experimental/temporal_memory/temporal_utils/helpers.py @@ -18,6 +18,8 @@ import numpy as np +from dimos.msgs.image import image_to_rgb + if TYPE_CHECKING: from dimos.perception.experimental.temporal_memory.frame_window_accumulator import Frame @@ -68,7 +70,9 @@ def is_scene_stale(frames: list["Frame"], stale_threshold: float = 5.0) -> bool: last_img = frames[-1].image if first_img is None or last_img is None: return False - if not hasattr(first_img, "data") or not hasattr(last_img, "data"): + first = image_to_rgb(first_img) + last = image_to_rgb(last_img) + if first.shape != last.shape: return False - diff = np.abs(first_img.data.astype(float) - last_img.data.astype(float)) + diff = np.abs(first.astype(float) - last.astype(float)) return bool(diff.mean() < stale_threshold) diff --git a/dimos/perception/experimental/temporal_memory/test_temporal_memory_module.py b/dimos/perception/experimental/temporal_memory/test_temporal_memory_module.py index 79265931d1d..be02eefef04 100644 --- a/dimos/perception/experimental/temporal_memory/test_temporal_memory_module.py +++ b/dimos/perception/experimental/temporal_memory/test_temporal_memory_module.py @@ -16,16 +16,27 @@ from __future__ import annotations +from contextlib import ExitStack import json import threading from typing import TYPE_CHECKING -from unittest.mock import MagicMock, create_autospec, patch +from unittest.mock import create_autospec, patch +from dimos_cdrgen.registry import decode as cdr_decode, encode as cdr_encode import numpy as np import pytest - +import rerun as rr + +from dimos.core.cdr_transport import CdrTransport +from dimos.messages.builtin_interfaces.msg import Time +from dimos.messages.dimos_msgs.msg import EntityMarker, EntityMarkers +from dimos.messages.geometry_msgs.msg import Point +from dimos.messages.sensor_msgs.msg import Image +from dimos.messages.std_msgs.msg import Header from dimos.models.vl.base import VlModel -from dimos.msgs.sensor_msgs.Image import Image +from dimos.msgs.image import image_from_array +from dimos.msgs.time import time_from_seconds +from dimos.perception.experimental.temporal_memory.clip_filter import _get_image_data from dimos.perception.experimental.temporal_memory.entity_graph_db import EntityGraphDB from dimos.perception.experimental.temporal_memory.frame_window_accumulator import ( Frame, @@ -38,7 +49,10 @@ from dimos.perception.experimental.temporal_memory.temporal_utils.graph_utils import ( extract_time_window, ) +from dimos.perception.experimental.temporal_memory.temporal_utils.helpers import is_scene_stale +from dimos.perception.experimental.temporal_memory.window_analyzer import WindowAnalyzer from dimos.utils.logging_config import setup_logger +from dimos.visualization.rerun.cdr import entity_points if TYPE_CHECKING: from pathlib import Path @@ -52,7 +66,7 @@ def _make_image(value: int = 128, shape: tuple[int, ...] = (64, 64, 3)) -> Image: data = np.full(shape, value, dtype=np.uint8) - return Image.from_numpy(data) + return image_from_array(data, encoding="rgb8") class TestFrameWindowAccumulator: @@ -61,7 +75,7 @@ def test_bounded_buffer(self) -> None: acc.set_start_time(0.0) for i in range(10): img = _make_image(i * 25) - img.ts = float(i) + img.header.stamp = time_from_seconds(float(i)) acc.add_frame(img, float(i)) assert acc.buffer_size == 5 assert acc.frame_count == 10 @@ -74,7 +88,7 @@ def test_window_extraction(self) -> None: # Add 3 frames for i in range(3): img = _make_image() - img.ts = float(i) + img.header.stamp = time_from_seconds(float(i)) acc.add_frame(img, float(i)) frames = acc.try_extract_window() assert frames is not None @@ -85,7 +99,7 @@ def test_stride_guard(self) -> None: acc.set_start_time(0.0) for i in range(3): img = _make_image() - img.ts = float(i) + img.header.stamp = time_from_seconds(float(i)) acc.add_frame(img, float(i)) # First extraction should succeed frames = acc.try_extract_window() @@ -104,7 +118,7 @@ def test_clear(self) -> None: acc = FrameWindowAccumulator(max_buffer_frames=50, window_s=1.0, stride_s=1.0, fps=1.0) acc.set_start_time(0.0) img = _make_image() - img.ts = 0.0 + img.header.stamp = time_from_seconds(0.0) acc.add_frame(img, 0.0) assert acc.buffer_size == 1 acc.clear() @@ -304,6 +318,7 @@ def test_new_memory_clears_db(self, tmp_path: Path) -> None: return_value=None, ): tm = TemporalMemory( + use_clip_filtering=False, vlm=create_autospec(VlModel, spec_set=True, instance=True), db_dir=str(db_dir), new_memory=True, @@ -329,6 +344,7 @@ def test_persistent_memory_survives(self, tmp_path: Path) -> None: return_value=None, ): tm = TemporalMemory( + use_clip_filtering=False, vlm=create_autospec(VlModel, spec_set=True, instance=True), db_dir=str(db_dir), new_memory=False, @@ -350,6 +366,7 @@ def test_log_entries(self, tmp_path: Path) -> None: return_value=log_dir, ): tm = TemporalMemory( + use_clip_filtering=False, vlm=create_autospec(VlModel, spec_set=True, instance=True), db_dir=str(db_dir), ) @@ -386,62 +403,74 @@ def test_publish_entity_markers(self, tmp_path: Path) -> None: return_value=None, ): tm = TemporalMemory( + use_clip_filtering=False, vlm=create_autospec(VlModel, spec_set=True, instance=True), db_dir=str(db_dir), visualize=True, ) - # Populate DB with world positions - tm._graph_db.upsert_entity( - "E1", - "person", - "walking person", - 1.0, - metadata={"world_x": 1.0, "world_y": 2.0, "world_z": 0.0}, - ) - tm._graph_db.upsert_entity( - "E2", - "object", - "table", - 1.0, - metadata={"world_x": 3.0, "world_y": 4.0, "world_z": 0.0}, - ) - - # Mock the output stream publish - tm.entity_markers = MagicMock() - tm._publish_entity_markers() - - # Should have published EntityMarkers - tm.entity_markers.publish.assert_called_once() - msg = tm.entity_markers.publish.call_args[0][0] - assert len(msg.markers) == 2 - ids = {m.entity_id for m in msg.markers} - assert ids == {"E1", "E2"} - e1 = next(m for m in msg.markers if m.entity_id == "E1") - assert e1.x == 1.0 - assert e1.y == 2.0 - tm.stop() + with ExitStack() as cleanup: + wire = CdrTransport("entity_markers", EntityMarkers, backend="lcm", url="memq://") + cleanup.callback(wire.stop) + cleanup.callback(tm.stop) + tm.entity_markers.transport = wire + tm._graph_db.upsert_entity( + "E1", + "person", + "walking person", + 1, + metadata={"world_x": 1.0, "world_y": 2.0, "world_z": 0.0}, + ) + tm._graph_db.upsert_entity( + "E2", + "object", + "table", + 1, + metadata={"world_x": 3.0, "world_y": 4.0, "world_z": 0.0}, + ) + received = [] + ready = threading.Event() + + def receive(value): + received.append(value) + ready.set() + + cleanup.callback(wire.subscribe(receive)) + tm._publish_entity_markers() + assert ready.wait(2) + assert len(received) == 1 + msg = received[0] + assert msg.header.frame_id == "world" + assert {m.entity_id for m in msg.markers} == {"E1", "E2"} + e1 = next(m for m in msg.markers if m.entity_id == "E1") + assert (e1.position.x, e1.position.y, e1.position.z) == (1, 2, 0.3) def test_markers_to_rerun(self) -> None: - from dimos.msgs.visualization_msgs.EntityMarkers import EntityMarkers, Marker - markers = EntityMarkers( markers=[ - Marker("E1", "person walking", "person", 1.0, 2.0, 0.3), - Marker("E2", "wooden table", "object", 3.0, 4.0, 0.3), - ] + EntityMarker( + entity_id="E1", + label="person walking", + entity_type="person", + position=Point(x=1.0, y=2.0, z=0.3), + ), + EntityMarker( + entity_id="E2", + label="wooden table", + entity_type="object", + position=Point(x=3.0, y=4.0, z=0.3), + ), + ], + header=Header(stamp=Time(sec=0, nanosec=0), frame_id=""), ) - archetype = markers.to_rerun() + archetype = entity_points(cdr_decode(cdr_encode(markers), EntityMarkers)) # Should return rr.Points3D - import rerun as rr assert isinstance(archetype, rr.Points3D) class TestWindowAnalyzer: def test_analyze_window_calls_vlm(self) -> None: - from dimos.perception.experimental.temporal_memory.window_analyzer import WindowAnalyzer - mock_vlm = create_autospec(VlModel, spec_set=True, instance=True) mock_vlm.query.return_value = json.dumps( { @@ -455,7 +484,7 @@ def test_analyze_window_calls_vlm(self) -> None: analyzer = WindowAnalyzer(mock_vlm) img = _make_image() - img.ts = 0.0 + img.header.stamp = time_from_seconds(0.0) frame = Frame(frame_index=0, timestamp_s=0.0, image=img) state_dict = {"entity_roster": [], "rolling_summary": ""} @@ -466,22 +495,18 @@ def test_analyze_window_calls_vlm(self) -> None: mock_vlm.query.assert_called_once() def test_analyze_window_vlm_error(self) -> None: - from dimos.perception.experimental.temporal_memory.window_analyzer import WindowAnalyzer - mock_vlm = create_autospec(VlModel, spec_set=True, instance=True) mock_vlm.query.side_effect = RuntimeError("VLM error") analyzer = WindowAnalyzer(mock_vlm) img = _make_image() - img.ts = 0.0 + img.header.stamp = time_from_seconds(0.0) frame = Frame(frame_index=0, timestamp_s=0.0, image=img) result = analyzer.analyze_window([frame], {}, 0.0, 2.0) assert result is None def test_update_summary(self) -> None: - from dimos.perception.experimental.temporal_memory.window_analyzer import WindowAnalyzer - mock_vlm = create_autospec(VlModel, spec_set=True, instance=True) mock_vlm.query.return_value = "Updated summary text" @@ -493,8 +518,6 @@ def test_update_summary(self) -> None: assert result.summary_text == "Updated summary text" def test_answer_query(self) -> None: - from dimos.perception.experimental.temporal_memory.window_analyzer import WindowAnalyzer - mock_vlm = create_autospec(VlModel, spec_set=True, instance=True) mock_vlm.query.return_value = "The answer is 42" @@ -504,3 +527,30 @@ def test_answer_query(self) -> None: result = analyzer.answer_query("What is the answer?", {}, img) assert result is not None assert result.answer == "The answer is 42" + + +def test_scene_staleness_compares_pixels_not_padding(): + first = Image( + width=2, + height=2, + encoding="rgb8", + step=8, + data=np.asarray([10] * 6 + [0, 0] + [10] * 6 + [0, 0], dtype=np.uint8), + header=Header(stamp=Time(sec=0, nanosec=0), frame_id=""), + is_bigendian=0, + ) + last = cdr_decode(cdr_encode(first), Image) + last.data = np.asarray([10] * 6 + [255, 255] + [10] * 6 + [255, 255], dtype=np.uint8) + assert is_scene_stale([Frame(0, 0, first), Frame(1, 1, last)], stale_threshold=1) + last.data = np.asarray([30] * 6 + [255, 255] + [30] * 6 + [255, 255], dtype=np.uint8) + assert not is_scene_stale([Frame(0, 0, first), Frame(1, 1, last)], stale_threshold=1) + + +def test_clip_frame_comparison_uses_rgb_colors_for_bgr_wire_images(): + pixels = np.zeros((4, 4, 3), dtype=np.uint8) + pixels[:, :, 0] = 255 + rgb = image_from_array(pixels, encoding="rgb8") + bgr = image_from_array(pixels[:, :, ::-1], encoding="bgr8") + bgr = cdr_decode(cdr_encode(bgr), Image) + np.testing.assert_array_equal(_get_image_data(rgb), _get_image_data(bgr)) + np.testing.assert_array_equal(_get_image_data(bgr)[0, 0], [255, 0, 0]) diff --git a/dimos/perception/experimental/temporal_memory/window_analyzer.py b/dimos/perception/experimental/temporal_memory/window_analyzer.py index afb21c3ca48..c5d617c5000 100644 --- a/dimos/perception/experimental/temporal_memory/window_analyzer.py +++ b/dimos/perception/experimental/temporal_memory/window_analyzer.py @@ -35,8 +35,8 @@ from dimos.utils.logging_config import setup_logger if TYPE_CHECKING: + from dimos.messages.sensor_msgs.msg import Image from dimos.models.vl.base import VlModel - from dimos.msgs.sensor_msgs.Image import Image from dimos.perception.experimental.temporal_memory.frame_window_accumulator import Frame logger = setup_logger() diff --git a/dimos/perception/experimental/test_perceive_loop_skill.py b/dimos/perception/experimental/test_perceive_loop_skill.py index 272669a1da0..20e1e719c40 100644 --- a/dimos/perception/experimental/test_perceive_loop_skill.py +++ b/dimos/perception/experimental/test_perceive_loop_skill.py @@ -18,8 +18,13 @@ from threading import Event +import numpy as np import pytest +from reactivex.testing import ReactiveTest, TestScheduler +from dimos.messages.std_msgs.msg import Header +from dimos.msgs.image import image_from_array +from dimos.msgs.time import time_from_seconds from dimos.perception.experimental import perceive_loop_skill as skill_module from dimos.perception.experimental.perceive_loop_skill import PerceiveLoopSkill @@ -61,3 +66,41 @@ def failing_create(name: str) -> object: assert module._lookout_subscription is None finally: module.stop() + + +def test_generated_sharpness_window_keeps_rolling_and_repeated_selection_and_disposes(): + scheduler = TestScheduler() + flat = image_from_array( + np.full((32, 32, 3), 128, dtype=np.uint8), + encoding="rgb8", + header=Header(frame_id="camera", stamp=time_from_seconds(0)), + ) + pixels = np.tile((np.indices((32, 32)).sum(axis=0) % 2 * 255)[:, :, None], (1, 1, 3)).astype( + np.uint8 + ) + sharp = image_from_array( + pixels, encoding="rgb8", header=Header(frame_id="camera", stamp=time_from_seconds(0.1)) + ) + latest = image_from_array( + np.zeros((32, 32, 3), dtype=np.uint8), + encoding="rgb8", + header=Header(frame_id="camera", stamp=time_from_seconds(0.3)), + ) + source = scheduler.create_hot_observable( + ReactiveTest.on_next(0.05, flat), + ReactiveTest.on_next(0.15, sharp), + ReactiveTest.on_next(0.35, latest), + ) + observer = scheduler.create_observer() + subscription = skill_module._sharpness_window(10, source, scheduler=scheduler).subscribe( + observer + ) + try: + scheduler.advance_to(0.5) + values = [message.value.value for message in observer.messages] + assert values == [flat, sharp, sharp, latest, latest] + finally: + subscription.dispose() + scheduler.advance_to(1) + assert len(observer.messages) == 5 + assert source.subscriptions[0].unsubscribe == 0.5 diff --git a/dimos/visualization/rerun/cdr.py b/dimos/visualization/rerun/cdr.py index b4c86915935..548402f831d 100644 --- a/dimos/visualization/rerun/cdr.py +++ b/dimos/visualization/rerun/cdr.py @@ -22,6 +22,7 @@ import numpy as np +from dimos.messages.dimos_msgs.msg import EntityMarkers from dimos.messages.geometry_msgs.msg import PointStamped, PoseStamped from dimos.messages.sensor_msgs.msg import CameraInfo, CompressedImage, Image, PointCloud2 from dimos.msgs.image import image_from_compressed, image_to_rgb, image_view @@ -101,6 +102,23 @@ def cloud_archetype(message: PointCloud2) -> rr.Points3D: ) +def entity_points(message: EntityMarkers) -> rr.Points3D: + """Render generated world entities with the existing labels, colors and radii.""" + import rerun as rr + + colors = { + "person": (255, 100, 100, 255), + "object": (100, 255, 100, 255), + "location": (100, 100, 255, 255), + } + return rr.Points3D( + positions=[[m.position.x, m.position.y, m.position.z] for m in message.markers], + labels=[f"{m.entity_id}: {m.label[:40]}" for m in message.markers], + colors=[colors.get(m.entity_type, (200, 200, 200, 255)) for m in message.markers], + radii=[0.15] * len(message.markers), + ) + + def generated_archetypes(message: Any, entity: str) -> RerunMulti | None: """Render stamped geometry and its exact fields; other domains remain explicit.""" import rerun as rr @@ -114,7 +132,11 @@ def generated_archetypes(message: Any, entity: str) -> RerunMulti | None: if frame: entries.append((entity, rr.Transform3D(parent_frame=f"tf#/{frame}"))) return entries - if type(message) is PoseStamped: + if type(message) is EntityMarkers: + entries = [(entity, entity_points(message))] + if message.header.frame_id: + entries.append((entity, rr.Transform3D(parent_frame=f"tf#/{message.header.frame_id}"))) + elif type(message) is PoseStamped: point, rotation = message.pose.position, message.pose.orientation entries = [ ( diff --git a/dimos/visualization/rerun/test_cdr.py b/dimos/visualization/rerun/test_cdr.py index edda1cb365a..73c669a4573 100644 --- a/dimos/visualization/rerun/test_cdr.py +++ b/dimos/visualization/rerun/test_cdr.py @@ -25,8 +25,10 @@ from dimos.core.cdr_transport import CdrTransport from dimos.core.global_config import GlobalConfig from dimos.messages.builtin_interfaces.msg import Time -from dimos.messages.geometry_msgs.msg import PointStamped +from dimos.messages.dimos_msgs.msg import EntityMarker, EntityMarkers +from dimos.messages.geometry_msgs.msg import Point, PointStamped from dimos.messages.sensor_msgs.msg import Image +from dimos.messages.std_msgs.msg import Header from dimos.msgs.camera_info import camera_info_from_intrinsics from dimos.msgs.image import image_from_array from dimos.msgs.pointcloud import pointcloud_from_xyz_rgb @@ -178,3 +180,30 @@ def test_blueprint_does_not_construct_pubsubs_and_resolution_keeps_backend(mocke expected = zenoh if backend == "zenoh" else lcm expected.assert_called_once_with() assert resolved == [expected.return_value] + + +def test_generated_entity_marker_adapter_keeps_labels_colors_frame_and_exact_fields(): + header = Header(frame_id="world", stamp=Time(sec=1700000000, nanosec=123456789)) + message = EntityMarkers( + header=header, + markers=[ + EntityMarker( + entity_id="E1", + label="person", + entity_type="person", + position=Point(x=1, y=2, z=0.3), + ) + ], + ) + message = decode(encode(message), EntityMarkers) + entries = generated_archetypes(message, "memory/entities") + assert entries is not None + points, frame, fields = [value for _, value in entries] + np.testing.assert_allclose(points.positions.as_arrow_array().to_pylist(), [[1, 2, 0.3]]) + assert points.labels.as_arrow_array().to_pylist() == ["E1: person"] + assert points.colors.as_arrow_array().to_pylist() == [0xFF6464FF] + assert frame.parent_frame.as_arrow_array().to_pylist() == ["tf#/world"] + assert ( + json.loads(fields.text.as_arrow_array().to_pylist()[0])["header"]["stamp"]["nanosec"] + == 123456789 + ) diff --git a/docs/capabilities/perception/index.md b/docs/capabilities/perception/index.md index adcd7e3d468..ce4088a6e30 100644 --- a/docs/capabilities/perception/index.md +++ b/docs/capabilities/perception/index.md @@ -173,6 +173,14 @@ source timestamp zero is not replaced by wall time. Until default encoding cutover, these generated ports require the explicit CDR transports described above. Offline boundary tests do not constitute full model or robot acceptance. +Temporal memory and PerceiveLoop also accept generated images. Pixel comparison +and CLIP frame preparation respect encoding and row stride. PerceiveLoop retains +its rolling sharpest-frame selection and repeated timer output, with disposable +source subscriptions. Temporal memory publishes generated `dimos_msgs/EntityMarkers` +with a world Header; the Rerun adapter owns marker labels, colors and geometry. +Generated values themselves have no rendering methods. Existing query and skill +APIs remain unchanged. + ## Experimental WorldBelief The experimental xArm6 WorldBelief stack records RGB-D observations and processes From 6b7c1d179de6e4ddaba284876a3f4f51163023ca Mon Sep 17 00:00:00 2001 From: "autofix-ci[bot]" <114827586+autofix-ci[bot]@users.noreply.github.com> Date: Tue, 20 Oct 2026 04:46:28 +0000 Subject: [PATCH 2/2] [autofix.ci] apply automated fixes --- dimos/models/embedding/clip.py | 2 +- dimos/models/embedding/dino.py | 2 +- dimos/models/embedding/siglip.py | 2 +- dimos/models/embedding/siglip2.py | 2 +- .../models/embedding/test_generated_images.py | 37 +++++++++++-------- dimos/models/embedding/treid.py | 2 +- 6 files changed, 26 insertions(+), 21 deletions(-) diff --git a/dimos/models/embedding/clip.py b/dimos/models/embedding/clip.py index ad5e3e0cc6c..f429dc62a93 100644 --- a/dimos/models/embedding/clip.py +++ b/dimos/models/embedding/clip.py @@ -17,12 +17,12 @@ from functools import cached_property from typing import overload -from dimos.messages.sensor_msgs.msg import Image from PIL import Image as PILImage import torch import torch.nn.functional as functional from transformers import CLIPModel as HFCLIPModel, CLIPProcessor +from dimos.messages.sensor_msgs.msg import Image from dimos.models.base import HuggingFaceModel from dimos.models.embedding.base import Embedding, EmbeddingModel, HuggingFaceEmbeddingModelConfig from dimos.msgs.image import image_to_rgb diff --git a/dimos/models/embedding/dino.py b/dimos/models/embedding/dino.py index 5b0d24fd137..022a442ebf7 100644 --- a/dimos/models/embedding/dino.py +++ b/dimos/models/embedding/dino.py @@ -17,12 +17,12 @@ from functools import cached_property from typing import Any, overload -from dimos.messages.sensor_msgs.msg import Image from PIL import Image as PILImage import torch import torch.nn.functional as functional from transformers import AutoImageProcessor, AutoModel +from dimos.messages.sensor_msgs.msg import Image from dimos.models.base import HuggingFaceModel from dimos.models.embedding.base import Embedding, EmbeddingModel, HuggingFaceEmbeddingModelConfig from dimos.msgs.image import image_to_rgb diff --git a/dimos/models/embedding/siglip.py b/dimos/models/embedding/siglip.py index bb0ed762943..8cabb1afdf4 100644 --- a/dimos/models/embedding/siglip.py +++ b/dimos/models/embedding/siglip.py @@ -17,12 +17,12 @@ from functools import cached_property from typing import overload -from dimos.messages.sensor_msgs.msg import Image from PIL import Image as PILImage import torch import torch.nn.functional as functional from transformers import SiglipModel as HFSiglipModel, SiglipProcessor +from dimos.messages.sensor_msgs.msg import Image from dimos.models.base import HuggingFaceModel from dimos.models.embedding.base import Embedding, EmbeddingModel, HuggingFaceEmbeddingModelConfig from dimos.msgs.image import image_to_rgb diff --git a/dimos/models/embedding/siglip2.py b/dimos/models/embedding/siglip2.py index f8f1fe8861b..c85932f9176 100644 --- a/dimos/models/embedding/siglip2.py +++ b/dimos/models/embedding/siglip2.py @@ -17,12 +17,12 @@ from functools import cached_property from typing import overload -from dimos.messages.sensor_msgs.msg import Image from PIL import Image as PILImage import torch import torch.nn.functional as functional from transformers import SiglipModel as HFSiglipModel, SiglipProcessor +from dimos.messages.sensor_msgs.msg import Image from dimos.models.base import HuggingFaceModel from dimos.models.embedding.base import Embedding, EmbeddingModel, HuggingFaceEmbeddingModelConfig from dimos.msgs.image import image_to_rgb diff --git a/dimos/models/embedding/test_generated_images.py b/dimos/models/embedding/test_generated_images.py index e3c1245ab07..a51f9778ba4 100644 --- a/dimos/models/embedding/test_generated_images.py +++ b/dimos/models/embedding/test_generated_images.py @@ -29,20 +29,25 @@ from dimos.msgs.image import image_from_array, image_view -@pytest.mark.parametrize("model_type", [CLIPModel,DINOModel,SigLIPModel,SigLIP2Model]) -def test_generated_embeddings_keep_zero_source_time_and_convert_rgb(mocker,model_type): - processor,network=mocker.Mock(),mocker.Mock() - features=torch.tensor([[3.0,4.0]]) - processor.return_value=BatchFeature(data={"pixel_values":torch.zeros(1,3,4,8)}) - network.get_image_features.return_value=features - network.return_value=BaseModelOutputWithPooling(last_hidden_state=features[:,None,:],pooler_output=features) - mocker.patch.object(model_type,"_processor",new_callable=PropertyMock,return_value=processor) - mocker.patch.object(model_type,"_model",new_callable=PropertyMock,return_value=network) - image=image_from_array(np.full((4,8,3),[0,0,255],dtype=np.uint8),encoding="bgr8", - header=Header(frame_id="camera",stamp=Time(sec=0,nanosec=0))) - model=model_type(device="cpu") - result=model.embed(image) +@pytest.mark.parametrize("model_type", [CLIPModel, DINOModel, SigLIPModel, SigLIP2Model]) +def test_generated_embeddings_keep_zero_source_time_and_convert_rgb(mocker, model_type): + processor, network = mocker.Mock(), mocker.Mock() + features = torch.tensor([[3.0, 4.0]]) + processor.return_value = BatchFeature(data={"pixel_values": torch.zeros(1, 3, 4, 8)}) + network.get_image_features.return_value = features + network.return_value = BaseModelOutputWithPooling( + last_hidden_state=features[:, None, :], pooler_output=features + ) + mocker.patch.object(model_type, "_processor", new_callable=PropertyMock, return_value=processor) + mocker.patch.object(model_type, "_model", new_callable=PropertyMock, return_value=network) + image = image_from_array( + np.full((4, 8, 3), [0, 0, 255], dtype=np.uint8), + encoding="bgr8", + header=Header(frame_id="camera", stamp=Time(sec=0, nanosec=0)), + ) + model = model_type(device="cpu") + result = model.embed(image) assert result.timestamp == 0.0 - torch.testing.assert_close(result.vector,torch.tensor([0.6,0.8])) - assert processor.call_args.kwargs["images"][0].getpixel((0,0)) == (255,0,0) - np.testing.assert_array_equal(image_view(image)[0,0],[0,0,255]) + torch.testing.assert_close(result.vector, torch.tensor([0.6, 0.8])) + assert processor.call_args.kwargs["images"][0].getpixel((0, 0)) == (255, 0, 0) + np.testing.assert_array_equal(image_view(image)[0, 0], [0, 0, 255]) diff --git a/dimos/models/embedding/treid.py b/dimos/models/embedding/treid.py index ce36b5d004b..3ce7331eec3 100644 --- a/dimos/models/embedding/treid.py +++ b/dimos/models/embedding/treid.py @@ -19,11 +19,11 @@ from functools import cached_property -from dimos.messages.sensor_msgs.msg import Image import torch import torch.nn.functional as functional from torchreid import utils as torchreid_utils +from dimos.messages.sensor_msgs.msg import Image from dimos.models.base import LocalModel from dimos.models.embedding.base import Embedding, EmbeddingModel, EmbeddingModelConfig from dimos.msgs.image import image_to_bgr