refactor(perception): migrate experimental object and tracker values to CDR

.github/workflows/message-codegen.yml

@@ -27,7 +27,8 @@ on:
       - dimos/perception/localize/**
       - dimos/perception/detection/project.py
       - dimos/models/embedding/**
-      - dimos/perception/experimental/moduleDB.py
+      - dimos/perception/experimental/**
+      - dimos/models/segmentation/**
       - dimos/models/vl/base.py
       - dimos/models/vl/qwen.py
       - dimos/models/vl/test_base.py
@@ -190,3 +191,9 @@ jobs:
         run: |
           .native-review/bin/python -I -c 'import dimos; import pytest; import sys; raise SystemExit(pytest.main(sys.argv[1:]))' --noconftest --import-mode=importlib -o addopts='' -q "$PWD/dimos/perception/localize/test_rig.py" "$PWD/dimos/perception/detection/type/detection3d/test_generated_projection.py" "$PWD/dimos/memory/store/test_cdr_recording.py" "$PWD/dimos/memory/codecs/test_codecs.py"
           MYPYPATH="$PWD/build/message-source/python" .native-review/bin/python -m mypy --follow-imports=silent dimos/perception/localize/association.py dimos/perception/localize/dandetect.py dimos/perception/localize/module.py dimos/perception/localize/rig.py dimos/perception/localize/support_plane.py dimos/perception/localize/types.py dimos/perception/localize/localize.py dimos/perception/localize/tools/tool_localize.py dimos/perception/localize/demo_blueprints/xarm_feed.py dimos/perception/detection/project.py dimos/memory/codecs/jpeg.py dimos/memory/codecs/base.py
+      - name: Verify generated experimental objects, tracking and segmentation boundaries
+        if: runner.os == 'Linux'
+        run: |
+          .native-review/bin/python -m pip install 'hydra-core==1.3.2'
+          .native-review/bin/python -I -c 'import dimos; import pytest; import sys; raise SystemExit(pytest.main(sys.argv[1:]))' --noconftest --import-mode=importlib -o addopts='' -o 'markers=self_hosted: optional heavyweight integration' --strict-markers -m 'not self_hosted' -q "$PWD/dimos/perception/experimental/test_object_generated.py" "$PWD/dimos/perception/experimental/test_object_scene_registration_temporal.py" "$PWD/dimos/perception/experimental/test_generated_tracker.py" "$PWD/dimos/perception/experimental/test_spatial_memory.py" "$PWD/dimos/models/segmentation/test_generated_edge_tam_images.py" "$PWD/dimos/models/segmentation/test_yoloe.py"
+          MYPYPATH="$PWD/build/message-source/python" .native-review/bin/python -m mypy --follow-imports=silent dimos/perception/experimental/object.py dimos/perception/experimental/objectDB.py dimos/perception/experimental/object_scene_registration.py dimos/perception/experimental/object_scene_registration_spec.py dimos/perception/experimental/object_tracker.py dimos/perception/experimental/object_tracker_2d.py dimos/perception/experimental/object_tracker_3d.py dimos/perception/experimental/spatial_perception.py dimos/models/segmentation/edge_tam.py dimos/models/segmentation/yoloe.py

dimos/models/segmentation/edge_tam.py

@@ -27,8 +27,9 @@
 from PIL import Image as PILImage
 import torch
 
+from dimos.messages.sensor_msgs.msg import Image
 from dimos.models.base import default_torch_device
-from dimos.msgs.sensor_msgs.Image import Image
+from dimos.msgs.image import image_to_bgr, image_to_jpeg
 from dimos.perception.detection.detectors.base import Detector
 from dimos.perception.detection.type.detection2d.bbox import Detection2DBBox
 from dimos.perception.detection.type.detection2d.imageDetections2D import ImageDetections2D
@@ -172,7 +173,7 @@ def propose_all(
             )
             self._mask_generators[points_per_side] = generator
 
-        rgb = cv2.cvtColor(image.to_opencv(), cv2.COLOR_BGR2RGB)
+        rgb = cv2.cvtColor(image_to_bgr(image), cv2.COLOR_BGR2RGB)
         amp = (
             torch.autocast("cuda", dtype=torch.bfloat16)
             if self._predictor.device.type == "cuda"
@@ -203,7 +204,7 @@ def segment(
             return ImageDetections2D(detections.image, [])
 
         image = detections.image
-        rgb = cv2.cvtColor(image.to_opencv(), cv2.COLOR_BGR2RGB)
+        rgb = cv2.cvtColor(image_to_bgr(image), cv2.COLOR_BGR2RGB)
 
         # bfloat16 autocast is CUDA-only; MPS runs in float32
         amp = (
@@ -254,7 +255,7 @@ def _prepare_frame(self, image: Image) -> torch.Tensor:
         """Prepare frame for SAM2 (resize, normalize, convert to tensor)."""
         import cv2
 
-        cv_image = image.to_opencv()
+        cv_image = image_to_bgr(image)
         rgb_image = cv2.cvtColor(cv_image, cv2.COLOR_BGR2RGB)
         pil_image = PILImage.fromarray(rgb_image)
 
@@ -407,7 +408,7 @@ def stop(self) -> None:
 def _temp_dir_context(image: Image) -> Generator[str, None, None]:
     path = tempfile.mkdtemp()
 
-    image.save(f"{path}/00000.jpg")
+    Path(path, "00000.jpg").write_bytes(image_to_jpeg(image))
 
     try:
         yield path

dimos/models/segmentation/test_generated_edge_tam_images.py

@@ -0,0 +1,50 @@
+# Copyright 2026 Dimensional Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from pathlib import Path
+from types import SimpleNamespace
+
+import cv2
+import numpy as np
+import torch
+
+from dimos.models.segmentation import edge_tam
+from dimos.msgs.image import image_from_array
+from dimos.perception.detection.type.detection2d.seg import Detection2DSeg
+
+
+def test_generated_tracker_frame_and_initial_jpeg(monkeypatch):
+    monkeypatch.setattr(
+        edge_tam, "_build_model", lambda: SimpleNamespace(image_size=8, device="cpu")
+    )
+    processor = edge_tam.EdgeTAMProcessor()
+    pixels = np.zeros((12, 16, 3), dtype=np.uint8)
+    pixels[:, :, 0] = 255
+    image = image_from_array(pixels, encoding="rgb8")
+    frame = processor._prepare_frame(image)
+    assert frame.shape == (3, 8, 8)
+    torch.testing.assert_close(
+        frame[:, 0, 0], torch.tensor([(1 - 0.485) / 0.229, -0.456 / 0.224, -0.406 / 0.225])
+    )
+    with edge_tam._temp_dir_context(image) as directory:
+        path = Path(directory)
+        decoded = cv2.imread(str(path / "00000.jpg"))
+        assert decoded.shape == (12, 16, 3)
+        assert decoded[0, 0, 2] > 240
+    assert not path.exists()
+    mask = torch.zeros((12, 16))
+    mask[2:8, 3:10] = 1
+    detection = Detection2DSeg.from_sam2_result(mask, 7, image)
+    assert detection.bbox == (3, 2, 9, 7)
+    assert detection.to_ros_detection2d().header == image.header

dimos/models/segmentation/test_yoloe.py

@@ -19,19 +19,21 @@
 import numpy as np
 from ultralytics.models.yolo.yoloe import YOLOEVPSegPredictor
 
+from dimos.messages.sensor_msgs.msg import Image
+from dimos.messages.std_msgs.msg import Header
 from dimos.models.segmentation.yoloe import YoloeBoxSegmenter
-from dimos.msgs.sensor_msgs.Image import Image, ImageFormat
+from dimos.msgs.image import image_from_array, image_to_bgr
+from dimos.msgs.time import time_from_seconds, to_seconds
 from dimos.perception.detection.type.detection2d.bbox import Detection2DBBox
 from dimos.perception.detection.type.detection2d.imageDetections2D import ImageDetections2D
 from dimos.perception.detection.type.detection2d.seg import Detection2DSeg
 
 
 def _image() -> Image:
-    return Image(
-        data=np.zeros((32, 48, 3), dtype=np.uint8),
-        format=ImageFormat.BGR,
-        frame_id="camera",
-        ts=12.5,
+    return image_from_array(
+        np.zeros((32, 48, 3), dtype=np.uint8),
+        encoding="bgr8",
+        header=Header(frame_id="camera", stamp=time_from_seconds(12.5)),
     )
 
 
@@ -42,7 +44,7 @@ def _box(image: Image, *, bbox: tuple[float, float, float, float]) -> Detection2
         class_id=7,
         confidence=0.73,
         name="white mug",
-        ts=image.ts,
+        ts=to_seconds(image.header.stamp),
         image=image,
     )
 
@@ -60,7 +62,7 @@ def _mask(
         class_id=prompt_id,
         confidence=0.11,
         name=f"prompt-{prompt_id}",
-        ts=image.ts,
+        ts=to_seconds(image.header.stamp),
         image=image,
         mask=np.full((32, 48), value, dtype=np.uint8),
     )
@@ -137,7 +139,7 @@ def test_visual_box_prompts_use_segmentation_predictor(mocker: Any) -> None:
     get_data.assert_called_once_with("models_yoloe")
     model_factory.assert_called_once_with(Path("/models/yoloe-11s-seg.pt"))
     kwargs = dict(model.predict.call_args.kwargs)
-    np.testing.assert_array_equal(kwargs.pop("source"), image.to_opencv())
+    np.testing.assert_array_equal(kwargs.pop("source"), image_to_bgr(image))
     visual_prompts = kwargs.pop("visual_prompts")
     np.testing.assert_array_equal(
         visual_prompts["bboxes"], np.asarray([detection.bbox], dtype=np.float64)

dimos/models/segmentation/yoloe.py

@@ -23,7 +23,8 @@
 from numpy.typing import NDArray
 from ultralytics.models.yolo.yoloe import YOLOEVPSegPredictor
 
-from dimos.msgs.sensor_msgs.Image import Image
+from dimos.messages.sensor_msgs.msg import Image
+from dimos.msgs.image import image_to_bgr
 from dimos.perception.detection.detectors.yoloe import Yoloe2DDetector, YoloePromptMode
 from dimos.perception.detection.type.detection2d.bbox import Bbox, Detection2DBBox
 from dimos.perception.detection.type.detection2d.imageDetections2D import ImageDetections2D
@@ -58,7 +59,7 @@ def predict_image(self, image: Image) -> ImageDetections2D:
             raise RuntimeError("YOLO-E segmentation requires visual box prompts")
         with self._lock:
             results = self.model.predict(
-                source=image.to_opencv(),
+                source=image_to_bgr(image),
                 device=self.device,
                 conf=self.conf,
                 iou=0.6,