feat: cut typed runtime and browser consumers over to CDR (5/8)

.pre-commit-config.yaml

@@ -140,6 +140,12 @@ repos:
           bash -c '
             set -euo pipefail
             git ls-files "*Cargo.toml" | while read -r m; do
+              # This consumer intentionally references generated packages absent in a clean checkout.
+              # Format its checked-in source directly; native CI builds and runs the full project.
+              if [ "$m" = examples/message-project/rust/Cargo.toml ]; then
+                rustfmt --edition 2024 examples/message-project/rust/src/main.rs
+                continue
+              fi
               cargo locate-project --manifest-path "$m" --workspace --message-format plain
             done | sort -u | while read -r root; do
               cargo fmt --manifest-path "$root" --all

AGENTS.md

@@ -157,7 +157,7 @@ Autonomous subsystems. Communicate via `In[T]`/`Out[T]` typed streams. Run in fo
 from dimos.core.module import Module
 from dimos.core.stream import In, Out
 from dimos.core.core import rpc
-from dimos.msgs.sensor_msgs import Image
+from dimos_generated.sensor_msgs.msg import Image
 
 class MyModule(Module):
     color_image: In[Image]

README.md

@@ -250,8 +250,10 @@ from dimos.core.coordination.blueprints import autoconnect
 from dimos.core.core import rpc
 from dimos.core.module import Module
 from dimos.core.stream import In, Out
-from dimos.msgs.geometry_msgs import Twist
-from dimos.msgs.sensor_msgs import Image, ImageFormat
+from dimos_generated.geometry_msgs.msg import Twist
+from dimos_generated.sensor_msgs.msg import Image
+from dimos.msgs.image import image_from_array
+from dimos.msgs.time import header_now
 
 class RobotConnection(Module):
     cmd_vel: In[Twist]
@@ -263,10 +265,10 @@ class RobotConnection(Module):
 
     def _image_loop(self):
         while True:
-            img = Image.from_numpy(
+            img = image_from_array(
                 np.zeros((120, 160, 3), np.uint8),
-                format=ImageFormat.RGB,
-                frame_id="camera_optical",
+                encoding="rgb8",
+                header=header_now(frame_id="camera_optical"),
             )
             self.color_image.publish(img)
             time.sleep(0.2)
@@ -296,7 +298,7 @@ A blueprint example that connects the image stream from a robot to an MCP-backed
 ```py skip
 from dimos.core.coordination.blueprints import autoconnect
 from dimos.core.transport import LCMTransport
-from dimos.msgs.sensor_msgs import Image
+from dimos_generated.sensor_msgs.msg import Image
 from dimos.robot.unitree.go2.connection import go2_connection
 from dimos.agents.mcp.mcp_client import McpClient
 from dimos.agents.mcp.mcp_server import McpServer
@@ -344,9 +346,9 @@ uv run pytest --numprocesses=auto dimos
 
 ## Multi Language Support
 
-Python is our glue and prototyping language, but we support many languages via LCM interop.
+Python, C++ and Rust share generated `.msg` value types and CDR codecs over raw LCM or Zenoh.
 
-Check our language interop examples:
-- [C++](examples/language-interop/cpp/)
-- [Lua](examples/language-interop/lua/)
-- [TypeScript](examples/language-interop/ts/)
+- [Add and use a message](docs/development/messages.md)
+- [Three-language relay and recording demos](examples/message-codegen/README.md)
+- [C++ virtual robot controller](examples/language-interop/cpp/README.md)
+- [Browser CDR SDK](docs/web/web_sdk.md)

dimos/agents/mcp/mcp_client.py

@@ -406,6 +406,11 @@ def _process_message(
 def _append_image_to_history(
     mcp_client: McpClient, func_name: str, uuid_: str, result: Any
 ) -> None:
+    if isinstance(result, dict) and result.get("type") == "image":
+        result = {
+            "type": "image_url",
+            "image_url": {"url": f"data:{result['mimeType']};base64,{result['data']}"},
+        }
     mcp_client.add_message(
         HumanMessage(
             content=[

dimos/agents/mcp/mcp_server.py

@@ -14,6 +14,7 @@
 from __future__ import annotations
 
 import asyncio
+import base64
 from collections.abc import AsyncGenerator, Callable
 import concurrent.futures
 import json
@@ -22,6 +23,7 @@
 from typing import TYPE_CHECKING, Any
 import uuid
 
+from dimos_generated.sensor_msgs.msg import Image
 from fastapi import FastAPI
 from fastapi.middleware.cors import CORSMiddleware
 from fastapi.responses import JSONResponse
@@ -36,6 +38,7 @@
 from dimos.core.module import Module
 from dimos.core.rpc_client import RpcCall, RPCClient
 from dimos.core.transport_factory import make_transport
+from dimos.msgs.image import image_to_jpeg
 from dimos.utils.logging_config import setup_logger
 from dimos.utils.safe_thread_map import safe_thread_map
 
@@ -221,6 +224,15 @@ def _can_wait(holder: str) -> bool:
     duration = f"{time.monotonic() - t0:.3f}s"
     response = str(result)[:200]
 
+    if isinstance(result, Image):
+        content = {
+            "type": "image",
+            "mimeType": "image/jpeg",
+            "data": base64.b64encode(image_to_jpeg(result)).decode("ascii"),
+        }
+        logger.info("MCP tool done", tool=name, duration=duration, response=response)
+        return _jsonrpc_result(req_id, {"content": [content]})
+
     if hasattr(result, "agent_encode"):
         logger.info("MCP tool done", tool=name, duration=duration, response=response)
         return _jsonrpc_result(req_id, {"content": result.agent_encode()})

dimos/agents/mcp/test_generated_image.py

@@ -0,0 +1,57 @@
+# Copyright 2026 Dimensional Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""A generated CDR image survives MCP serialization and agent history conversion."""
+
+import asyncio
+import base64
+
+import cv2
+from dimos_cdrgen.registry import decode as cdr_decode, encode as cdr_encode
+from dimos_generated.sensor_msgs.msg import Image
+import numpy as np
+
+from dimos.agents.mcp.mcp_client import _append_image_to_history
+from dimos.agents.mcp.mcp_server import handle_request
+from dimos.msgs.image import image_from_array
+
+
+def test_generated_image_mcp_result_reaches_model_history(mocker):
+    pixels = np.zeros((12, 16, 3), dtype=np.uint8)
+    pixels[..., 0] = 255
+    original = image_from_array(pixels, encoding="rgb8")
+    response = asyncio.run(
+        handle_request(
+            {"method": "tools/call", "id": 1, "params": {"name": "observe"}},
+            [],
+            {"observe": lambda: cdr_decode(cdr_encode(original), Image)},
+        )
+    )
+    assert response is not None
+    content = response["result"]["content"]
+    assert len(content) == 1
+    image = content[0]
+    assert image["type"] == "image"
+    assert image["mimeType"] == "image/jpeg"
+    jpeg = base64.b64decode(image["data"])
+    decoded = cv2.imdecode(np.frombuffer(jpeg, dtype=np.uint8), cv2.IMREAD_COLOR)
+    assert decoded.shape == (12, 16, 3)
+    assert decoded[0, 0].tolist() == [0, 0, 254]
+    client = mocker.Mock()
+    _append_image_to_history(client, "observe", "test-image", image)
+    message = client.add_message.call_args.args[0]
+    assert message.content[1] == {
+        "type": "image_url",
+        "image_url": {"url": "data:image/jpeg;base64," + image["data"]},
+    }

dimos/agents/mcp/test_mcp_client.py

@@ -14,11 +14,12 @@
 
 from typing import Any
 
+from dimos_generated.sensor_msgs.msg import Image
 from langchain_core.messages import HumanMessage
 
 from dimos.agents.annotation import skill
 from dimos.core.module import Module
-from dimos.msgs.sensor_msgs.Image import Image
+from dimos.msgs.image import image_from_file
 from dimos.utils.data import get_data
 
 
@@ -177,7 +178,7 @@ class Visualizer(Module):
     @skill
     def take_a_picture(self) -> Image:
         """Takes a picture."""
-        return Image.from_file(get_data("cafe-smol.jpg")).to_rgb()
+        return image_from_file(get_data("cafe-smol.jpg"))
 
 
 def test_image(agent_setup):

dimos/agents/skills/navigation.py

@@ -15,6 +15,8 @@
 import time
 from typing import Any
 
+from dimos_generated.geometry_msgs.msg import Point, Pose, PoseStamped
+from dimos_generated.sensor_msgs.msg import Image
 from reactivex.disposable import Disposable
 
 from dimos.agents.annotation import skill
@@ -24,10 +26,8 @@
 from dimos.core.module import Module
 from dimos.core.stream import In
 from dimos.models.qwen.bbox import BBox
-from dimos.msgs.geometry_msgs.PoseStamped import PoseStamped
-from dimos.msgs.geometry_msgs.Quaternion import Quaternion
-from dimos.msgs.geometry_msgs.Vector3 import Vector3, make_vector3
-from dimos.msgs.sensor_msgs.Image import Image
+from dimos.msgs.geometry import quaternion_euler, quaternion_from_euler
+from dimos.msgs.time import header_now
 from dimos.navigation.base import NavigationState
 from dimos.navigation.go2.replanning_a_star.spec import NavigationInterfaceSpec
 from dimos.perception.experimental.object_tracking_spec import ObjectTrackingSpec
@@ -96,13 +96,13 @@ def tag_location(self, location_name: str) -> str:
         if not self._latest_odom:
             return "No odometry data received yet, cannot tag location."
 
-        position = self._latest_odom.position
-        rotation = self._latest_odom.orientation
+        position = self._latest_odom.pose.position
+        rotation = self._latest_odom.pose.orientation
 
         location = RobotLocation(
             name=location_name,
             position=(position.x, position.y, position.z),
-            rotation=(rotation.x, rotation.y, rotation.z),
+            rotation=quaternion_euler(rotation),
         )
 
         if not self._spatial_memory.tag_location(location):
@@ -158,16 +158,22 @@ def _navigate_by_tagged_location(self, query: str) -> str | None:
 
         logger.info("Found tagged location", location=robot_location)
         goal_pose = PoseStamped(
-            position=make_vector3(*robot_location.position),
-            orientation=Quaternion.from_euler(Vector3(*robot_location.rotation)),
-            frame_id="map",
+            header=header_now("map"),
+            pose=Pose(
+                position=Point(
+                    x=robot_location.position[0],
+                    y=robot_location.position[1],
+                    z=robot_location.position[2],
+                ),
+                orientation=quaternion_from_euler(*robot_location.rotation),
+            ),
         )
 
         return self._navigate_to(goal_pose, f"Found a tagged location called '{query}'.")
 
     def _navigate_to(self, pose: PoseStamped, message: str) -> str:
         logger.info(
-            f"Navigating to pose: ({pose.position.x:.2f}, {pose.position.y:.2f}, {pose.position.z:.2f})"
+            f"Navigating to pose: ({pose.pose.position.x:.2f}, {pose.pose.position.y:.2f}, {pose.pose.position.z:.2f})"
         )
         self._navigation.set_goal(pose)
 
@@ -281,7 +287,9 @@ def _get_goal_pose_from_result(self, result: dict[str, Any]) -> PoseStamped | No
         theta = first.get("rot_z", 0)
 
         return PoseStamped(
-            position=make_vector3(pos_x, pos_y, 0),
-            orientation=Quaternion.from_euler(make_vector3(0, 0, theta)),
-            frame_id="map",
+            header=header_now("map"),
+            pose=Pose(
+                position=Point(x=pos_x, y=pos_y, z=0.0),
+                orientation=quaternion_from_euler(0, 0, theta),
+            ),
         )

dimos/agents/skills/observe_skill.py

@@ -12,10 +12,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from dimos_generated.sensor_msgs.msg import Image
+
 from dimos.agents.annotation import skill
 from dimos.core.module import Module
 from dimos.core.stream import In
-from dimos.msgs.sensor_msgs.Image import Image
 
 
 class ObserveSkill(Module):