diff --git a/.github/workflows/cdr-native.yml b/.github/workflows/cdr-native.yml index bf99ece4e7b..571f1039f7e 100644 --- a/.github/workflows/cdr-native.yml +++ b/.github/workflows/cdr-native.yml @@ -25,6 +25,9 @@ jobs: with: python-version: '3.12' enable-cache: true + - uses: denoland/setup-deno@v2 + with: + deno-version: v2.6.10 - name: Install build and media dependencies run: | sudo apt-get update @@ -42,7 +45,36 @@ jobs: dimos/message_utils/test_pointcloud.py \ dimos/message_utils/test_pointcloud_colors.py \ dimos/protocol/tf/test_buffer.py - - name: Test native SDK + - name: Test SDK and build native consumers run: | cargo test --locked --manifest-path native/rust/dimos-module/Cargo.toml cargo test --locked --manifest-path native/rust/dimos-module-macros/Cargo.toml + cargo test --locked --manifest-path dimos/mapping/ray_tracing/rust/Cargo.toml --lib + cargo test --locked --manifest-path dimos/navigation/global_planner/mls_planner/rust/Cargo.toml --lib + cargo test --locked --manifest-path dimos/navigation/local_planner/rust/Cargo.toml --features module,test-bodies + cargo test --locked --manifest-path dimos/navigation/trajectory_follower/fancy/rust/Cargo.toml --features module + cargo test --locked --manifest-path dimos/hardware/sensors/camera/v4l2/rust/Cargo.toml --lib + cargo check --locked --manifest-path dimos/perception/depth2depth_cloud/rust/Cargo.toml --lib + cargo check --locked --manifest-path dimos/hardware/sensors/camera/realsense/rust/Cargo.toml --all-targets --features docs-only + cargo build --locked --manifest-path dimos/mapping/ray_tracing/rust/Cargo.toml --bin voxel_ray_tracing + cargo build --locked --manifest-path dimos/navigation/global_planner/mls_planner/rust/Cargo.toml --bin mls_planner + cargo build --locked --manifest-path dimos/navigation/local_planner/rust/Cargo.toml --features module --bin local_planner + cargo build --locked --manifest-path dimos/navigation/trajectory_follower/fancy/rust/Cargo.toml --features module --bin trajectory_follower + - name: Prepare DimSim dependencies + working-directory: misc/DimSim/cli + run: deno install --node-modules-dir=auto --entrypoint bridge/cdr_test.ts + - name: Exchange generated messages with real native and TypeScript consumers + run: | + export DIMOS_NATIVE_RAY_TRACING_BINARY="$CARGO_TARGET_DIR/debug/voxel_ray_tracing" + export DIMOS_NATIVE_MLS_BINARY="$CARGO_TARGET_DIR/debug/mls_planner" + export DIMOS_NATIVE_LOCAL_PLANNER_BINARY="$CARGO_TARGET_DIR/debug/local_planner" + export DIMOS_NATIVE_FOLLOWER_BINARY="$CARGO_TARGET_DIR/debug/trajectory_follower" + export DIMOS_DIMSIM_DENO="$(command -v deno)" + uv run --no-sync pytest --noconftest --import-mode=importlib -o addopts='' -q \ + dimos/mapping/ray_tracing/test_generated_native.py \ + dimos/navigation/global_planner/mls_planner/test_generated_native.py \ + dimos/navigation/local_planner/test_generated_native.py \ + dimos/robot/unitree/test_dimsim_connection.py \ + dimos/navigation/trajectory_follower/fancy/test_generated_native.py \ + dimos/hardware/sensors/camera/realsense/test_camera.py \ + dimos/perception/depth2depth_cloud/test_depth2depth_cloud_config.py diff --git a/docs/development/cdr-cutover-status.md b/docs/development/cdr-cutover-status.md new file mode 100644 index 00000000000..4c96d1509b0 --- /dev/null +++ b/docs/development/cdr-cutover-status.md @@ -0,0 +1,38 @@ +# CDR cutover and legacy cleanup status + +**WIP.** The shared factory supports generated CDR types and legacy LCM types. +The repository migration is incomplete. Keep legacy schemas, decoders and value +wrappers until their callers are migrated and verified. + +## Inspect remaining boundaries + +Run the static audit from this checkout. It does not import runtime modules or +contact hardware: + +```sh +python scripts/audit_cdr_boundaries.py --output /tmp/cdr-boundaries.json +python scripts/audit_cdr_boundaries.py --check +``` + +`--check` returns nonzero while candidates remain. This snapshot has **312 legacy +In/Out declarations** and **998 legacy imports**. These are syntax matches, not +validated runtime failures. Internal driver values can be valid conversion +boundaries. Inspect complete blueprint port pairs before changing them. + +## Current validation limits + +The default-layer check has 37 passing tests and four failures: + +- The Python `TransportRecorder` skips generated values on LCM and Zenoh. The new + Rust recording path is separate; this does not prove Python capture cutover. +- The XArm and Piper collection graphs do not expose the expected generated + aggregate `coordinator_joint_state` input on their recorder. + +Agent, drone, Habitat and other legacy graphs still need migration. Original +work and prior heads remain in the consolidation backups. This restack does not +complete or discard those migrations. + +Before deleting an API, migrate its callers and verify the selected graph, +capture/replay, source timestamps and schema identity on an installed package. +Keep the converter's explicit historical decode boundary for old recordings. +Nix hardware packaging and physical robot behavior are not verified here. diff --git a/scripts/audit_cdr_boundaries.py b/scripts/audit_cdr_boundaries.py new file mode 100644 index 00000000000..43ede95b5fa --- /dev/null +++ b/scripts/audit_cdr_boundaries.py @@ -0,0 +1,125 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +"""Inventory legacy runtime imports and typed ports without importing robot modules.""" + +from __future__ import annotations + +import argparse +import ast +from collections import Counter +import json +from pathlib import Path + + +def qualified(node: ast.expr, imports: dict[str, str]) -> str: + if isinstance(node, ast.Name): + return imports.get(node.id, node.id) + if isinstance(node, ast.Attribute): + return qualified(node.value, imports) + "." + node.attr + return ast.unparse(node) + + +def legacy(name: str) -> bool: + if name == "dimos_lcm" or name.startswith("dimos_lcm."): + return True + parts = name.split(".") + return len(parts) >= 3 and parts[:2] == ["dimos", "msgs"] and parts[2].endswith("_msgs") + + +def audit(root: Path) -> dict: + imports_found = [] + ports = [] + for path in sorted((root / "dimos").rglob("*.py")): + relative = path.relative_to(root) + if path.name.startswith(("test_", "demo_", "tool_")) or path.name == "conftest.py": + continue + if relative.parts[:2] == ("dimos", "msgs"): + continue + tree = ast.parse(path.read_text(), filename=str(relative)) + imports = {} + for node in ast.walk(tree): + if isinstance(node, ast.ImportFrom): + for alias in node.names: + full = f"{node.module}.{alias.name}" + imports[alias.asname or alias.name] = full + if legacy(full): + imports_found.append( + {"file": str(relative), "line": node.lineno, "symbol": full} + ) + elif isinstance(node, ast.Import): + for alias in node.names: + imports[alias.asname or alias.name.split(".")[0]] = ( + alias.name if alias.asname else alias.name.split(".")[0] + ) + if legacy(alias.name): + imports_found.append( + {"file": str(relative), "line": node.lineno, "symbol": alias.name} + ) + for node in ast.walk(tree): + if not isinstance(node, ast.AnnAssign) or not isinstance( + node.annotation, ast.Subscript + ): + continue + annotation = node.annotation + direction = qualified(annotation.value, imports) + if direction not in {"dimos.core.stream.In", "dimos.core.stream.Out"}: + continue + candidates = [ + qualified(n, imports) + for n in ast.walk(annotation.slice) + if isinstance(n, (ast.Name, ast.Attribute)) + ] + types = sorted({name for name in candidates if legacy(name)}) + if types: + ports.append( + { + "file": str(relative), + "line": node.lineno, + "port": ast.unparse(node.target), + "direction": direction.rsplit(".", 1)[1], + "legacy_types": types, + } + ) + return { + "status": "unfinished" if ports or imports_found else "clear", + "method": "Static Python import and typed In/Out scan; no runtime graph or hardware imports. Internal driver values are not automatically wire defects.", + "legacy_port_count": len(ports), + "legacy_import_count": len(imports_found), + "ports_by_subsystem": dict(sorted(Counter(p["file"].split("/")[1] for p in ports).items())), + "legacy_ports": ports, + "legacy_imports": imports_found, + } + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--root", type=Path, default=Path(__file__).resolve().parents[1]) + parser.add_argument("--output", type=Path) + parser.add_argument( + "--check", action="store_true", help="Fail while any legacy imports or ports remain." + ) + args = parser.parse_args() + report = audit(args.root.resolve()) + text = json.dumps(report, indent=2) + "\n" + if args.output: + args.output.write_text(text) + else: + print(text, end="") + return int(args.check and report["status"] != "clear") + + +if __name__ == "__main__": + raise SystemExit(main())