From b3deeb7e4fc74b7bd3e5de4c96e4a42ce5953be7 Mon Sep 17 00:00:00 2001 From: cc Date: Mon, 21 Sep 2026 10:23:27 -0700 Subject: [PATCH 1/2] feat: package generated messages for external applications --- .github/scripts/check_sdist.py | 4 + .github/scripts/wheel_smoke.py | 6 ++ .github/workflows/ci.yml | 8 ++ .github/workflows/message-codegen.yml | 13 ++- .github/workflows/release-build-check.yml | 3 + .github/workflows/release.yml | 35 +++++++- MANIFEST.in | 4 + dimos/message_codegen/README.md | 16 ++++ dimos/message_codegen/builtin_provider.py | 33 +++++++ dimos/message_codegen/cpp.py | 24 +++++ dimos/message_codegen/definitions.py | 11 ++- dimos/message_codegen/distribution.py | 74 ++++++++++++++++ dimos/message_codegen/generate.py | 67 ++++++++++++-- dimos/message_codegen/native_build.py | 64 ++++++++++++++ dimos/message_codegen/providers.py | 32 +++++++ dimos/message_codegen/registry.py | 29 ++++++ dimos/message_codegen/templates/dimos_cdr.hpp | 4 +- dimos/message_codegen/test_native.py | 5 ++ dimos/message_codegen/test_packages.py | 88 +++++++++++++++++++ examples/message-codegen/EVIDENCE.md | 20 +++++ examples/message-codegen/README.md | 55 ++++++++++++ .../external-app/CMakeLists.txt | 6 ++ .../external-app/demo_installed.py | 55 ++++++++++++ .../message-codegen/external-app/relay.cpp | 18 ++++ .../message-codegen/external-app/relay.rs | 25 ++++++ .../replace-lcm-message-encoding/design.md | 28 ++++++ .../replace-lcm-message-encoding/tasks.md | 14 +-- pyproject.toml | 10 ++- scripts/install.sh | 5 +- scripts/package_messages.sh | 18 ++++ scripts/setup_message_codegen.sh | 7 +- scripts/test_message_packages.sh | 51 +++++++++++ setup.py | 16 +++- uv.lock | 8 +- 34 files changed, 828 insertions(+), 28 deletions(-) mode change 100644 => 100755 .github/scripts/check_sdist.py mode change 100644 => 100755 .github/scripts/wheel_smoke.py create mode 100644 dimos/message_codegen/builtin_provider.py create mode 100644 dimos/message_codegen/distribution.py create mode 100644 dimos/message_codegen/native_build.py create mode 100644 dimos/message_codegen/providers.py create mode 100644 dimos/message_codegen/registry.py create mode 100644 dimos/message_codegen/test_packages.py create mode 100644 examples/message-codegen/external-app/CMakeLists.txt create mode 100644 examples/message-codegen/external-app/demo_installed.py create mode 100644 examples/message-codegen/external-app/relay.cpp create mode 100644 examples/message-codegen/external-app/relay.rs create mode 100644 scripts/package_messages.sh create mode 100644 scripts/test_message_packages.sh diff --git a/.github/scripts/check_sdist.py b/.github/scripts/check_sdist.py old mode 100644 new mode 100755 index 248e62baae5..93c6790f098 --- a/.github/scripts/check_sdist.py +++ b/.github/scripts/check_sdist.py @@ -27,6 +27,10 @@ # Paths below the dimos-/ root that every release sdist must carry. REQUIRED = ( + "dimos/message_codegen/schemas/sensor_msgs/msg/Image.msg", + "dimos/message_codegen/templates/dimos_cdr.hpp", + "dimos/message_codegen/_vendor/rosidl_parser.py", + "scripts/setup_message_codegen.sh", "web/cockpit/dist/index.html", "web/deno.lock", "web/relay/main.ts", diff --git a/.github/scripts/wheel_smoke.py b/.github/scripts/wheel_smoke.py old mode 100644 new mode 100755 index 981efc19d49..d9640729614 --- a/.github/scripts/wheel_smoke.py +++ b/.github/scripts/wheel_smoke.py @@ -25,6 +25,7 @@ from pathlib import Path import urllib.request +from dimos.message_codegen.registry import message_types from dimos.navigation.replanning_a_star.min_cost_astar_ext import min_cost_astar_cpp # noqa: F401 from dimos.web.relay_bridge import locate from dimos.web.relay_bridge.relay_process import RelayProcess @@ -44,6 +45,11 @@ def main() -> None: + types = message_types() + image_type = types["sensor_msgs/msg/Image"] + value = image_type(height=1, width=2, step=2, encoding="mono8", data=[3, 7]) + assert list(image_type.decode(value.encode()).data) == [3, 7] + assert "MSG: std_msgs/msg/Header" in image_type.schema dist = Path(locate.__file__).resolve().parent / "_relay_dist" for rel in REQUIRED: if not (dist / rel).is_file(): diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fc88b3763f3..54b683c5137 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -605,6 +605,8 @@ jobs: sudo apt-get install -y portaudio19-dev libturbojpeg ffmpeg - name: Fetch the go2_short replay dataset run: git lfs pull --include="data/.lfs/go2_short.db.tar.gz" --exclude="" + - name: Build standalone CDR dependency + run: bash scripts/setup_message_codegen.sh - name: Install dependencies run: uv sync --group tests --group browser-tests --frozen - name: Install Playwright browsers # the e2e runs in both engines @@ -709,6 +711,8 @@ jobs: - name: Provision native modules from Cachix if: steps.native-marker.outputs.cache-hit == 'true' run: python3 bin/build-native-modules --link-results .cachix-marker/links.txt + - name: Build standalone CDR dependency + run: bash scripts/setup_message_codegen.sh - name: Install dependencies run: uv sync --group tests --frozen - name: Run tests @@ -842,6 +846,8 @@ jobs: run: | uv venv .codecov-venv uv pip install --python .codecov-venv/bin/python codecov-cli==11.2.8 + - name: Build standalone CDR dependency + run: bash scripts/setup_message_codegen.sh - name: Install dependencies run: uv sync --group tests-self-hosted --frozen - name: Install web_browser e2e dependencies (macOS) @@ -1089,6 +1095,8 @@ jobs: libpango-1.0-0 libxcomposite1 libxdamage1 libxkbcommon0 libxrandr2 \ `# Rerun viewer pulls in X11 keyboard bindings when DISPLAY is set.` \ libxkbcommon-x11-0 + - name: Build standalone CDR dependency + run: bash scripts/setup_message_codegen.sh - name: Install dependencies run: uv sync --group tests-self-hosted --frozen - name: Build C++ extensions in-place diff --git a/.github/workflows/message-codegen.yml b/.github/workflows/message-codegen.yml index 59085c86b28..1655f0564ef 100644 --- a/.github/workflows/message-codegen.yml +++ b/.github/workflows/message-codegen.yml @@ -7,6 +7,7 @@ on: - examples/message-codegen/** - native/rust/dimos-lcm-transport/** - scripts/*message*codegen* + - scripts/test_message_packages.sh - scripts/vendor_message_definitions.py - .github/workflows/message-codegen.yml - .github/docker/message-codegen-reference.Dockerfile @@ -28,12 +29,22 @@ jobs: run: | test ! -d /opt/ros uv venv .venv --python 3.12 - uv pip install --python .venv/bin/python pybind11==3.0.1 rosbags==0.11.0 numpy pytest pytest-asyncio pytest-env lcm-dimos-fork + uv pip install --python .venv/bin/python pybind11==3.0.1 rosbags==0.11.0 numpy pytest pytest-asyncio pytest-env lcm-dimos-fork setuptools wheel .venv/bin/python -c "import importlib.util; assert importlib.util.find_spec('rclpy') is None" - name: Build the pinned standalone CDR library run: bash scripts/setup_message_codegen.sh - name: Generate, build, test, and demonstrate all three languages run: bash scripts/test_message_codegen.sh + - name: Build and consume wheel, sdist, CMake, and Cargo packages + run: bash scripts/test_message_packages.sh + - name: Retain external application and packages + uses: actions/upload-artifact@v7 + with: + name: message-package-evidence + path: | + build/message-codegen/external-app/evidence/ + build/message-codegen/external-app/dist/ + build/message-codegen/external-app/generated/rust/target/package/*.crate - name: Retain executable demo evidence if: always() uses: actions/upload-artifact@v7 diff --git a/.github/workflows/release-build-check.yml b/.github/workflows/release-build-check.yml index 60e33bdd775..a77e6f37757 100644 --- a/.github/workflows/release-build-check.yml +++ b/.github/workflows/release-build-check.yml @@ -13,6 +13,8 @@ on: - .github/scripts/wheel_smoke.py - pyproject.toml - setup.py + - dimos/message_codegen/** + - scripts/setup_message_codegen.sh - MANIFEST.in - web/cockpit/** - web/sdk/** @@ -31,6 +33,7 @@ jobs: env: # Keep in sync with the build-wheels env in release.yml CIBW_MANYLINUX_X86_64_IMAGE: manylinux_2_34 + CIBW_BEFORE_BUILD: "bash {project}/scripts/setup_message_codegen.sh" CIBW_TEST_COMMAND: "python {project}/.github/scripts/wheel_smoke.py" steps: - name: Checkout diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 51cb70d6b4c..c16b43fbb41 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -122,6 +122,7 @@ jobs: CIBW_MANYLINUX_X86_64_IMAGE: manylinux_2_34 # open3d>=0.19 requires glibc 2.31+ CIBW_MANYLINUX_AARCH64_IMAGE: manylinux_2_34 # open3d>=0.19 requires glibc 2.31+ # Asserts the packaged relay + Cockpit and serves them from the installed wheel. + CIBW_BEFORE_BUILD: "bash {project}/scripts/setup_message_codegen.sh" CIBW_TEST_COMMAND: "python {project}/.github/scripts/wheel_smoke.py" MACOSX_DEPLOYMENT_TARGET: "11.0" steps: @@ -141,6 +142,26 @@ jobs: name: wheels-${{ matrix.os }}-${{ matrix.arch }} path: wheelhouse/*.whl + build-message-packages: + needs: validate + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - uses: actions/checkout@v7 + - uses: astral-sh/setup-uv@v10.0.1 + - uses: dtolnay/rust-toolchain@stable + - run: uv venv .venv --python 3.12 + - run: bash scripts/setup_message_codegen.sh + - name: Package CMake headers, schemas, and native Rust sources + env: + MESSAGE_VERSION: ${{ needs.validate.outputs.version }} + run: bash scripts/package_messages.sh "$MESSAGE_VERSION" + - uses: actions/upload-artifact@v7 + with: + name: native-message-packages + path: build/message-codegen/release/dist/* + publish-testpypi: if: ${{ inputs.dry-run }} needs: [validate, build-sdist, build-wheels] @@ -155,6 +176,12 @@ jobs: - name: Download wheels/sdist uses: actions/download-artifact@v8 with: + name: sdist + path: dist + - name: Download wheels + uses: actions/download-artifact@v8 + with: + pattern: wheels-* path: dist merge-multiple: true - name: Install uv @@ -184,6 +211,12 @@ jobs: - name: Download wheels/sdist uses: actions/download-artifact@v8 with: + name: sdist + path: dist + - name: Download wheels + uses: actions/download-artifact@v8 + with: + pattern: wheels-* path: dist merge-multiple: true - name: Install uv @@ -199,7 +232,7 @@ jobs: tag-and-release: if: ${{ ! inputs.dry-run }} - needs: [validate, publish-pypi] + needs: [validate, publish-pypi, build-message-packages] runs-on: ubuntu-latest environment: release-tag permissions: diff --git a/MANIFEST.in b/MANIFEST.in index 3aee0bf04e2..23c70f4be9c 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -9,6 +9,10 @@ global-exclude .DS_Store # runtime data and must stay out. Add a new extension here when code starts # loading a new data type — never go back to per-package globs. recursive-include dimos *.yaml *.yml *.json *.urdf *.html *.css *.js *.svg *.tcss +graft dimos/message_codegen/schemas +graft dimos/message_codegen/templates +include dimos/message_codegen/_vendor/*.pyi +include scripts/setup_message_codegen.sh # --- Exclusions (must come after the includes above so they win) --- # Test fixtures must never ship. diff --git a/dimos/message_codegen/README.md b/dimos/message_codegen/README.md index 447d5f58602..3a2cb4ae568 100644 --- a/dimos/message_codegen/README.md +++ b/dimos/message_codegen/README.md @@ -47,3 +47,19 @@ Applications and builds never run the maintenance downloader. This work is being delivered through the `replace-lcm-message-encoding` OpenSpec change. The generated pipeline is under development; the old runtime message APIs have not yet been replaced. + +## Distribution + +Pass `--package` to emit a setuptools source project in `python/`, alongside the +CMake project and Cargo crate. `--python-module` gives independent message +packages distinct extension names; `--version` sets the package version. +The Python sdist contains the generator, its pinned parser, and all definition +inputs. Building the sdist regenerates source without ROS. The wheel contains +native code plus definitions, licenses, and the `dimos.messages` provider. + +DimOS's own wheel uses the same `MessageBuildExt` and generator from `setup.py`. +The release workflow preserves the existing Linux x86_64/aarch64 and macOS arm64 +wheel matrix and adds CMake/schema and Cargo source packages to GitHub releases. +Source developers run `bash scripts/setup_message_codegen.sh` before building; +`scripts/install.sh --mode dev` does this during dependency setup. Ordinary wheel +users need no compiler, generator run, or ROS installation. diff --git a/dimos/message_codegen/builtin_provider.py b/dimos/message_codegen/builtin_provider.py new file mode 100644 index 00000000000..02ac387693d --- /dev/null +++ b/dimos/message_codegen/builtin_provider.py @@ -0,0 +1,33 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""The message definitions and native types shipped in the DimOS wheel.""" + +from importlib import import_module +from pathlib import Path +from typing import Any + +from dimos.message_codegen.definitions import Definitions + + +def schema_root() -> Path: + return Path(__file__).with_name("schemas") + + +def message_types() -> dict[str, Any]: + extension = import_module("dimos_generated") + return { + message.name: getattr(getattr(extension, message.package).msg, message.short_name) + for message in Definitions([]).resolve() + } diff --git a/dimos/message_codegen/cpp.py b/dimos/message_codegen/cpp.py index de665776833..6cac6108dfe 100644 --- a/dimos/message_codegen/cpp.py +++ b/dimos/message_codegen/cpp.py @@ -16,6 +16,8 @@ from __future__ import annotations +from dataclasses import asdict +from hashlib import sha256 import json import math from typing import Any @@ -201,6 +203,22 @@ def validation(message: Message) -> list[str]: def generate(messages: tuple[Message, ...]) -> str: + guards = { + message.name: "DIMOS_MESSAGE_" + + sha256( + json.dumps( + [ + message.name, + [asdict(field) for field in message.fields], + [asdict(constant) for constant in message.constants], + ], + sort_keys=True, + ).encode() + ) + .hexdigest() + .upper() + for message in messages + } lines = [ "// Generated from ROS2 .msg definitions. Do not edit.", "#pragma once", @@ -215,6 +233,8 @@ def generate(messages: tuple[Message, ...]) -> str: '#include "dimos_cdr.hpp"', ] for message in messages: + guard = guards[message.name] + "_TYPE" + lines.extend([f"#ifndef {guard}", f"#define {guard}"]) namespace = qualified(message.name.rsplit("/", 1)[0]) name = identifier(message.short_name) lines.extend([f"namespace {namespace} {{", f"struct {name} {{"]) @@ -236,9 +256,12 @@ def generate(messages: tuple[Message, ...]) -> str: lines.append("void validate() const {") lines.extend(validation(message)) lines.extend(["}", f'static constexpr const char* msg_name = "{message.name}";', "};", "}"]) + lines.append("#endif") lines.append("namespace eprosima::fastcdr {") for message in messages: + guard = guards[message.name] + "_CODEC" + lines.extend([f"#ifndef {guard}", f"#define {guard}"]) name = qualified(message.name) lines.extend( [ @@ -269,5 +292,6 @@ def generate(messages: tuple[Message, ...]) -> str: if not message.fields: lines.append("uint8_t unused; cdr >> unused;") lines.extend(["value.validate();", "}"]) + lines.append("#endif") lines.append("}") return "\n".join(lines) + "\n" diff --git a/dimos/message_codegen/definitions.py b/dimos/message_codegen/definitions.py index 380d5ac4a56..7ad27e45b72 100644 --- a/dimos/message_codegen/definitions.py +++ b/dimos/message_codegen/definitions.py @@ -28,6 +28,7 @@ InvalidValue, parse_message_string, ) +from dimos.message_codegen.providers import schema_roots BUNDLED_SCHEMAS = Path(__file__).with_name("schemas") @@ -142,9 +143,15 @@ def parse_message(path: Path) -> Message: class Definitions: """A closed set of local definitions with dependency-first resolution.""" - def __init__(self, roots: Iterable[Path], *, bundled: bool = True) -> None: + def __init__( + self, roots: Iterable[Path], *, bundled: bool = True, installed: bool = False + ) -> None: self._messages: dict[str, Message] = {} - for root in ([BUNDLED_SCHEMAS] if bundled else []) + list(roots): + for root in ( + ([BUNDLED_SCHEMAS] if bundled else []) + + list(roots) + + (list(schema_roots()) if installed else []) + ): if not root.is_dir(): raise ValueError(f"Schema root does not exist: {root}") for path in sorted(root.rglob("*.msg")): diff --git a/dimos/message_codegen/distribution.py b/dimos/message_codegen/distribution.py new file mode 100644 index 00000000000..f2be856bf92 --- /dev/null +++ b/dimos/message_codegen/distribution.py @@ -0,0 +1,74 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Emit a standard setuptools source package around generated native messages.""" + +from __future__ import annotations + +import json +from pathlib import Path +import shutil + + +def write_distribution( + output: Path, module: str, names: tuple[str, ...], version: str = "0.1.0" +) -> None: + project = output / "python" + support = module + "_schemas" + package = project / support + package.mkdir(parents=True, exist_ok=True) + (package / "__init__.py").write_text("") + if (package / "schemas").exists(): + shutil.rmtree(package / "schemas") + shutil.copytree(output / "schemas", package / "schemas") + shutil.copyfile(output / "schemas.json", package / "schemas.json") + (package / "provider.py").write_text( + "from importlib import import_module\nimport json\nfrom pathlib import Path\n\n" + "def schema_root():\n return Path(__file__).with_name('schemas')\n\n" + "def message_types():\n" + f" extension = import_module({module!r})\n" + " names = json.loads(Path(__file__).with_name('schemas.json').read_text())\n" + " return {name: getattr(getattr(extension, name.split('/')[0]).msg, name.split('/')[-1]) for name in names}\n" + ) + toolkit = project / "_codegen" / "dimos" / "message_codegen" + shutil.copytree( + Path(__file__).parent, + toolkit, + dirs_exist_ok=True, + ignore=shutil.ignore_patterns("__pycache__", "test_*.py"), + ) + (project / "pyproject.toml").write_text( + '[build-system]\nrequires = ["setuptools>=70", "wheel", "pybind11==3.0.1"]\n' + 'build-backend = "setuptools.build_meta"\n' + ) + (project / "setup.py").write_text( + "from pathlib import Path\nimport sys\nfrom setuptools import setup\n" + "sys.path.insert(0, str(Path(__file__).parent / '_codegen'))\n" + "from dimos.message_codegen.generate import generate\n" + "from dimos.message_codegen.native_build import MessageExtension, MessageBuildExt\n" + "root = Path(__file__).parent\n" + f"generate([root / {support!r} / 'schemas'], root / 'build/messages', {list(names)!r}, {module!r}, version={version!r})\n" + f"setup(name={module.replace('_', '-')!r}, version={version!r}, packages=[{support!r}],\n" + f" package_data={{{support!r}: ['schemas.json', 'schemas/**/*']}},\n" + f" entry_points={{'dimos.messages': [{(module + '=' + support + '.provider')!r}]}},\n" + f" ext_modules=[MessageExtension({module!r}, root / 'build/messages/cpp')],\n" + " cmdclass={'build_ext': MessageBuildExt})\n" + ) + (project / "MANIFEST.in").write_text( + f"graft _codegen\ngraft {support}\ninclude pyproject.toml setup.py\n" + "global-exclude __pycache__ *.pyc\n" + ) + (project / "message-package.json").write_text( + json.dumps({"module": module, "types": names}, indent=2) + "\n" + ) diff --git a/dimos/message_codegen/generate.py b/dimos/message_codegen/generate.py index 6ee47e1443a..c1acf694ab9 100644 --- a/dimos/message_codegen/generate.py +++ b/dimos/message_codegen/generate.py @@ -25,17 +25,26 @@ from dimos.message_codegen import cpp, python, rust from dimos.message_codegen.definitions import Definitions +from dimos.message_codegen.distribution import write_distribution TEMPLATES = Path(__file__).with_name("templates") def generate( - roots: list[Path], output: Path, names: list[str] | None = None, module: str = "dimos_generated" + roots: list[Path], + output: Path, + names: list[str] | None = None, + module: str = "dimos_generated", + *, + installed: bool = False, + version: str = "0.1.0", ) -> tuple[str, ...]: """Emit all three native language outputs, resolving dependencies first.""" if not re.fullmatch(r"[a-z][a-z0-9_]*", module) or keyword.iskeyword(module): raise ValueError(f"Invalid Python extension module name: {module}") - definitions = Definitions(roots) + if not re.fullmatch(r"[0-9]+\.[0-9]+\.[0-9]+", version): + raise ValueError("Package version must have the form major.minor.patch") + definitions = Definitions(roots, installed=installed) messages = definitions.resolve(names) for message in messages: for field in message.fields: @@ -55,26 +64,61 @@ def generate( sources = " ".join(sorted(bindings)) (native / "CMakeLists.txt").write_text( "cmake_minimum_required(VERSION 3.20)\n" - "project(dimos_generated_messages LANGUAGES CXX)\n" + f"project({module} VERSION {version} LANGUAGES CXX)\n" "find_package(fastcdr 2.4.0 EXACT REQUIRED)\n" + "include(GNUInstallDirs)\n" + f"add_library({module}_messages INTERFACE)\n" + f"set_target_properties({module}_messages PROPERTIES EXPORT_NAME messages)\n" + f"target_compile_features({module}_messages INTERFACE cxx_std_17)\n" + f"target_include_directories({module}_messages INTERFACE $ $)\n" + f"target_link_libraries({module}_messages INTERFACE fastcdr)\n" + f"install(TARGETS {module}_messages EXPORT {module}Targets)\n" + f"install(FILES messages.hpp dimos_cdr.hpp DESTINATION include/{module})\n" + f"install(EXPORT {module}Targets NAMESPACE {module}:: DESTINATION lib/cmake/{module})\n" + f"install(FILES {module}Config.cmake DESTINATION lib/cmake/{module})\n" + f"install(DIRECTORY ../schemas DESTINATION share/{module})\n" + 'option(DIMOS_BUILD_PYTHON "Build Python bindings" ON)\n' + "if(DIMOS_BUILD_PYTHON)\n" "find_package(Python COMPONENTS Interpreter Development.Module REQUIRED)\n" 'execute_process(COMMAND "${Python_EXECUTABLE}" -m pybind11 --cmakedir OUTPUT_VARIABLE pybind11_DIR OUTPUT_STRIP_TRAILING_WHITESPACE COMMAND_ERROR_IS_FATAL ANY)\n' "find_package(pybind11 3.0.1 EXACT REQUIRED)\n" f"pybind11_add_module({module} NO_EXTRAS {sources})\n" f"target_compile_features({module} PRIVATE cxx_std_17)\n" f"target_link_libraries({module} PRIVATE fastcdr)\n" + "endif()\n" + ) + (native / f"{module}Config.cmake").write_text( + "include(CMakeFindDependencyMacro)\nfind_dependency(fastcdr 2.4.0 EXACT)\n" + f'include("${{CMAKE_CURRENT_LIST_DIR}}/{module}Targets.cmake")\n' ) crate = output / "rust" (crate / "src").mkdir(parents=True, exist_ok=True) (crate / "src" / "lib.rs").write_text(rust.generate(messages, definitions)) shutil.copyfile(TEMPLATES / "codec.rs", crate / "src" / "codec.rs") (crate / "Cargo.toml").write_text( - '[package]\nname = "dimos-generated-messages"\nversion = "0.1.0"\nedition = "2024"\n' + f'[package]\nname = "{module.replace("_", "-")}-messages"\nversion = "{version}"\nedition = "2024"\n' + 'license = "Apache-2.0"\ndescription = "Native CDR messages generated from ROS2 definitions"\n' + 'include = ["src/*.rs", "schemas.json", "schemas/**", "Cargo.toml"]\n' '[workspace]\n[dependencies]\nserde = { version = "1.0", features = ["derive"] }\n' 'serde-big-array = "=0.5.1"\nre_cdr = "=0.1.0"\n' ) metadata = {message.name: definitions.schema(message.name) for message in messages} (output / "schemas.json").write_text(json.dumps(metadata, indent=2, sort_keys=True) + "\n") + shutil.copyfile(output / "schemas.json", crate / "schemas.json") + schema_root = output / "schemas" + if schema_root.exists(): + shutil.rmtree(schema_root) + for message in messages: + path = schema_root / (message.name + ".msg") + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(message.text) + shutil.copytree(TEMPLATES.parent / "schemas" / "licenses", schema_root / "licenses") + for source in (TEMPLATES.parent / "schemas").glob("*/package.xml"): + if (schema_root / source.parent.name).is_dir(): + shutil.copyfile(source, schema_root / source.parent.name / "package.xml") + if (crate / "schemas").exists(): + shutil.rmtree(crate / "schemas") + shutil.copytree(schema_root, crate / "schemas", dirs_exist_ok=True) return tuple(message.name for message in messages) @@ -86,9 +130,22 @@ def main() -> None: ) parser.add_argument("--output", type=Path, required=True) parser.add_argument("--python-module", default="dimos_generated") + parser.add_argument("--version", default="0.1.0") + parser.add_argument( + "--package", action="store_true", help="Emit a reproducible Python wheel/sdist project" + ) args = parser.parse_args() try: - names = generate(args.package_root, args.output, args.names, args.python_module) + names = generate( + args.package_root, + args.output, + args.names, + args.python_module, + installed=True, + version=args.version, + ) + if args.package: + write_distribution(args.output, args.python_module, names, args.version) except ValueError as exc: parser.error(str(exc)) print(f"Generated {len(names)} message types for Python, C++, and Rust in {args.output}") diff --git a/dimos/message_codegen/native_build.py b/dimos/message_codegen/native_build.py new file mode 100644 index 00000000000..c374215bb75 --- /dev/null +++ b/dimos/message_codegen/native_build.py @@ -0,0 +1,64 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Build generated extensions using the project's setuptools native build path.""" + +from __future__ import annotations + +import os +from pathlib import Path +import subprocess +import sys +from typing import Any + +from pybind11.setup_helpers import build_ext +from setuptools import Extension + + +class MessageExtension(Extension): + def __init__(self, name: str, source_dir: Path, prefix: Path | None = None) -> None: + super().__init__(name, sources=[]) + self.cmake_source_dir = source_dir.resolve() + self.cmake_prefix = prefix + + +class MessageBuildExt(build_ext): + def build_extension(self, extension: Any) -> None: + if not isinstance(extension, MessageExtension): + super().build_extension(extension) + return + destination = Path(self.get_ext_fullpath(extension.name)).resolve().parent + build = Path(self.build_temp).resolve() / extension.name + extra = [] + if extension.cmake_prefix is not None: + prefixes = [ + str(extension.cmake_prefix.resolve()), + os.environ.get("CMAKE_PREFIX_PATH", ""), + ] + extra.append("-DCMAKE_PREFIX_PATH=" + ";".join(prefixes)) + subprocess.run( + [ + "cmake", + "-S", + str(extension.cmake_source_dir), + "-B", + str(build), + "-DCMAKE_BUILD_TYPE=Release", + f"-DPython_EXECUTABLE={sys.executable}", + f"-DCMAKE_LIBRARY_OUTPUT_DIRECTORY={destination}", + *extra, + ], + check=True, + ) + subprocess.run(["cmake", "--build", str(build), "--parallel", "2"], check=True) diff --git a/dimos/message_codegen/providers.py b/dimos/message_codegen/providers.py new file mode 100644 index 00000000000..edb5b42eb35 --- /dev/null +++ b/dimos/message_codegen/providers.py @@ -0,0 +1,32 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Installed message providers, discovered through Python package entry points.""" + +from __future__ import annotations + +from importlib.metadata import entry_points +from pathlib import Path +from types import ModuleType + + +def providers() -> tuple[ModuleType, ...]: + return tuple( + entry.load() + for entry in sorted(entry_points(group="dimos.messages"), key=lambda item: item.name) + ) + + +def schema_roots() -> tuple[Path, ...]: + return tuple(Path(provider.schema_root()) for provider in providers()) diff --git a/dimos/message_codegen/registry.py b/dimos/message_codegen/registry.py new file mode 100644 index 00000000000..b1f3b87b4ab --- /dev/null +++ b/dimos/message_codegen/registry.py @@ -0,0 +1,29 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Discover installed generated types after checking their definition closure.""" + +from typing import Any + +from dimos.message_codegen.definitions import Definitions +from dimos.message_codegen.providers import providers, schema_roots + + +def message_types() -> dict[str, Any]: + Definitions(schema_roots()).resolve() + result: dict[str, Any] = {} + for provider in providers(): + for name, message_type in provider.message_types().items(): + result.setdefault(name, message_type) + return result diff --git a/dimos/message_codegen/templates/dimos_cdr.hpp b/dimos/message_codegen/templates/dimos_cdr.hpp index 40f81ff51de..844b54b23af 100644 --- a/dimos/message_codegen/templates/dimos_cdr.hpp +++ b/dimos/message_codegen/templates/dimos_cdr.hpp @@ -12,7 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -#pragma once +#ifndef DIMOS_CDR_RUNTIME_V1_HPP +#define DIMOS_CDR_RUNTIME_V1_HPP #include #include @@ -69,3 +70,4 @@ T decode(const std::vector& bytes) { } } // namespace dimos::cdr +#endif diff --git a/dimos/message_codegen/test_native.py b/dimos/message_codegen/test_native.py index 7e29108028f..e1c2da6a49e 100644 --- a/dimos/message_codegen/test_native.py +++ b/dimos/message_codegen/test_native.py @@ -23,6 +23,11 @@ from rosbags.typesys import Stores, get_types_from_msg, get_typestore generated = pytest.importorskip("dimos_generated", reason="Build examples/message-codegen first") +if not hasattr(generated, "demo_msgs"): + pytest.skip( + "Build examples/message-codegen and add its extension to PYTHONPATH", + allow_module_level=True, + ) Telemetry = generated.demo_msgs.msg.Telemetry ImageEnvelope = generated.demo_msgs.msg.ImageEnvelope Image = generated.sensor_msgs.msg.Image diff --git a/dimos/message_codegen/test_packages.py b/dimos/message_codegen/test_packages.py new file mode 100644 index 00000000000..950d5f37489 --- /dev/null +++ b/dimos/message_codegen/test_packages.py @@ -0,0 +1,88 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from types import ModuleType + +import pytest + +from dimos.message_codegen import providers, registry +from dimos.message_codegen.definitions import Definitions +from dimos.message_codegen.distribution import write_distribution +from dimos.message_codegen.generate import generate + + +def test_generation_is_reproducible_and_removes_stale_schemas(tmp_path): + first, second = tmp_path / "one", tmp_path / "two" + generate([], first, ["sensor_msgs/msg/Image"]) + generate([], second, ["sensor_msgs/msg/Image"]) + + def contents(root): + return {str(p.relative_to(root)): p.read_bytes() for p in root.rglob("*") if p.is_file()} + + assert contents(first) == contents(second) + + generate([], first, ["geometry_msgs/msg/Point"]) + + assert not (first / "schemas/sensor_msgs/msg/Image.msg").exists() + assert not (first / "rust/schemas/sensor_msgs/msg/Image.msg").exists() + assert (first / "schemas/geometry_msgs/msg/Point.msg").exists() + + +def test_invalid_definition_does_not_create_output(tmp_path): + source = tmp_path / "input/custom_msgs/msg/Value.msg" + source.parent.mkdir(parents=True) + source.write_text("missing_msgs/Absent value\n") + output = tmp_path / "output" + + with pytest.raises(ValueError, match="unresolved"): + generate([tmp_path / "input"], output, ["custom_msgs/msg/Value"]) + + assert not output.exists() + + +def test_distribution_carries_source_closure_and_pinned_generator(tmp_path): + names = generate([], tmp_path, ["sensor_msgs/msg/Image"]) + write_distribution(tmp_path, "example_messages", names) + project = tmp_path / "python" + + assert (project / "example_messages_schemas/schemas/std_msgs/msg/Header.msg").is_file() + assert (project / "_codegen/dimos/message_codegen/_vendor/rosidl_parser.py").is_file() + assert "dimos.messages" in (project / "setup.py").read_text() + assert "pybind11==3.0.1" in (project / "pyproject.toml").read_text() + + +def test_installed_schema_discovery_does_not_import_native_types(tmp_path, monkeypatch): + source = tmp_path / "custom_msgs/msg/Value.msg" + source.parent.mkdir(parents=True) + source.write_text("geometry_msgs/Point position\n") + provider = ModuleType("external_provider") + monkeypatch.setattr(provider, "schema_root", lambda: tmp_path, raising=False) + monkeypatch.setattr(providers, "providers", lambda: (provider,)) + + names = [ + message.name + for message in Definitions([], installed=True).resolve(["custom_msgs/msg/Value"]) + ] + + assert names == ["geometry_msgs/msg/Point", "custom_msgs/msg/Value"] + + +def test_registry_rejects_conflicting_installed_definitions(tmp_path, monkeypatch): + source = tmp_path / "geometry_msgs/msg/Point.msg" + source.parent.mkdir(parents=True) + source.write_text("int32 x\n") + monkeypatch.setattr(registry, "schema_roots", lambda: (tmp_path,)) + + with pytest.raises(ValueError, match="Conflicting definition geometry_msgs/msg/Point"): + registry.message_types() diff --git a/examples/message-codegen/EVIDENCE.md b/examples/message-codegen/EVIDENCE.md index ceb872a97de..441b0eabd23 100644 --- a/examples/message-codegen/EVIDENCE.md +++ b/examples/message-codegen/EVIDENCE.md @@ -76,3 +76,23 @@ ROS2 Jazzy confirms declared defaults, nested fields, bounds, arrays, Unicode, a Reproduce with the commands in [README.md](README.md). CI retains fresh output and binary payloads as artifacts; generated sources and build products are ignored. + +## Stage 2 installed application + +Built a Python wheel from its sdist, installed CMake packages, and compiled a +separate consumer from the Cargo archive. The Python consumer ran with `-I` in a +fresh environment containing only the external message wheel. + +```text +Installed Python package sends new field: added-locally +Installed C++ package received: added-locally +Packaged Rust crate received: added-locally/cpp +Installed Python package receives: added-locally/cpp/rust +A locally added field crossed three installed native packages; no DimOS source change or upstream PR. +``` + +The full DimOS wheel also built with web assets. Its sdist passed the repository +content check (1,928 entries; 10.7 MB). A fresh environment with both wheels +discovered 141 types and accepted a built-in Point inside external Telemetry. +Two separately installed CMake packages can include the same standard Point +without duplicate definitions. Focused tests now pass 34 cases. diff --git a/examples/message-codegen/README.md b/examples/message-codegen/README.md index cf13a5b30c8..22d307f8e24 100644 --- a/examples/message-codegen/README.md +++ b/examples/message-codegen/README.md @@ -117,3 +117,58 @@ are reproducible measurements, not platform-independent performance guarantees. This stage provides standalone generation and codecs. DimOS runtime consumers still use the old types; their coordinated replacement follows in the stack. + +## Stage 2: an installed external application + +After the stage-1 setup, run: + +```bash +uv pip install --python .venv/bin/python setuptools wheel +bash scripts/test_message_packages.sh +``` + +This copies the demo definitions into an ignored external project, adds +`string application_note "added-locally"` to its own Telemetry definition, +and invokes the same generator with `--package`. It builds a wheel **from the +sdist**, installs the wheel in a fresh virtual environment, installs the CMake +package, and builds a separate Rust consumer from the packaged `.crate` archive. +The terminal prints: + +```text +Installed Python package sends new field: added-locally +Installed C++ package received: added-locally +Packaged Rust crate received: added-locally/cpp +Installed Python package receives: added-locally/cpp/rust +``` + +The application runs with Python's isolated flag and no `PYTHONPATH`. Its wheel +contains its own extension, schema dependency closure, upstream licenses, and +`dimos.messages` provider. It needs neither DimOS nor ROS to encode/decode. +No background services are started. Outputs and the captured terminal demo live +under `build/message-codegen/external-app/`; remove that directory to clean up. + +For your own package, use: + +```bash +python -m dimos.message_codegen.generate \ + --package-root path/to/interfaces --type my_msgs/msg/Reading \ + --python-module my_robot_messages --version 0.1.0 --package --output build/messages +CMAKE_PREFIX_PATH=/path/to/fastcdr uv build build/messages/python +cmake -S build/messages/cpp -B build/messages/cmake \ + -DDIMOS_BUILD_PYTHON=OFF -DCMAKE_PREFIX_PATH=/path/to/fastcdr \ + -DCMAKE_INSTALL_PREFIX=/path/to/message-prefix +cmake --install build/messages/cmake +cargo package --manifest-path build/messages/rust/Cargo.toml +``` + +The source package includes the pinned generator and definition inputs; source +builds regenerate locally. Fast CDR must be installed at build time, using the +setup script or an equivalent pinned installation. Wheels statically link it. +Consumers use `find_package(my_robot_messages CONFIG REQUIRED)` and link +`my_robot_messages::messages`, or add the generated Rust crate through Cargo. +Python schema providers are discovered through the standard `dimos.messages` +entry-point group. Definition discovery does not import the native extension; +type discovery validates conflicting qualified definitions before loading types. +Generation, installation from built artifacts, and runtime require no schema +service or runtime download. Dependency setup can use the network; cached Cargo +builds and local wheel installation are exercised offline by the demo. diff --git a/examples/message-codegen/external-app/CMakeLists.txt b/examples/message-codegen/external-app/CMakeLists.txt new file mode 100644 index 00000000000..d5c0813fe28 --- /dev/null +++ b/examples/message-codegen/external-app/CMakeLists.txt @@ -0,0 +1,6 @@ +cmake_minimum_required(VERSION 3.20) +project(external_message_demo LANGUAGES CXX) +find_package(external_telemetry CONFIG REQUIRED) +find_package(dimos_generated CONFIG REQUIRED) +add_executable(relay relay.cpp) +target_link_libraries(relay PRIVATE external_telemetry::messages dimos_generated::messages) diff --git a/examples/message-codegen/external-app/demo_installed.py b/examples/message-codegen/external-app/demo_installed.py new file mode 100644 index 00000000000..cd0e9d5f069 --- /dev/null +++ b/examples/message-codegen/external-app/demo_installed.py @@ -0,0 +1,55 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Run from a fresh environment containing only the external message wheel.""" + +import argparse +from importlib.metadata import entry_points +from pathlib import Path +import subprocess + +from external_telemetry.demo_msgs.msg import Telemetry + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--build", type=Path, required=True) + args = parser.parse_args() + build = args.build.resolve() + provider = next( + entry.load() + for entry in entry_points(group="dimos.messages") + if entry.name == "external_telemetry" + ) + assert provider.message_types()[Telemetry.msg_name] is Telemetry + assert (provider.schema_root() / "demo_msgs/msg/Telemetry.msg").is_file() + value = Telemetry(application_note="added-locally") + print(f"Installed Python package sends new field: {value.application_note}", flush=True) + first, second, third = [build / name for name in ("python.cdr", "cpp.cdr", "rust.cdr")] + first.write_bytes(value.encode()) + subprocess.run([str(build / "cpp-consumer/relay"), str(first), str(second)], check=True) + subprocess.run( + [str(build / "rust-consumer/target/debug/external-consumer"), str(second), str(third)], + check=True, + ) + decoded = Telemetry.decode(third.read_bytes()) + assert decoded.application_note == "added-locally/cpp/rust" + print(f"Installed Python package receives: {decoded.application_note}") + print( + "A locally added field crossed three installed native packages; no DimOS source change or upstream PR." + ) + + +if __name__ == "__main__": + main() diff --git a/examples/message-codegen/external-app/relay.cpp b/examples/message-codegen/external-app/relay.cpp new file mode 100644 index 00000000000..9eced06291f --- /dev/null +++ b/examples/message-codegen/external-app/relay.cpp @@ -0,0 +1,18 @@ +#include +#include +#include +#include +#include + +int main(int argc, char** argv) { + if (argc != 3) return 2; + std::ifstream input(argv[1], std::ios::binary); + std::vector bytes{std::istreambuf_iterator(input), {}}; + auto value = dimos::cdr::decode(bytes); + std::cout << "Installed C++ package received: " << value.application_note << '\n'; + value.application_note += "/cpp"; + auto encoded = dimos::cdr::encode(value); + std::ofstream output(argv[2], std::ios::binary); + output.write(reinterpret_cast(encoded.data()), encoded.size()); + return output ? 0 : 1; +} diff --git a/examples/message-codegen/external-app/relay.rs b/examples/message-codegen/external-app/relay.rs new file mode 100644 index 00000000000..3b41d4d89b2 --- /dev/null +++ b/examples/message-codegen/external-app/relay.rs @@ -0,0 +1,25 @@ +// Copyright 2026 Dimensional Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use external_telemetry_messages::{codec::Message, demo_msgs::msg::Telemetry}; +use std::{env, fs}; + +fn main() -> Result<(), Box> { + let args: Vec<_> = env::args().collect(); + let mut value = Telemetry::decode(&fs::read(&args[1])?)?; + println!("Packaged Rust crate received: {}", value.application_note); + value.application_note.push_str("/rust"); + fs::write(&args[2], value.encode()?)?; + Ok(()) +} diff --git a/openspec/changes/replace-lcm-message-encoding/design.md b/openspec/changes/replace-lcm-message-encoding/design.md index c2c6d003a36..ff424faa897 100644 --- a/openspec/changes/replace-lcm-message-encoding/design.md +++ b/openspec/changes/replace-lcm-message-encoding/design.md @@ -180,3 +180,31 @@ These references informed exploration; pinned versions and executable conformanc - [MCAP schema and encoding registry](https://mcap.dev/spec/registry) - [Foxglove custom schema encodings](https://docs.foxglove.dev/docs/getting-started/custom/custom-schema-encodings) - [Rerun 0.32 MCAP message formats](https://github.com/rerun-io/rerun/blob/0.32.0/docs/content/concepts/logging-and-ingestion/mcap/message-formats.md) + +### Stage 2 packaging evidence + +`--package` emits a standard setuptools project with native bindings, schema +closure, notices, and a `dimos.messages` provider. Its sdist contains the pinned +generator and inputs; the wheel contains compiled code and schema resources. +CMake exports a relocatable target and headers/schema installation; Cargo packages +the native source crate and schemas. Per-definition C++ guards allow two packages +to include the same standard types without duplicate definitions; conflicting +layouts still fail rather than silently overriding one another. + +The external application adds `application_note` in its own copied definition, +builds a wheel from the sdist, installs it into an isolated environment, installs +CMake packages, and builds a Rust consumer from the `.crate` archive. The field +returns as `added-locally/cpp/rust`. A second clean environment installed the full +DimOS wheel and external wheel together: 141 types were discovered, the external +schema resolved through installed providers, and a standard Point from the built-in +package was accepted by the external native Telemetry type. The full DimOS wheel +includes its existing web assets; the sdist content check passed at 10.7 MB. + +The official setuptools build invokes the same generator and native build helper. +Release CI retains its existing platform/Python matrix, builds the pinned Fast +CDR dependency before native extensions, and adds CMake/Cargo source artifacts to +GitHub releases. PyPI jobs download only Python distribution artifacts. Ordinary +wheel users do not run setup downloads, message generation, or a ROS build. +Source developers install Fast CDR during explicit environment setup, including +the existing developer installer. Release publishing itself remains tied to the +normal release workflow; this implementation has not published a release. diff --git a/openspec/changes/replace-lcm-message-encoding/tasks.md b/openspec/changes/replace-lcm-message-encoding/tasks.md index 0c6e633286b..b573868b2fe 100644 --- a/openspec/changes/replace-lcm-message-encoding/tasks.md +++ b/openspec/changes/replace-lcm-message-encoding/tasks.md @@ -15,13 +15,13 @@ ## 2. PR 2 — Package and consume messages outside DimOS -- [ ] 2.1 Add Python wheel/sdist generation using the existing native-build infrastructure; include schema resources and the inputs needed to reproduce source builds. -- [ ] 2.2 Add an installable/exported CMake message package and a consumable generated Rust crate with schema metadata and dependency declarations. -- [ ] 2.3 Register installed Python schema/type providers through `dimos.messages`; support external packages without editing the built-in message registry. -- [ ] 2.4 Wire the same pinned local generation into CI and release packaging, preserving the supported build matrix; upload review artifacts and publish official packages through the normal release workflow. -- [ ] 2.5 Verify deterministic generated sources/metadata, packaged schema closure, clean wheel/sdist/CMake/Cargo consumers, and absence of generated source from version control. -- [ ] 2.6 Extend the demo into a separate application: install built packages, add a field to a local custom `.msg`, regenerate/build all three consumers, and visibly exchange the new field without changing DimOS or publishing a package. -- [ ] 2.7 Document exact external-project commands and setup-time dependencies, verify no runtime downloading/generation, capture demo evidence, and rerun PR 1's demo and automated checks. +- [x] 2.1 Add Python wheel/sdist generation using the existing native-build infrastructure; include schema resources and the inputs needed to reproduce source builds. +- [x] 2.2 Add an installable/exported CMake message package and a consumable generated Rust crate with schema metadata and dependency declarations. +- [x] 2.3 Register installed Python schema/type providers through `dimos.messages`; support external packages without editing the built-in message registry. +- [x] 2.4 Wire the same pinned local generation into CI and release packaging, preserving the supported build matrix; upload review artifacts and publish official packages through the normal release workflow. +- [x] 2.5 Verify deterministic generated sources/metadata, packaged schema closure, clean wheel/sdist/CMake/Cargo consumers, and absence of generated source from version control. +- [x] 2.6 Extend the demo into a separate application: install built packages, add a field to a local custom `.msg`, regenerate/build all three consumers, and visibly exchange the new field without changing DimOS or publishing a package. +- [x] 2.7 Document exact external-project commands and setup-time dependencies, verify no runtime downloading/generation, capture demo evidence, and rerun PR 1's demo and automated checks. ## 3. PR 3 — Inspect the same MCAP in Foxglove and Rerun diff --git a/pyproject.toml b/pyproject.toml index c19d564ca06..a74192b58fd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -2,7 +2,8 @@ requires = [ "setuptools>=70", "wheel", - "pybind11>=2.12", + "pybind11==3.0.1", + "cmake>=3.20", "tomli>=2.0; python_version < '3.11'", ] build-backend = "setuptools.build_meta" @@ -49,6 +50,8 @@ exclude = [ "**/*.svg", "**/*.tcss", "**/*.pyi", + "message_codegen/schemas/**/*", + "message_codegen/templates/*", ] [tool.setuptools.exclude-package-data] @@ -163,6 +166,9 @@ dependencies = [ ] +[project.entry-points."dimos.messages"] +builtin = "dimos.message_codegen.builtin_provider" + [project.scripts] lcmspy = "dimos.cli.spy.run_spy:lcm_main" agentspy = "dimos.cli.agentspy.agentspy:main" @@ -495,7 +501,7 @@ tests-self-hosted = [ # doesn't expose to the build — and these tests never open a live DDS link. "mcap>=1.2.0", # Needed to compile the in-tree extensions. - "pybind11>=2.12", + "pybind11==3.0.1", "dim-odom>=0.4.1", ] docs = [ diff --git a/scripts/install.sh b/scripts/install.sh index 4aaa8f49027..394e7ecfbba 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -21,8 +21,8 @@ trap 'exit 143' TERM INSTALLER_VERSION="0.3.0" # ─── package lists (edit these when dependencies change) ────────────────────── -UBUNTU_PACKAGES="ca-certificates curl git g++ portaudio19-dev git-lfs libturbojpeg pre-commit libgl1 libegl1 libglib2.0-0 ffmpeg libsndfile1 pkg-config" -MACOS_PACKAGES="gnu-sed gcc portaudio git-lfs libjpeg-turbo pre-commit ffmpeg libsndfile pkg-config" +UBUNTU_PACKAGES="ca-certificates curl git g++ cmake portaudio19-dev git-lfs libturbojpeg pre-commit libgl1 libegl1 libglib2.0-0 ffmpeg libsndfile1 pkg-config" +MACOS_PACKAGES="gnu-sed gcc cmake portaudio git-lfs libjpeg-turbo pre-commit ffmpeg libsndfile pkg-config" INSTALL_MODE="${DIMOS_INSTALL_MODE:-}" EXTRAS="${DIMOS_EXTRAS:-}" @@ -780,6 +780,7 @@ do_install_dev() { info "Developer installs use locked PyTorch builds; Linux x86_64 includes CUDA libraries even for CPU use." dim "will run: uv sync ${sync_args[*]}" if ! prompt_confirm "Install dependencies now?" yes; then INSTALL_DEPS=0; return; fi + project_cmd bash scripts/setup_message_codegen.sh project_cmd uv sync "${sync_args[@]}" ok "developer environment ready in $dir" } diff --git a/scripts/package_messages.sh b/scripts/package_messages.sh new file mode 100644 index 00000000000..54286636a0a --- /dev/null +++ b/scripts/package_messages.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +set -euo pipefail +cd "$(dirname "$0")/.." +message_version="${1:?Pass the release version}" +message_output="$PWD/build/message-codegen/release" +.venv/bin/python -m dimos.message_codegen.generate \ + --version "$message_version" --output "$message_output" +cmake -S "$message_output/cpp" -B "$message_output/cmake" \ + -DDIMOS_BUILD_PYTHON=OFF -DCMAKE_PREFIX_PATH="$PWD/build/message-codegen/install" \ + -DCMAKE_INSTALL_PREFIX="$message_output/install" +cmake --build "$message_output/cmake" +cmake --install "$message_output/cmake" +cargo package --manifest-path "$message_output/rust/Cargo.toml" --allow-dirty +mkdir -p "$message_output/dist" +tar -czf "$message_output/dist/dimos-messages-cmake-$message_version.tar.gz" \ + -C "$message_output/install" . +cp "$message_output/rust/target/package/dimos-generated-messages-$message_version.crate" \ + "$message_output/dist/" diff --git a/scripts/setup_message_codegen.sh b/scripts/setup_message_codegen.sh index afddb5fbc98..ec758c30f5a 100644 --- a/scripts/setup_message_codegen.sh +++ b/scripts/setup_message_codegen.sh @@ -8,7 +8,12 @@ mkdir -p build/message-codegen curl --fail --location --retry 3 \ https://github.com/eProsima/Fast-CDR/archive/refs/tags/v2.4.0.tar.gz \ -o build/message-codegen/fastcdr.tar.gz -echo '79d8466107dd6b7d1defe961c4aa31735038937cf9dd1175cf6b0da0df2209ab build/message-codegen/fastcdr.tar.gz' | sha256sum -c - +python3 - <<'VERIFY' +from hashlib import sha256 +from pathlib import Path +archive = Path("build/message-codegen/fastcdr.tar.gz") +assert sha256(archive.read_bytes()).hexdigest() == "79d8466107dd6b7d1defe961c4aa31735038937cf9dd1175cf6b0da0df2209ab" +VERIFY tar -xzf build/message-codegen/fastcdr.tar.gz -C build/message-codegen cmake -S build/message-codegen/Fast-CDR-2.4.0 -B build/message-codegen/fastcdr-build \ -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX="$PWD/build/message-codegen/install" \ diff --git a/scripts/test_message_packages.sh b/scripts/test_message_packages.sh new file mode 100644 index 00000000000..89295775395 --- /dev/null +++ b/scripts/test_message_packages.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash +set -euo pipefail +cd "$(dirname "$0")/.." +package_demo="$PWD/build/message-codegen/external-app" +mkdir -p "$package_demo/local" "$package_demo/evidence" +cp -R examples/message-codegen/demo_msgs "$package_demo/local/" +.venv/bin/python - "$package_demo/local/demo_msgs/msg/Telemetry.msg" <<'PY' +from pathlib import Path +import sys +path = Path(sys.argv[1]) +path.write_text(path.read_text() + '\nstring application_note "added-locally"\n') +PY +.venv/bin/python -m dimos.message_codegen.generate \ + --package-root "$package_demo/local" --type demo_msgs/msg/Telemetry \ + --python-module external_telemetry --package --output "$package_demo/generated" +export CMAKE_PREFIX_PATH="$PWD/build/message-codegen/install" +uv build --no-build-isolation --python .venv/bin/python \ + "$package_demo/generated/python" --out-dir "$package_demo/dist" +uv venv --clear --python .venv/bin/python "$package_demo/venv" +uv pip install --offline --no-deps --python "$package_demo/venv/bin/python" "$package_demo"/dist/*.whl +cmake -S "$package_demo/generated/cpp" -B "$package_demo/cpp-package" \ + -DDIMOS_BUILD_PYTHON=OFF -DCMAKE_INSTALL_PREFIX="$package_demo/install" +cmake --build "$package_demo/cpp-package" +cmake --install "$package_demo/cpp-package" +.venv/bin/python -m dimos.message_codegen.generate --type geometry_msgs/msg/Point \ + --output "$package_demo/standard" +cmake -S "$package_demo/standard/cpp" -B "$package_demo/standard-build" \ + -DDIMOS_BUILD_PYTHON=OFF -DCMAKE_INSTALL_PREFIX="$package_demo/install" +cmake --install "$package_demo/standard-build" +cmake -S examples/message-codegen/external-app -B "$package_demo/cpp-consumer" \ + -DCMAKE_PREFIX_PATH="$package_demo/install;$CMAKE_PREFIX_PATH" +cmake --build "$package_demo/cpp-consumer" --parallel 2 +cargo package --offline --manifest-path "$package_demo/generated/rust/Cargo.toml" --allow-dirty +mkdir -p "$package_demo/rust-crate" "$package_demo/rust-consumer/src" +tar -xzf "$package_demo/generated/rust/target/package/external-telemetry-messages-0.1.0.crate" \ + -C "$package_demo/rust-crate" +cp examples/message-codegen/external-app/relay.rs "$package_demo/rust-consumer/src/main.rs" +cat > "$package_demo/rust-consumer/Cargo.toml" < bool: """ @@ -151,7 +156,14 @@ def _copy_relay_dist(self): extra_compile_args.append("-march=native") # C++ extensions +message_output = Path(__file__).parent / "build" / "messages" +generate([], message_output) ext_modules = [ + MessageExtension( + "dimos_generated", + message_output / "cpp", + Path(__file__).parent / "build/message-codegen/install", + ), Pybind11Extension( "dimos.navigation.replanning_a_star.min_cost_astar_ext", [os.path.join("dimos", "navigation", "replanning_a_star", "min_cost_astar_cpp.cpp")], @@ -166,5 +178,5 @@ def _copy_relay_dist(self): packages=find_packages(), package_dir={"": "."}, ext_modules=ext_modules, - cmdclass={"build_ext": build_ext, "build_py": build_py}, + cmdclass={"build_ext": MessageBuildExt, "build_py": build_py}, ) diff --git a/uv.lock b/uv.lock index b28700b2eae..6f29fe09f28 100644 --- a/uv.lock +++ b/uv.lock @@ -2638,7 +2638,7 @@ tests-self-hosted = [ { name = "openai" }, { name = "pre-commit", specifier = "==4.2.0" }, { name = "py-spy" }, - { name = "pybind11", specifier = ">=2.12" }, + { name = "pybind11", specifier = "==3.0.1" }, { name = "pygame", specifier = ">=2.6.1" }, { name = "pytest", specifier = "==8.3.5" }, { name = "pytest-asyncio", specifier = "==1.4.0" }, @@ -7695,11 +7695,11 @@ wheels = [ [[package]] name = "pybind11" -version = "3.0.4" +version = "3.0.1" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/cc/f0/35145a3c3baffeef55d4b8324caa33abaa8fa56ab345ecd4b2211d09163e/pybind11-3.0.4.tar.gz", hash = "sha256:3286b59c8a774b9ee650169302dd5a4eedc30a8617905a0560dd8ee44775130c", size = 589533, upload-time = "2026-04-19T03:08:15.925Z" } +sdist = { url = "https://files.pythonhosted.org/packages/2f/7b/a6d8dcb83c457e24a9df1e4d8fd5fb8034d4bbc62f3c324681e8a9ba57c2/pybind11-3.0.1.tar.gz", hash = "sha256:9c0f40056a016da59bab516efb523089139fcc6f2ba7e4930854c61efb932051", size = 546914, upload-time = "2025-08-22T20:09:27.265Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/b3/06/c3a23c9a0263b136c519f033a58d4641e73065fefc7754e9667ec206d992/pybind11-3.0.4-py3-none-any.whl", hash = "sha256:961720ee652da51d531b7b2451a6bd2bc042b0106e6d9baa48ecb7d58034ce63", size = 314166, upload-time = "2026-04-19T03:08:14.091Z" }, + { url = "https://files.pythonhosted.org/packages/cd/8a/37362fc2b949d5f733a8b0f2ff51ba423914cabefe69f1d1b6aab710f5fe/pybind11-3.0.1-py3-none-any.whl", hash = "sha256:aa8f0aa6e0a94d3b64adfc38f560f33f15e589be2175e103c0a33c6bce55ee89", size = 293611, upload-time = "2025-08-22T20:09:25.235Z" }, ] [[package]] From 5be9f0fb02f3e96a23cbd0f2dbc21fa1eeda84c3 Mon Sep 17 00:00:00 2001 From: cc Date: Mon, 21 Sep 2026 19:31:21 -0700 Subject: [PATCH 2/2] fix: build external messages with their bundled generator --- dimos/message_codegen/distribution.py | 3 +++ dimos/message_codegen/test_packages.py | 25 +++++++++++++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/dimos/message_codegen/distribution.py b/dimos/message_codegen/distribution.py index f2be856bf92..e8b506a20bd 100644 --- a/dimos/message_codegen/distribution.py +++ b/dimos/message_codegen/distribution.py @@ -48,6 +48,9 @@ def write_distribution( dirs_exist_ok=True, ignore=shutil.ignore_patterns("__pycache__", "test_*.py"), ) + # A namespace directory loses to an installed regular `dimos` package, + # even with _codegen first on sys.path. Build with the bundled generator. + (toolkit.parent / "__init__.py").write_text("") (project / "pyproject.toml").write_text( '[build-system]\nrequires = ["setuptools>=70", "wheel", "pybind11==3.0.1"]\n' 'build-backend = "setuptools.build_meta"\n' diff --git a/dimos/message_codegen/test_packages.py b/dimos/message_codegen/test_packages.py index 950d5f37489..87a45d868e2 100644 --- a/dimos/message_codegen/test_packages.py +++ b/dimos/message_codegen/test_packages.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +import subprocess +import sys from types import ModuleType import pytest @@ -62,6 +64,29 @@ def test_distribution_carries_source_closure_and_pinned_generator(tmp_path): assert "pybind11==3.0.1" in (project / "pyproject.toml").read_text() +def test_bundled_generator_wins_over_installed_dimos(tmp_path): + names = generate([], tmp_path / "messages", ["geometry_msgs/msg/Point"]) + write_distribution(tmp_path / "messages", "example_messages", names) + installed = tmp_path / "installed" / "dimos" + installed.mkdir(parents=True) + (installed / "__init__.py").write_text("raise RuntimeError('wrong installed generator')\n") + bundled = tmp_path / "messages/python/_codegen" + subprocess.run( + [ + sys.executable, + "-I", + "-c", + "import sys; from pathlib import Path; sys.path[:0] = sys.argv[1:]; " + "from dimos.message_codegen import native_build; " + "assert Path(native_build.__file__).is_relative_to(sys.argv[1])", + str(bundled), + str(installed.parent), + ], + check=True, + cwd=tmp_path, + ) + + def test_installed_schema_discovery_does_not_import_native_types(tmp_path, monkeypatch): source = tmp_path / "custom_msgs/msg/Value.msg" source.parent.mkdir(parents=True)