From 086c16670b9e684e814072cde829a7edcab4b100 Mon Sep 17 00:00:00 2001 From: cc Date: Tue, 20 Oct 2026 21:31:16 -0700 Subject: [PATCH] feat(messages): install and register external CDR definitions --- dimos/message_codegen/test_project_install.py | 170 ++++++++++++++++++ docs/development/messages-external-project.md | 50 ++++++ docs/web/development.md | 2 +- examples/external-messages/.gitignore | 2 + examples/external-messages/README.md | 8 + examples/external-messages/demo_reading.py | 40 +++++ .../telemetry_msgs/msg/DeviceReading.msg | 4 + examples/external-messages/pyproject.toml | 13 ++ examples/external-messages/setup.py | 20 +++ 9 files changed, 308 insertions(+), 1 deletion(-) create mode 100644 dimos/message_codegen/test_project_install.py create mode 100644 docs/development/messages-external-project.md create mode 100644 examples/external-messages/.gitignore create mode 100644 examples/external-messages/README.md create mode 100644 examples/external-messages/demo_reading.py create mode 100644 examples/external-messages/interfaces/telemetry_msgs/msg/DeviceReading.msg create mode 100644 examples/external-messages/pyproject.toml create mode 100644 examples/external-messages/setup.py diff --git a/dimos/message_codegen/test_project_install.py b/dimos/message_codegen/test_project_install.py new file mode 100644 index 00000000000..cf59f66ef55 --- /dev/null +++ b/dimos/message_codegen/test_project_install.py @@ -0,0 +1,170 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Real isolated pip source/wheel/sdist/editable acceptance for message authors.""" + +import os +from pathlib import Path +import shutil +import subprocess +from tempfile import TemporaryDirectory +import venv + +import pytest + +ROOT = Path(__file__).resolve().parents[2] + + +def run(python, *arguments, cwd, env): + result = subprocess.run( + [str(python), *arguments], check=False, cwd=cwd, env=env, capture_output=True, text=True + ) + assert result.returncode == 0, result.stdout + result.stderr + return result.stdout + + +@pytest.fixture(scope="module") +def installed_toolchain(tmp_path_factory): + location = os.environ.get("DIMOS_EXTERNAL_WHEELHOUSE") + if not location: + pytest.skip("Set DIMOS_EXTERNAL_WHEELHOUSE to the complete review wheelhouse") + wheelhouse = Path(location).resolve() + parent = Path(os.environ.get("DIMOS_EXTERNAL_TEST_ROOT", tmp_path_factory.getbasetemp())) + parent.mkdir(parents=True, exist_ok=True) + with TemporaryDirectory(prefix="message-install-", dir=parent) as directory: + folder = Path(directory) + environment = folder / "venv" + venv.EnvBuilder(with_pip=True).create(environment) + python = environment / "bin/python" + temporary = folder / "pip-tmp" + temporary.mkdir() + env = { + **os.environ, + "PIP_NO_INDEX": "1", + "PIP_FIND_LINKS": str(wheelhouse), + "TMPDIR": str(temporary), + } + env.pop("PYTHONPATH", None) + run(python, "-m", "pip", "install", "dimos==0.0.14", cwd=folder, env=env) + forbidden = folder / "no-native-tools" + forbidden.mkdir() + for name in ("cmake", "cargo", "rustc", "c++", "g++", "cc"): + script = forbidden / name + script.write_text("#!/bin/sh\nexit 97\n") + script.chmod(0o755) + env.update( + PATH=str(forbidden) + os.pathsep + str(environment / "bin") + os.pathsep + env["PATH"], + CC="/bin/false", + CXX="/bin/false", + ) + yield python, env, folder + + +@pytest.mark.parametrize("mode", ["source", "wheel", "sdist", "editable"]) +def test_clean_isolated_message_install_and_builtin_identity(installed_toolchain, tmp_path, mode): + python, env, _ = installed_toolchain + project = tmp_path / "project" + shutil.copytree(ROOT / "examples/external-messages", project) + source = project + if mode == "wheel": + run( + python, + "-m", + "pip", + "wheel", + "--no-deps", + "--wheel-dir", + str(tmp_path / "dist"), + ".", + cwd=project, + env=env, + ) + source = next((tmp_path / "dist").glob("acme_telemetry-*.whl")) + elif mode == "sdist": + run( + python, + "-c", + "from setuptools.build_meta import build_sdist; build_sdist('dist')", + cwd=project, + env=env, + ) + source = next((project / "dist").glob("*.tar.gz")) + run( + python, + "-m", + "pip", + "install", + "--force-reinstall", + "--no-deps", + *(["-e"] if mode == "editable" else []), + str(source), + cwd=tmp_path, + env=env, + ) + run( + python, + "-I", + "-c", + """from importlib.metadata import entry_points +from acme_telemetry.telemetry_msgs.msg import DeviceReading +from acme_telemetry_schemas.provider import package_root +from dimos.messages.std_msgs.msg import Header +from dimos.messages.builtin_interfaces.msg import Time +from dimos_cdrgen.registry import encode, decode +value = DeviceReading(Header(Time(17, 23), "map"), 7, 2.5, "sensor") +assert type(value.header) is Header +assert decode(encode(value), DeviceReading) == value +assert type(decode(encode(value), DeviceReading).header) is Header +assert len([ep for ep in entry_points(group='dimos.messages') if ep.name == 'acme_telemetry']) == 1 +assert (package_root() / 'schemas/std_msgs/msg/Header.msg').is_file() +assert (package_root() / 'cpp/CMakeLists.txt').is_file() +assert (package_root() / 'rust/Cargo.toml').is_file() +""", + cwd=tmp_path, + env=env, + ) + output = run(python, "-I", str(project / "demo_reading.py"), cwd=tmp_path, env=env) + assert "PASS: external message registered" in output + run(python, "-m", "pip", "check", cwd=tmp_path, env=env) + if mode == "editable": + definition = project / "interfaces/telemetry_msgs/msg/DeviceReading.msg" + definition.write_text(definition.read_text() + "uint32 status\n") + run(python, "-m", "pip", "install", "--no-deps", "-e", str(project), cwd=tmp_path, env=env) + run( + python, + "-I", + "-c", + """from acme_telemetry.telemetry_msgs.msg import DeviceReading +from dimos.messages.std_msgs.msg import Header +from dimos.messages.builtin_interfaces.msg import Time +value = DeviceReading(Header(Time(0, 0), 'map'), 7, 2.5, 'sensor', 9) +assert value.status == 9 +""", + cwd=tmp_path, + env=env, + ) + definition.rename(definition.with_name("RenamedReading.msg")) + run(python, "-m", "pip", "install", "--no-deps", "-e", str(project), cwd=tmp_path, env=env) + run( + python, + "-I", + "-c", + """from acme_telemetry.telemetry_msgs import msg +assert hasattr(msg, 'RenamedReading') +assert not hasattr(msg, 'DeviceReading') +""", + cwd=tmp_path, + env=env, + ) + assert not (project / "build/message-source/rust/target").exists() diff --git a/docs/development/messages-external-project.md b/docs/development/messages-external-project.md new file mode 100644 index 00000000000..7452454738a --- /dev/null +++ b/docs/development/messages-external-project.md @@ -0,0 +1,50 @@ +# External message projects + +An external project owns its `.msg` definitions and registers its generated +message provider through the `dimos.messages` entry-point group. + +The [standalone example](/examples/external-messages/README.md) declares +dimOS as both a build and runtime dependency and `dimos.messages` as its message +dependency. Its small `setup.py` passes cdrgen's `setup_arguments` to setuptools. +Configuration, generation, provider registration and packaging belong to cdrgen; +dimOS does not implement another backend or generator. + +## Install and run + +Run from the repository root. First complete the +[built-in Python installation](/docs/development/message-source-packages.md#install-and-run-python) +to create `.venv-msg-review` with this checkout's dimOS package. No CMake or Cargo +is needed for this external registration example. + +```sh skip +uv pip install --python .venv-msg-review/bin/python 'setuptools>=70' wheel +uv pip install --python .venv-msg-review/bin/python --no-build-isolation ./examples/external-messages +.venv-msg-review/bin/python -I examples/external-messages/demo_reading.py +``` + +`--no-build-isolation` uses the matching dimOS already installed in this environment. +Ordinary isolated source, wheel, sdist and editable installs are also supported +when pip has access to that matching review wheel. An older release with the same +version cannot supply these built-ins. + +Expected output: + +```text +Installed external message sources: /lib/python3.12/site-packages/acme_telemetry_schemas/package +PASS: external message registered; CDR roundtrip reuses built-in Header +``` + +The example verifies a single registered provider, a custom-message CDR roundtrip, +and exact identity of the Header class supplied by dimOS. + +## Edit a definition + +Edit `examples/external-messages/interfaces/telemetry_msgs/msg/DeviceReading.msg`, +then rerun the installation command and restart Python processes. Generated +artifacts are outputs; edit the `.msg` source instead. Editable installations +also need reinstalling after definitions change. Tests cover field changes, +renames and deletion of obsolete generated types. + +The native built-in example is documented in +[Run C++ and Rust](/docs/development/message-source-packages.md#run-c-and-rust). +Transport and SDK integration are separate features. diff --git a/docs/web/development.md b/docs/web/development.md index f09ef74aea7..e33c5c79e29 100644 --- a/docs/web/development.md +++ b/docs/web/development.md @@ -56,7 +56,7 @@ After changing cockpit or SDK dependencies, run `deno install` in `web/` and com Without vite, `--local-relay` serves the built `cockpit/dist` at `/` and `sdk/dist/sdk.js` at `/sdk.js`. In a checkout, `ensure_web_dist` in [`relay_process.py`](/dimos/web/relay_bridge/relay_process.py#L76) builds both first when either is missing or older than the sources. It stamps the shared, SDK and cockpit sources, the workspace config and the lockfile, builds under a cross-process lock into temporary directories, and swaps both products in together. -Release wheels ship both bundles prebuilt inside `dimos/web/relay_bridge/_relay_dist` (built by the release workflow, see [`setup.py`](/setup.py#L116)), so a pip-installed dimos never builds or downloads npm packages. +Release wheels ship both bundles prebuilt inside `dimos/web/relay_bridge/_relay_dist` (built by the release workflow, see [root packaging configuration](/setup.py#L116)), so a pip-installed dimos never builds or downloads npm packages. Two overrides: `--no-web-build` skips the staleness check and the build, and `DIMOS_WEB_DIR` points the bridge at another web tree. diff --git a/examples/external-messages/.gitignore b/examples/external-messages/.gitignore new file mode 100644 index 00000000000..e2cae228c89 --- /dev/null +++ b/examples/external-messages/.gitignore @@ -0,0 +1,2 @@ +/build/ +/dist/ diff --git a/examples/external-messages/README.md b/examples/external-messages/README.md new file mode 100644 index 00000000000..b8286f0217a --- /dev/null +++ b/examples/external-messages/README.md @@ -0,0 +1,8 @@ +# External message registration example + +`DeviceReading.msg` defines an external message that reuses the installed DimOS +Header. `demo_reading.py` checks registration, a CDR roundtrip and type identity. + +See the [installation and run instructions](../../docs/development/messages-external-project.md#install-and-run) +for complete commands using the built-in example environment, expected output +and how to reinstall after changing a definition. diff --git a/examples/external-messages/demo_reading.py b/examples/external-messages/demo_reading.py new file mode 100644 index 00000000000..61b94c53829 --- /dev/null +++ b/examples/external-messages/demo_reading.py @@ -0,0 +1,40 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Register an external message and reuse the installed built-in Header.""" + +from importlib.metadata import entry_points + +from acme_telemetry.telemetry_msgs.msg import DeviceReading +from acme_telemetry_schemas.provider import package_root +from dimos_cdrgen.registry import decode, encode + +from dimos.messages.builtin_interfaces.msg import Time +from dimos.messages.std_msgs.msg import Header + + +def main() -> None: + assert ( + len([ep for ep in entry_points(group="dimos.messages") if ep.name == "acme_telemetry"]) == 1 + ) + value = DeviceReading(Header(Time(17, 23), "map"), 7, 2.5, "sensor") + restored = decode(encode(value), DeviceReading) + assert restored == value + assert type(restored.header) is Header + print(f"Installed external message sources: {package_root()}") + print("PASS: external message registered; CDR roundtrip reuses built-in Header") + + +if __name__ == "__main__": + main() diff --git a/examples/external-messages/interfaces/telemetry_msgs/msg/DeviceReading.msg b/examples/external-messages/interfaces/telemetry_msgs/msg/DeviceReading.msg new file mode 100644 index 00000000000..1a307878127 --- /dev/null +++ b/examples/external-messages/interfaces/telemetry_msgs/msg/DeviceReading.msg @@ -0,0 +1,4 @@ +std_msgs/Header header +uint32 sequence +float64 value +string label diff --git a/examples/external-messages/pyproject.toml b/examples/external-messages/pyproject.toml new file mode 100644 index 00000000000..c3072a0d706 --- /dev/null +++ b/examples/external-messages/pyproject.toml @@ -0,0 +1,13 @@ +[build-system] +requires = ["setuptools>=70", "wheel", "dimos-cdrgen==0.1.1", "dimos==0.0.14"] +build-backend = "setuptools.build_meta" + +[project] +name = "acme-telemetry" +version = "0.1.0" +description = "An external message reusing the installed DimOS Header" +requires-python = ">=3.12" +dependencies = ["dimos==0.0.14", "dimos-cdrgen==0.1.1"] + +[tool.dimos.messages.dependencies] +"dimos.messages" = "0.0.14" diff --git a/examples/external-messages/setup.py b/examples/external-messages/setup.py new file mode 100644 index 00000000000..6034502d921 --- /dev/null +++ b/examples/external-messages/setup.py @@ -0,0 +1,20 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from pathlib import Path + +from dimos_cdrgen.source_package import setup_arguments +from setuptools import setup + +setup(**setup_arguments(Path(__file__).resolve().parent))