Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
70 changes: 70 additions & 0 deletions .github/scripts/test_relock_shared_flakes.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,70 @@
# Copyright 2026 Dimensional Inc.
# SPDX-License-Identifier: Apache-2.0

"""Exact SDK API pins must survive automatic shared-source refreshes."""

from importlib.machinery import SourceFileLoader
from importlib.util import module_from_spec, spec_from_loader
from pathlib import Path
from tempfile import TemporaryDirectory
import unittest
from unittest.mock import patch

ROOT = Path(__file__).resolve().parents[2]
loader = SourceFileLoader("relock_shared_flakes", str(ROOT / "bin/relock-shared-flakes"))
spec = spec_from_loader(loader.name, loader)
relock = module_from_spec(spec)
loader.exec_module(relock)


class ExactSdkVersionTests(unittest.TestCase):
def stale(self, requirement: str, revision_matches: bool = True) -> list[str]:
revision = "a" * 40
manifest_revision = revision if revision_matches else "b" * 40
with TemporaryDirectory() as directory:
manifest = Path(directory) / "Cargo.toml"
manifest.write_text(
'[dependencies]\ndimos-module = { git = "'
+ relock.GIT_URL
+ '", rev = "'
+ manifest_revision
+ '"'
+ requirement
+ " }\n"
)
with (
patch.object(
relock,
"_pinned_crates",
return_value=[
("dimos-module", "0.1.0", revision),
("dimos-module-macros", "0.1.0", revision),
],
),
patch.object(
relock,
"_crate_dirs",
return_value={
"dimos-module": "native/rust/dimos-module",
"dimos-module-macros": "native/rust/dimos-module-macros",
},
),
patch.object(relock, "_git", return_value='[package]\nversion = "0.2.0"'),
patch.object(relock, "tree_at", side_effect=lambda rev, _: rev),
):
return relock._stale_pin(manifest, manifest.with_name("Cargo.lock"))

def test_exact_legacy_api_keeps_its_sdk_and_transitive_macro_revision(self):
self.assertEqual(self.stale(', version = "=0.1.0"'), [])

def test_unconstrained_dependencies_still_refresh_changed_sources(self):
self.assertEqual(len(self.stale("")), 2)

def test_exact_constraint_does_not_hide_a_mismatched_lock_revision(self):
failures = self.stale(', version = "=0.1.0"', revision_matches=False)
self.assertEqual(len(failures), 2)
self.assertTrue(all("Cargo.lock pins" in error for error in failures))


if __name__ == "__main__":
unittest.main()
4 changes: 3 additions & 1 deletion .github/workflows/autofix.yml
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,9 @@ jobs:
run: bash docker/ros/install-nix.sh
- name: Relock the modules onto this revision
if: steps.shared.outputs.changed == 'true'
run: python3 bin/relock-shared-flakes
run: |
python3 .github/scripts/test_relock_shared_flakes.py
python3 bin/relock-shared-flakes

- name: autofix.ci
uses: autofix-ci/action@v1
41 changes: 0 additions & 41 deletions .github/workflows/message-codegen.yml

This file was deleted.

3 changes: 3 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -106,3 +106,6 @@ optuna.db

# policy blobs are loaded from outside the repository
*.bin

# Prepared from schema-sources.json; keep downloaded schemas out of Git.
/packages/dimos-generated/interfaces/
5 changes: 5 additions & 0 deletions MANIFEST.in
Original file line number Diff line number Diff line change
Expand Up @@ -47,3 +47,8 @@ prune dimos/web/command-center-extension

global-exclude test_*.py
global-exclude conftest.py

# Built-in message inputs are prepared automatically during source builds; the sdist is self-contained.
include packages/dimos-generated/schema-sources.json
recursive-include packages/dimos-generated/custom *.msg LICENSE
Comment thread
TomCC7 marked this conversation as resolved.
recursive-include packages/dimos-generated/interfaces *.msg *LICENSE*
17 changes: 15 additions & 2 deletions bin/relock-shared-flakes
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,8 @@ rewrites a pull request's merge ref, so a pin on it would rot.
What counts as stale is the pinned *content*, not the revision. A pin may name an
older commit quite legitimately -- what matters is that the shared tree at that
commit is the tree in this one.
An explicit exact Cargo version keeps its pinned revision, including transitive
SDK crates. Updating that API is a caller migration rather than an automatic relock.

Stdlib only, so it runs on a bare `python3` like the other bin/ scripts.
"""
Expand Down Expand Up @@ -296,10 +298,19 @@ def _hashes_file(manifest: Path) -> Path:
def _stale_pin(manifest: Path, lock: Path) -> list[str]:
"""Why the pin on this repository's crates is not this commit's; empty when it is."""
revs = {m.group("rev") for m in _GIT_DEP.finditer(manifest.read_text())}
dependencies = tomllib.loads(manifest.read_text()).get("dependencies", {})
packages = _pinned_crates(lock)
fixed_revs = set()
for name, version, rev in packages:
dependency = dependencies.get(name, {})
if isinstance(dependency, dict) and dependency.get("version") == f"={version}":
fixed_revs.add(rev)
stale = []
for name, _, rev in _pinned_crates(lock):
for name, _, rev in packages:
if rev not in revs:
stale.append(f"{name}: Cargo.lock pins {rev[:10]}, Cargo.toml another revision")
if rev in fixed_revs:
continue
subdir = _crate_dirs().get(name)
pinned = tree_at(rev, subdir) if subdir else None
if subdir is None or (pinned is not None and pinned != tree_at("HEAD", subdir)):
Expand Down Expand Up @@ -345,7 +356,9 @@ def relock_cargo(manifest: Path) -> tuple[Path, bool, list[str]]:
return manifest, False, [_unpushed(manifest, " and ".join(subdirs))]
manifest.write_text(
_GIT_DEP.sub(
lambda m: f'{m.group("name")} = {{ git = "{GIT_URL}", rev = "{rev}"{m.group("rest")}}}',
lambda m: (
f'{m.group("name")} = {{ git = "{GIT_URL}", rev = "{rev}"{m.group("rest")}}}'
),
before[manifest],
)
)
Expand Down
2 changes: 1 addition & 1 deletion dimos/experimental/memory/rust/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ license = "Apache-2.0"
anyhow = "1"
crossbeam-channel = "0.5"
crossbeam-utils = "0.8"
dimos-module = { git = "https://github.com/dimensionalOS/dimos", rev = "6663bcb6e5e18f43a6cb9cf77ccbec613e310443" }
dimos-module = { git = "https://github.com/dimensionalOS/dimos", rev = "6663bcb6e5e18f43a6cb9cf77ccbec613e310443", version = "=0.1.0" }
lcm-msgs = { git = "https://github.com/dimensionalOS/dimos-lcm.git", branch = "rust-codegen" }
lz4_flex = "0.14.0"
mcap = { version = "0.25.0", default-features = false, features = ["zstd"] }
Expand Down
2 changes: 1 addition & 1 deletion dimos/hardware/sensors/camera/realsense/rust/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ infrared_left_camera_info = "sensor_msgs.CameraInfo"
infrared_right_camera_info = "sensor_msgs.CameraInfo"

[dependencies]
dimos-module = { git = "https://github.com/dimensionalOS/dimos", rev = "6663bcb6e5e18f43a6cb9cf77ccbec613e310443" }
dimos-module = { git = "https://github.com/dimensionalOS/dimos", rev = "6663bcb6e5e18f43a6cb9cf77ccbec613e310443", version = "=0.1.0" }
lcm-msgs = { git = "https://github.com/dimensionalOS/dimos-lcm.git", branch = "rust-codegen" }
realsense-rust = "1.3"
realsense-sys = "2.56"
Expand Down
2 changes: 1 addition & 1 deletion dimos/hardware/sensors/camera/v4l2/rust/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ python = "dimos.hardware.sensors.camera.v4l2.module:V4L2Camera"
jpeg_out = "sensor_msgs.CompressedImage"

[dependencies]
dimos-module = { git = "https://github.com/dimensionalOS/dimos", rev = "6663bcb6e5e18f43a6cb9cf77ccbec613e310443" }
dimos-module = { git = "https://github.com/dimensionalOS/dimos", rev = "6663bcb6e5e18f43a6cb9cf77ccbec613e310443", version = "=0.1.0" }
lcm-msgs = { git = "https://github.com/dimensionalOS/dimos-lcm.git", branch = "rust-codegen" }
tokio = { version = "1", features = ["rt-multi-thread", "macros", "signal"] }
serde = { version = "1", features = ["derive"] }
Expand Down
2 changes: 1 addition & 1 deletion dimos/hardware/sensors/lidar/livox/rust/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ imu = "sensor_msgs.Imu"

[dependencies]
livox-wire = { git = "https://github.com/dimensionalOS/livox-wire", rev = "441c4a16dbe2adb0683ece976cc800c6d7fbc790" }
dimos-module = { git = "https://github.com/dimensionalOS/dimos", rev = "6663bcb6e5e18f43a6cb9cf77ccbec613e310443" }
dimos-module = { git = "https://github.com/dimensionalOS/dimos", rev = "6663bcb6e5e18f43a6cb9cf77ccbec613e310443", version = "=0.1.0" }
lcm-msgs = { git = "https://github.com/dimensionalOS/dimos-lcm.git", branch = "rust-codegen" }
serde = { version = "1", features = ["derive"] }
serde_json = "1"
Expand Down
2 changes: 1 addition & 1 deletion dimos/hardware/sensors/lidar/pointlio/rust/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ odometry = "nav_msgs.Odometry"
livox-wire = { git = "https://github.com/dimensionalOS/livox-wire", rev = "441c4a16dbe2adb0683ece976cc800c6d7fbc790" }
# Direct dependencies of the copied livox wire/pipeline/pcap modules. See src/lib.rs.
pointlio-core = { git = "https://github.com/dimensionalOS/dimos-module-pointlio.git", rev = "22a5403f827d030b248ae473c2808b5d405b3d55" }
dimos-module = { git = "https://github.com/dimensionalOS/dimos", rev = "6663bcb6e5e18f43a6cb9cf77ccbec613e310443" }
dimos-module = { git = "https://github.com/dimensionalOS/dimos", rev = "6663bcb6e5e18f43a6cb9cf77ccbec613e310443", version = "=0.1.0" }
lcm-msgs = { git = "https://github.com/dimensionalOS/dimos-lcm.git", branch = "rust-codegen" }
serde = { version = "1", features = ["derive"] }
serde_json = "1"
Expand Down
2 changes: 1 addition & 1 deletion dimos/hardware/sensors/lidar/virtual_mid360/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ path = "src/main.rs"
[dependencies]
livox-wire = { git = "https://github.com/dimensionalOS/livox-wire", rev = "441c4a16dbe2adb0683ece976cc800c6d7fbc790" }
# Direct dependencies of the copied livox wire/pipeline/pcap modules. See src/main.rs.
dimos-module = { git = "https://github.com/dimensionalOS/dimos", rev = "6663bcb6e5e18f43a6cb9cf77ccbec613e310443" }
dimos-module = { git = "https://github.com/dimensionalOS/dimos", rev = "6663bcb6e5e18f43a6cb9cf77ccbec613e310443", version = "=0.1.0" }
tokio = { version = "1", features = ["rt-multi-thread", "macros", "time", "net", "sync"] }
serde = { version = "1", features = ["derive"] }
socket2 = "0.6"
Expand Down
2 changes: 1 addition & 1 deletion dimos/mapping/dim_slam/rust/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ dim_slam = { git = "https://github.com/dimensionalOS/dimSLAM", tag = "v0.4.1" }
# Unused directly, but cargo hands DEP_CUVSLAM_LIB_DIR only to immediate dependents
# and build.rs needs it for the rpath.
cu_vslam_rs = "0.2.0"
dimos-module = { git = "https://github.com/dimensionalOS/dimos", rev = "6663bcb6e5e18f43a6cb9cf77ccbec613e310443" }
dimos-module = { git = "https://github.com/dimensionalOS/dimos", rev = "6663bcb6e5e18f43a6cb9cf77ccbec613e310443", version = "=0.1.0" }
lcm-msgs = { git = "https://github.com/dimensionalOS/dimos-lcm.git", branch = "rust-codegen" }
tokio = { version = "1", features = ["rt-multi-thread", "macros"] }
tracing = "0.1"
Expand Down
2 changes: 1 addition & 1 deletion dimos/mapping/ray_tracing/rust/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ region_bounds = "geometry_msgs.PoseStamped"
map_regions = "sensor_msgs.PointCloud2"

[dependencies]
dimos-module = { git = "https://github.com/dimensionalOS/dimos", rev = "6663bcb6e5e18f43a6cb9cf77ccbec613e310443" }
dimos-module = { git = "https://github.com/dimensionalOS/dimos", rev = "6663bcb6e5e18f43a6cb9cf77ccbec613e310443", version = "=0.1.0" }
lcm-msgs = { git = "https://github.com/dimensionalOS/dimos-lcm.git", branch = "rust-codegen" }
tokio = { version = "1", features = ["rt-multi-thread", "macros", "signal"] }
serde = { version = "1", features = ["derive"] }
Expand Down
122 changes: 122 additions & 0 deletions dimos/message_codegen/build.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,122 @@
# Copyright 2026 Dimensional Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""DimOS source-build inputs; imported only by packaging and preparation tools."""

from hashlib import sha256
import os
from pathlib import Path, PurePosixPath
import shutil
from tempfile import TemporaryDirectory
from typing import Annotated, Literal

from dimos_cdrgen.source_inputs import prepare_sources
from pydantic import AfterValidator, BaseModel, ConfigDict, Field

ROOT = Path(__file__).resolve().parents[2]
PROJECT = ROOT / "packages/dimos-generated"


def _relative_path(value: str) -> str:
path = PurePosixPath(value)
if not path.parts or path.is_absolute() or ".." in path.parts:
raise ValueError("Unsafe source input path")
return value


RelativePath = Annotated[str, AfterValidator(_relative_path)]
Sha256 = Annotated[str, Field(pattern=r"^[0-9a-f]{64}$")]


class SourceFile(BaseModel):
model_config = ConfigDict(strict=True, extra="forbid")

path: RelativePath
sha256: Sha256


class SourceArchive(BaseModel):
model_config = ConfigDict(strict=True, extra="forbid")

repository: str | None = None
revision: str | None = None
url: str
sha256: Sha256
prefix: RelativePath
files: dict[RelativePath, SourceFile]


class SourceLock(BaseModel):
model_config = ConfigDict(strict=True, extra="forbid")

format: Literal[1]
sources: list[SourceArchive]


def _matches(directory: Path, files: dict[str, str]) -> bool:
return all(
(directory / name).is_file()
and sha256((directory / name).read_bytes()).hexdigest() == digest
for name, digest in files.items()
)


def prepare_builtin_sources(
cache: Path | None = None, *, offline: bool = False, project: Path = PROJECT
) -> Path:
cache = cache or Path(
os.environ.get("DIMOS_MESSAGE_SOURCE_CACHE", ROOT / "build/builtin-source-cache")
)
lock = project / "schema-sources.json"
data = SourceLock.model_validate_json(lock.read_text())
upstream = {
name: entry.sha256 for source in data.sources for name, entry in source.files.items()
}
custom_root = project / "custom/dimos_msgs"
custom = {
"dimos_msgs/msg/" + path.name: sha256(path.read_bytes()).hexdigest()
for path in (custom_root / "msg").glob("*.msg")
}
custom["licenses/dimos_msgs-LICENSE"] = sha256(
(custom_root / "LICENSE").read_bytes()
).hexdigest()
expected = {**upstream, **custom}
target = project / "interfaces"
actual = {str(path.relative_to(target)) for path in target.rglob("*") if path.is_file()}
if actual == expected.keys() and _matches(target, expected):
return target
# Stage both downloaded and local definitions before replacing previous inputs.
with TemporaryDirectory(prefix=".interfaces-", dir=project) as directory:
stage = Path(directory) / "inputs"
if _matches(target, upstream):
stage.mkdir()
for name in upstream:
output = stage / name
output.parent.mkdir(parents=True, exist_ok=True)
shutil.copyfile(target / name, output)
else:
prepare_sources(lock, stage, cache / "archives", offline=offline)
shutil.copytree(custom_root / "msg", stage / "dimos_msgs/msg")
(stage / "licenses").mkdir(exist_ok=True)
shutil.copyfile(custom_root / "LICENSE", stage / "licenses/dimos_msgs-LICENSE")
backup = Path(directory) / "previous"
if target.exists():
target.rename(backup)
try:
stage.rename(target)
except OSError:
if backup.exists():
backup.rename(target)
raise
return target
Loading
Loading