Refuse to drop types from generated namespace files unless --force (#403); sort namespace contents

src/nunavut/__init__.py

@@ -28,6 +28,7 @@
     generate_all_from_namespace_with_generators,
 )
 from ._namespace import Namespace
+from ._namespace_files import NamespaceConflictError
 from ._utilities import TEMPLATE_SUFFIX, DefaultValue, ResourceSearchPolicy, ResourceType, YesNoDefault
 from ._version import __author__, __copyright__, __email__, __license__, __version__
 from .jinja import CodeGenerator, DSDLCodeGenerator, SupportGenerator
@@ -58,6 +59,7 @@
     "LanguageContext",
     "LanguageContextBuilder",
     "Namespace",
+    "NamespaceConflictError",
     "ResourceType",
     "ResourceSearchPolicy",
     "SupportGenerator",

src/nunavut/_generators.py

@@ -16,6 +16,12 @@
 from typing import Any, Dict, Iterable, List, Mapping, Optional, Type, Union
 
 from ._namespace import Generatable, Namespace
+from ._namespace_files import (
+    NamespaceConflictError,
+    describe_namespace_conflicts,
+    find_namespace_conflicts,
+    remove_generated_namespaces,
+)
 from ._utilities import ResourceType, YesNoDefault
 from .lang import LanguageContext, LanguageContextBuilder
 from .lang._language import Language
@@ -308,6 +314,7 @@ def generate_all(
     omit_dependencies: bool = False,
     code_generator_type: Optional[Type[AbstractGenerator]] = None,
     support_generator_type: Optional[Type[AbstractGenerator]] = None,
+    force: bool = False,
     **kwargs: Any,
 ) -> GenerationResult:
     """
@@ -415,6 +422,10 @@ def generate_all(
     :param code_generator_type: The type of code generator to use. If None then the default code generator is used.
     :param support_generator_type: The type of support generator to use. If None then the default support generator is
         used.
+    :param bool force:
+        If True then files generated into `outdir` by an earlier run are removed before generating (see
+        :func:`generate_all_from_namespace_with_generators`). Without it, generation is refused if it would replace a
+        namespace file that lists types this generation does not include.
     :param kwargs: Additional arguments passed into the language context builder and generator constructors. See the
         documentation for :func:`basic_language_context_builder_from_args` and for specific generator types for details
         on supported arguments.
@@ -445,6 +456,7 @@ def generate_all(
         omit_dependencies,
         code_generator_type,
         support_generator_type,
+        force=force,
         **kwargs,
     )
 
@@ -463,6 +475,7 @@ def generate_all_for_language(
     omit_dependencies: bool = False,
     code_generator_type: Optional[Type[AbstractGenerator]] = None,
     support_generator_type: Optional[Type[AbstractGenerator]] = None,
+    force: bool = False,
     **generator_args: Any,
 ) -> GenerationResult:
     """
@@ -500,6 +513,9 @@ def generate_all_for_language(
     :param code_generator_type: The type of code generator to use. If None then the default code generator is used.
     :param support_generator_type: The type of support generator to use. If None then the default support generator is
         used.
+    :param force: If True then files generated into `outdir` by an earlier run are removed before generating (see
+        :func:`generate_all_from_namespace_with_generators`). Without it, generation is refused if it would replace a
+        namespace file that lists types this generation does not include.
     :param generator_args: Additional arguments to pass into the generator constructors. See the documentation for
         specific generator types for details on supported arguments.
     :return: A dataclass containing explicit inputs, discovered inputs, and determined outputs.
@@ -525,6 +541,7 @@ def generate_all_for_language(
         no_overwrite,
         code_generator_type,
         support_generator_type,
+        force=force,
         **generator_args,
     )
 
@@ -537,6 +554,7 @@ def generate_all_from_namespace(
     no_overwrite: bool = False,
     code_generator_type: Optional[Type[AbstractGenerator]] = None,
     support_generator_type: Optional[Type[AbstractGenerator]] = None,
+    force: bool = False,
     **generator_args: Any,
 ) -> GenerationResult:
     """
@@ -555,6 +573,9 @@ def generate_all_from_namespace(
     :param code_generator_type: The type of code generator to use. If None then the default code generator is used.
     :param support_generator_type: The type of support generator to use. If None then the default support generator is
         used.
+    :param force: If True then files generated into `outdir` by an earlier run are removed before generating (see
+        :func:`generate_all_from_namespace_with_generators`). Without it, generation is refused if it would replace a
+        namespace file that lists types this generation does not include.
     :param generator_args: Additional arguments to pass into the generator constructors. See the documentation for
         specific generator types for details on supported arguments.
     :return: A dataclass containing explicit inputs, discovered inputs, and determined outputs.
@@ -587,7 +608,9 @@ def generate_all_from_namespace(
         index, resource_types, embed_auditing_info=embed_auditing_info, **support_generator_args
     )
 
-    return generate_all_from_namespace_with_generators(index, code_generator, support_generator, dry_run, no_overwrite)
+    return generate_all_from_namespace_with_generators(
+        index, code_generator, support_generator, dry_run, no_overwrite, force
+    )
 
 
 def generate_all_from_namespace_with_generators(
@@ -596,6 +619,7 @@ def generate_all_from_namespace_with_generators(
     support_generator: AbstractGenerator,
     dry_run: bool = False,
     no_overwrite: bool = False,
+    force: bool = False,
 ) -> GenerationResult:
     """
     Given a populated namespace, generate code for the types found within it using the provided generators. This is the
@@ -609,11 +633,28 @@ def generate_all_from_namespace_with_generators(
         logging and errors.
     :param no_overwrite: If True then generated files will not be allowed to overwrite existing files under the `outdir`
         path causing errors.
+    :param force: If True then, before generating, remove the files an earlier run generated into the output
+        directory: every namespace file with a member list and the files of the types it lists. Without it, generation
+        is refused (:class:`NamespaceConflictError`) if it would replace a namespace file that lists types this
+        generation does not include, or a namespace file without a member list. Generating the same types again, or
+        more types into a namespace, is always allowed. Only applies when namespace files are generated.
     :return: A dataclass containing explicit inputs, discovered inputs, and determined outputs.
     :raises pydsdl.FrontendError: Exceptions thrown from the pydsdl frontend. For example, parsing malformed DSDL will
         raise this exception.
+    :raises NamespaceConflictError: If ``force`` is False and generating would replace a namespace file unsafely.
     """
 
+    # Namespace files list their namespace's types, so check (or clean up) before writing anything: a refused run
+    # leaves the output directory untouched. See https://github.com/OpenCyphal/nunavut/issues/403.
+    generates_types = (code_generator.resource_types & ResourceType.ONLY.value) != ResourceType.ONLY.value
+    if code_generator.generate_namespace_types and generates_types:
+        if force:
+            remove_generated_namespaces(index, dry_run)
+        else:
+            conflicts = find_namespace_conflicts(index)
+            if conflicts:
+                raise NamespaceConflictError(describe_namespace_conflicts(conflicts))
+
     template_files = list(set(support_generator.get_templates()).union(set(code_generator.get_templates())))
     generated_files = list(code_generator.generate_all(dry_run, not no_overwrite))
 

src/nunavut/_namespace.py

@@ -874,10 +874,11 @@ def get_root_namespace(self, root_namespace_name: str, create_if_missing: bool =
 
     def get_nested_namespaces(self) -> Iterator["Namespace"]:
         """
-        Get an iterator over all the nested namespaces within this namespace.
+        Get an iterator over all the nested namespaces within this namespace, sorted by name.
         This is a shallow iterator that only provides directly nested namespaces.
         """
-        return iter(self._nested_namespaces.values())
+        # Sorted because insertion order depends on the order the DSDL files were read in, which varies between runs.
+        return iter([self._nested_namespaces[name] for name in sorted(self._nested_namespaces)])
 
     def get_nested_namespace(self, namespace_name: str, create_if_missing: bool = False) -> "Namespace":
         """
@@ -905,9 +906,16 @@ def get_nested_types(self) -> ItemsView[pydsdl.CompositeType, Generatable]:
         """
         Get a view of a tuple relating datatypes in this namespace to the path for the
         type's generated output. This is a shallow view including only the types
-        directly within this namespace.
-        """
-        return self._data_type_to_outputs.items()
+        directly within this namespace, sorted by name and then by version.
+        """
+        # Sorted because insertion order depends on the order the DSDL files were read in, which varies between runs.
+        # Generated namespace files list these types, so an unstable order makes the generated output unstable.
+        return dict(
+            sorted(
+                self._data_type_to_outputs.items(),
+                key=lambda item: (item[0].full_name, item[0].version.major, item[0].version.minor),
+            )
+        ).items()
 
     def get_all_datatypes(self) -> Generator[Tuple[pydsdl.CompositeType, Generatable], None, None]:
         """