From 82c9636667302f58147a8f455c9e8db9fe33087a Mon Sep 17 00:00:00 2001 From: Andrey Golovanov Date: Sun, 4 Oct 2026 09:27:53 -0700 Subject: [PATCH] Simplify topology generation and remove legacy code --- CHANGELOG.md | 29 + Makefile | 6 +- README.md | 106 +- build.sh | 36 +- dev/STREAMLINING.md | 363 ++++ dev/analyze_integrated_graph.py | 323 +--- examples/small_baseline.yml | 24 +- examples/small_clos.yml | 24 +- examples/small_dragonfly.yml | 24 +- examples/small_dragonfly_custom.yml | 24 +- pyproject.toml | 4 +- .../test_blueprint_diagram_integration.py | 65 +- tests/integration/test_netgraph_scenarios.py | 15 +- .../test_site_graph_render_smoke.py | 2 + .../test_validation_optics_checks.py | 17 +- .../test_validation_port_budget.py | 11 +- tests/integration/test_visualization_maps.py | 24 +- tests/test_assembly_contract.py | 110 ++ tests/test_audit_streamlining.py | 21 + tests/test_build_integration.py | 23 +- tests/test_cli.py | 216 +-- tests/test_components_config.py | 194 +- tests/test_components_lib.py | 76 +- tests/test_components_scenario_builder.py | 39 +- tests/test_config.py | 22 +- tests/test_config_streamlining.py | 47 + tests/test_corridor_discovery.py | 132 +- tests/test_end_to_end_risk_groups.py | 93 +- .../test_failure_policies_workflows_config.py | 363 +--- ...ure_policies_workflows_scenario_builder.py | 122 +- tests/test_geo_utils.py | 97 - tests/test_geographic_contract.py | 125 ++ tests/test_highway_graph.py | 5 +- tests/test_integrated_graph.py | 33 +- tests/test_metro_clusters.py | 148 +- tests/test_review_regressions.py | 232 +++ tests/test_risk_groups_functional.py | 13 +- tests/test_runtime_contract.py | 128 ++ tests/test_scenario_risk_groups.py | 99 +- tests/test_script_contracts.py | 34 + tests/test_site_graph_sot.py | 5 +- tests/test_site_graph_visualization.py | 2 + tests/test_site_streamlining.py | 162 ++ tests/test_striping_equivalence.py | 7 +- tests/test_striping_semantics.py | 13 +- tests/test_tm_sizing_contract.py | 79 +- tests/test_traffic_streamlining.py | 50 + tests/test_validation.py | 244 +-- tests/topogen/scenario/test_config_unit.py | 13 +- .../scenario/test_graph_pipeline_unit.py | 99 +- tests/topogen/test_blueprint_viz_unit.py | 127 +- .../topogen/validation/test_expand_checks.py | 36 +- tests/topogen/validation/test_helpers.py | 29 +- topogen/__init__.py | 2 + topogen/blueprint_viz.py | 296 +-- topogen/blueprints_lib.py | 23 +- topogen/cli.py | 270 +-- topogen/components_lib.py | 58 +- topogen/config.py | 855 +++------ topogen/context.py | 14 + topogen/corridors.py | 521 ++---- topogen/failure_policies_lib.py | 23 +- topogen/geo_utils.py | 38 +- topogen/highway_graph.py | 48 +- topogen/integrated_graph.py | 544 ++---- topogen/library_io.py | 19 + topogen/metro_clusters.py | 265 +-- topogen/naming.py | 7 +- topogen/roles.py | 26 + topogen/scenario/__init__.py | 6 +- topogen/scenario/artifacts.py | 74 + topogen/scenario/assembly.py | 498 +----- topogen/scenario/config.py | 255 +-- topogen/scenario/expansion.py | 90 + topogen/scenario/graph_pipeline.py | 1580 +++-------------- topogen/scenario/libraries.py | 120 +- topogen/scenario/network.py | 204 ++- topogen/scenario/policies.py | 39 +- topogen/scenario/risk.py | 105 +- topogen/scenario/sizing.py | 321 ++++ topogen/scenario/striping.py | 228 +-- topogen/scenario/traffic.py | 25 +- topogen/scenario_builder.py | 30 - topogen/schemas/topogen_config.json | 188 +- topogen/traffic_matrix.py | 1228 ++++--------- topogen/validation/audits/dc_capacity.py | 90 + topogen/validation/audits/expand_checks.py | 207 +-- topogen/validation/audits/hw_capacity.py | 60 +- topogen/validation/audits/ngraph_schema.py | 31 - topogen/validation/audits/node_hw_presence.py | 86 +- topogen/validation/audits/node_role.py | 41 +- topogen/validation/audits/optics_checks.py | 338 +--- topogen/validation/audits/pipeline.py | 124 +- topogen/validation/audits/port_budget.py | 162 +- topogen/validation/helpers.py | 32 +- topogen/validation/scenario_dict.py | 194 +- topogen/validation/yaml_validation.py | 54 +- topogen/visualization.py | 1153 ++---------- topogen/workflows_lib.py | 24 +- typings/contextily/__init__.pyi | 17 + 100 files changed, 5325 insertions(+), 9623 deletions(-) create mode 100644 dev/STREAMLINING.md create mode 100644 tests/test_assembly_contract.py create mode 100644 tests/test_audit_streamlining.py create mode 100644 tests/test_config_streamlining.py delete mode 100644 tests/test_geo_utils.py create mode 100644 tests/test_geographic_contract.py create mode 100644 tests/test_review_regressions.py create mode 100644 tests/test_runtime_contract.py create mode 100644 tests/test_script_contracts.py create mode 100644 tests/test_site_streamlining.py create mode 100644 tests/test_traffic_streamlining.py create mode 100644 topogen/context.py create mode 100644 topogen/library_io.py create mode 100644 topogen/roles.py create mode 100644 topogen/scenario/artifacts.py create mode 100644 topogen/scenario/expansion.py create mode 100644 topogen/scenario/sizing.py delete mode 100644 topogen/scenario_builder.py create mode 100644 topogen/validation/audits/dc_capacity.py delete mode 100644 topogen/validation/audits/ngraph_schema.py create mode 100644 typings/contextily/__init__.pyi diff --git a/CHANGELOG.md b/CHANGELOG.md index 6d36daf..ca6f18a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,19 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- **Diagnostics**: Report metro anchors removed by highway filtering as a configuration/data error; correct ring-mesh, representative-point and geometry-cleanup descriptions. Stop inserting heuristic YAML section comments that could label the wrong links. +- **Traffic**: Keep top-K gravity partners independent of metro-name ordering; seed jitter locally and distribute uniform traffic between DC sites rather than individual devices. +- **Striping**: Respect one-to-one site mode and endpoint matches; identify devices by full literal paths and preserve disabled risk-group settings. +- **Config**: Reject fractional/duplicate priority keys, duplicate metro overrides, unknown striping fields and unmatched forced metro patterns. Apply configured link cost as a minimum without changing geographic distances. +- **Blueprints**: Include recursive dependencies and reject cycles; derive diagrams from resolved devices instead of a duplicate DSL interpreter. +- **Geography**: Filter urban areas using their actual CRS and return YAML-safe coordinates for direct Python generation/build calls. +- **Validation**: Reject non-string device roles and non-finite chassis capacities. +- **Tools**: Return failure from batch builds when any configuration fails; analyze vertex connectivity and k-cores on simple metro adjacency while preserving corridor counts. + +- **Traffic**: Reject infeasible or unconverged hose fits; preserve small demands when rounding is disabled. +- **Validation**: Audit each demand set against expanded external DC capacity; resolve regex selectors, report unmatched demands, and handle malformed network structures without crashing. +- **Contracts**: Preserve exact integer strings; reject non-finite config values, non-metre/geographic CRS and duplicate metro identities; check PoP coordinates without DC groups. +- **Components**: Collect blueprint component dependencies after hardware overrides, removing replaced platform references. - **TM Sizing**: Split uniform traffic over ordered DC pairs, including same-metro pairs, to match NetGraph demand expansion. - **TM Sizing**: Size corridors for peak directional load even when `respect_min_base_capacity` is disabled. - **TM Sizing**: Preserve separate corridors with repeated edge keys across different PoP pairs. @@ -18,9 +31,19 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - **Workflows**: Bind built-in demand references to `traffic.matrix_name`. - **Failures**: Make the `empty` policy a valid no-failure mode. - **Docs**: Install from GitHub; the `topogen` name on PyPI belongs to an unrelated project. +- **Geography**: Preserve alternate highway routes, isolated rings and all `k_paths` through corridor JSON and site expansion; form nearest-neighbor pairs independently of metro ID order. Resolve risk distance from its owning path and reject unknown risk references. +- **Hardware**: Resolve optic counts for each site edge and endpoint; preserve caller link attributes and geographic corridor metadata. +- **Validation**: Preserve node rules in expansion audits, require scenario-local component definitions, and audit declared optic port usage. +- **CLI**: Validate before atomically replacing scenario YAML, including `--print`; scope quiet output to the command. ### Changed +- **Internal**: Remove duplicate corridor metadata and intermediate edge copying; decouple site identifiers from device expansion and consolidate sizing capacity passes. +- **Checks**: Detect unused Ruff suppressions; make CLI tests verify successful quiet dispatch and required exception propagation. +- **Typing**: Describe the Contextily API used by TopoGen in a scoped local stub; keep missing-stub diagnostics enabled. +- **Internal**: Share traffic pair selection/rounding/emission, skip disabled debug formatting, route aggregated metro-pair demand once, reuse finalized blueprint libraries and build workflows once. +- **Internal**: Separate site construction, sizing and serialization; centralize schema bounds and metro naming. Remove the unused metro-loader formatting argument; configuration dataclasses reject undeclared attributes. + - **BREAKING**: **Scenarios**: Emit NetGraph `demands` and `failures` sections; update consumers of generated YAML. - **BREAKING**: **Failures**: Custom rules must use `scope`, `mode` and `match`; migrate `lib/failure_policies.yml`. - **BREAKING**: **Blueprints**: Link selectors in `lib/blueprints.yml` use NetGraph `$var`/`${var}` placeholders; `{var}` is no longer expanded. @@ -32,6 +55,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - **Workflows**: Use `parallelism: auto` in built-in and sample workflows. - **Validation**: Construct the full NetGraph scenario to check workflow arguments and failure policies before topology audits. - **Internal**: Remove redundant validation and scenario code, the PyPI publish workflow, the pre-commit Pyright hook and benchmark flags. +- **BREAKING**: **Contracts**: Require corridor `MultiGraph` JSON with explicit edge keys; regenerate saved graphs. Use `topogen.scenario.build_scenario`, `RunContext` and public visualization settings. +- **BREAKING**: **Config**: Remove ignored metadata, road-processing controls, `edge_select`, component assignments/library fields and scenario overrides; reject invalid integer quantities. +- **BREAKING**: **Optics**: Require explicit `local->remote` endpoint mappings; remove implicit reverse and legacy delimiter behavior. Link role selectors use `role|role` strings. +- **Assembly**: Generate traffic and expand the device network once per build; reuse the result for budgets, hardware and artifacts. +- **Internal**: Consolidate link parsing and user-library loading; remove format guessing, swallowed computation/render errors and duplicated graph copies. ### Added @@ -39,6 +67,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Removed +- **BREAKING**: Remove unused point/bearing utilities, `MetroCluster` distance/overlap/array helpers, `TopologyConfig.summary()` and singular/list/filter component helpers. Use the merged `get_builtin_components()` mapping and `topogen info`. - **Dependencies**: Unused `seaborn`, `rich`, `nbformat`, `nbconvert`, `ipykernel`, `itables`, `rasterio` and `scikit-learn` requirements. ## [0.4.0] - 2026-03-15 diff --git a/Makefile b/Makefile index 6d25680..03f6464 100644 --- a/Makefile +++ b/Makefile @@ -126,11 +126,7 @@ qt: validate: @echo "📋 Validating TopoGen config YAMLs..." - @if $(PYTHON) -c "import jsonschema" >/dev/null 2>&1; then \ - $(PYTHON) -c "import json, yaml, jsonschema, pathlib; from importlib import resources as res; f=res.files('topogen.schemas').joinpath('topogen_config.json').open('r', encoding='utf-8'); schema=json.load(f); f.close(); cfg_dirs=['examples']; cfg_files=[]; [cfg_files.extend(list(pathlib.Path(d).rglob('*.yaml'))+list(pathlib.Path(d).rglob('*.yml'))) for d in cfg_dirs if pathlib.Path(d).exists()]; [jsonschema.validate(yaml.safe_load(open(fp)), schema) for fp in cfg_files]; print(f'✅ Validated {len(cfg_files)} TopoGen config YAML files')"; \ - else \ - echo "⚠️ jsonschema not installed. Skipping schema validation"; \ - fi + @$(PYTHON) -c "from pathlib import Path; from topogen.config import TopologyConfig; files=sorted(p for p in Path('examples').rglob('*') if p.suffix in {'.yml', '.yaml'}); [TopologyConfig.from_yaml(p) for p in files]; print(f'Validated {len(files)} TopoGen configs')" # Build and Package build: diff --git a/README.md b/README.md index f4851bd..dd22ac5 100644 --- a/README.md +++ b/README.md @@ -45,12 +45,13 @@ ngraph run output/config_scenario.yml ``` `generate` writes `output/config_integrated_graph.json`, a metro-to-metro corridor -graph, and a preview JPEG. `build` reads the graph and writes +multigraph. Preview JPEGs are written when their export settings are enabled. `build` reads the graph and writes `output/config_scenario.yml`. File prefixes come from the config filename. `build` checks the schema, NetGraph Scenario construction, topology, and hardware. `ngraph run` executes the workflows. Add `--print` to `topogen build` to also print -the YAML and skip scenario validation. +the validated YAML. A failed build preserves the previous scenario YAML; successful +writes replace it atomically. To process a folder of configs, use `./build.sh examples output`. It reuses saved graphs; pass `--force` to regenerate them or `--build-only` to rebuild scenarios. @@ -60,18 +61,34 @@ See `./build.sh --help` for filename filters. ```python from pathlib import Path -from topogen import TopologyConfig, build_integrated_graph, save_to_json +from topogen import RunContext, TopologyConfig, build_integrated_graph, save_to_json +from topogen.scenario import build_scenario +from topogen.validation import validate_scenario_yaml config = TopologyConfig.from_yaml(Path("config.yml")) -graph = build_integrated_graph(config) +context = RunContext(Path("output"), stem="config") +graph = build_integrated_graph(config, context=context) save_to_json( graph, - Path("output/config_integrated_graph.json"), + context.path("integrated_graph.json"), config.projection.target_crs, config.output.formatting, ) +scenario_yaml = build_scenario(graph, config, context=context) +issues = validate_scenario_yaml( + scenario_yaml, + hw_component_map=config.components.hw_component, + optics_map=config.components.optics, +) +assert not issues, issues ``` +Without `context`, the Python generation/build functions return their results +without exporting files. Configuration holds topology and visualization settings; +`RunContext` holds output paths, the filename stem and the optional debug directory. +The CLI supplies this context automatically. `--debug-dir` exports the exact traffic +matrices used for sizing and emission, for all traffic models. + ## Configuration Configs are checked against [the JSON schema](topogen/schemas/topogen_config.json). @@ -95,8 +112,83 @@ workflow arguments when the library is loaded. Traffic models are `uniform`, `gravity`, and `hose`. With `build.tm_sizing.enabled`, TopoGen routes traffic on a collapsed metro graph and sizes corridors for the larger directional load, then applies headroom and capacity increments. Uniform -traffic includes same-metro DC pairs in the total split. Sizing models one demand -node per DC region and rejects unsupported endpoint patterns. +traffic uses `group_mode: group_pairwise`: each ordered pair of distinct DC sites +receives an equal share, including same-metro sites. Device counts within a DC do +not change its share. Sizing aggregates traffic by ordered metro pair before +routing and rejects unsupported endpoint patterns. + +### Current contracts and migration + +There is one supported format at each stage; obsolete inputs are rejected. + +- Regenerate saved graphs with `topogen generate`: JSON requires + `graph_type: corridors` and an explicit key for every parallel edge. The Python + build API accepts an undirected `networkx.MultiGraph` of metro nodes only. + Each metro supplies `name`, `name_orig`, `metro_id`, `x`, `y` and `radius_km`. + Names and IDs must be unique nonempty strings; coordinates must be finite. + The target CRS must be projected with metre units. +- Every discovered path survives extraction, JSON and site expansion. Chain + contraction preserves alternate routes and both arcs of a ring. Risk tags are + aggregated along each path, including risks shared with other paths. +- Import `build_scenario` from `topogen.scenario`; `topogen.scenario_builder` has + been removed. Replace private configuration output attributes with `RunContext` + and public `config.visualization` fields. +- Use `get_builtin_components()` and ordinary dictionary lookup/filtering for the + effective component library. The unused singular/list/filter helpers, + `TopologyConfig.summary()`, standalone point/bearing helpers, and unused + `MetroCluster` distance/overlap/array helpers have been removed. `topogen info` + remains the configuration summary command. +- Delete `highway_processing.min_cycle_nodes`, `validation_sample_size`, + `build.tm_sizing.edge_select`, `output.scenario_metadata`, + `components.assignments`, `components.library`, and failure/workflow + `assignments.scenario_overrides`. These settings did not affect generation. + Library definitions belong in the corresponding `lib/*.yml` file. +- Link `role_pairs` use strings such as `core|leaf`. Optics mappings explicitly + name the **local and remote roles**, using `local->remote`. Specify both + directions when both ends need optics; the module types may differ: + + ```yaml + components: + optics: + leaf->spine: 800G-DR4 + spine->leaf: 1600G-2xDR4 + ``` + + There is no implicit reverse mapping. Old `a-b` and `a|b` optics keys are rejected. +- Scenario hardware definitions must be present in the scenario's `components`. + The builder includes components referenced by configured assignments and used + blueprints. Audits never substitute the local built-in component library. + Port audits count declared modules; optics audits check their capacity. +- Capacity/cost integer inputs must be finite and integral; booleans and fractional + values are errors. Integer strings retain exact precision, and non-finite + configuration numbers are rejected. Unknown routing policies and malformed library files fail + explicitly. Optional libraries may be absent, but a present file must be a mapping. + +Link `cost` is a minimum routing cost; the geometric distance remains separately +available as `distance_km`. Striping accepts only `width` (with optional +`mode: width`) or `mode: by_attr` plus `attribute`. It respects `one_to_one`, +endpoint matches and full device paths, including literal regex metacharacters. +Duplicate normalized priorities or metro overrides, unknown striping options and +unmatched forced metro patterns are errors. Nested blueprint dependencies are +included recursively; reference cycles are rejected. + +Assembly generates traffic once and expands the complete device network once. +Capacity and optics are resolved per site edge against that expansion. Validation +constructs one full NetGraph scenario; coverage audits additionally expand tagged +rules and probe blueprint definitions, retaining node/link rules and variables. +Risk definitions use the length of their owning corridor path; references without +a corresponding path are errors. +DC capacity audits resolve NetGraph selectors and check each demand set separately +against external device-link capacity. `run_ngraph=False` checks metadata and +references only. Combined demand endpoints spanning multiple DCs require workflow +execution to determine their per-DC placement. +Hose generation rejects infeasible margins or a fit that does not converge. +With `rounding_gbps: 0`, gravity and hose preserve unquantized demand values. +Gravity jitter uses the scenario seed without changing Python's global PRNG. +Blueprint diagrams aggregate the already expanded devices and link capacities; +they do not implement a second version of the NetGraph DSL. +Map rendering uses network tiles through Contextily and may dominate wall time; +render failures propagate to the caller. ## Development diff --git a/build.sh b/build.sh index 5beaeb2..b19f8ff 100755 --- a/build.sh +++ b/build.sh @@ -106,17 +106,8 @@ mkdir -p "$OUTPUT_DIR_RAW" || die "Cannot create output directory: $OUTPUT_DIR_R CONFIGS_DIR=$(abs_path "$CONFIGS_DIR_RAW") OUTPUT_DIR=$(abs_path "$OUTPUT_DIR_RAW") -# Detect TopoGen invoker once: prefer installed CLI; otherwise python -m. -TOPGEN_INVOKE=(topogen) -if ! command -v "${TOPGEN_INVOKE[0]}" >/dev/null 2>&1; then - if command -v python3 >/dev/null 2>&1; then - TOPGEN_INVOKE=(python3 -m topogen) - elif command -v python >/dev/null 2>&1; then - TOPGEN_INVOKE=(python -m topogen) - else - die "Neither 'topogen' nor Python found on PATH. Activate your venv or install TopoGen." - fi -fi +# Use the installed CLI from the active environment. +command -v topogen >/dev/null 2>&1 || die "Activate the TopoGen venv or install TopoGen." echo "🚀 TopoGen batch run" OUTPUT_DIR_PRINT=${OUTPUT_DIR%/.} @@ -213,11 +204,10 @@ while IFS= read -r -d '' cfg; do if [[ "$run_generate" == "true" ]]; then # Run generate writing artefacts directly to workdir via -o - ("${TOPGEN_INVOKE[@]}" generate "$cfg_abs" -o "$workdir") 2>&1 | tee "$workdir/generate.log" + topogen generate "$cfg_abs" -o "$workdir" 2>&1 | tee "$workdir/generate.log" gen_ec=${PIPESTATUS[0]} else # Use cached artefacts - : # artefacts already in workdir from previous run echo "⏭️ Skipping generate: found existing ${stem}_integrated_graph.json" | tee "$workdir/generate.log" >/dev/null gen_ec=100 # special code for 'cached' fi @@ -238,27 +228,24 @@ while IFS= read -r -d '' cfg; do fi # Build after successful generation or when a saved graph is available. - build_icon="⏭️" - build_note="skipped" build_ec=-1 scenario_out="$workdir/${stem}_scenario.yml" if [[ $gen_ec -eq 0 || $gen_ec -eq 100 ]]; then # Run build writing scenario into workdir via -o - ("${TOPGEN_INVOKE[@]}" build "$cfg_abs" -o "$scenario_out") 2>&1 | tee "$workdir/build.log" + topogen build "$cfg_abs" -o "$scenario_out" 2>&1 | tee "$workdir/build.log" build_ec=${PIPESTATUS[0]} case "$build_ec" in 0) - build_icon="✅"; build_note="ok"; build_ok=$((build_ok + 1));; + build_ok=$((build_ok + 1));; 3) - build_icon="⚠️"; build_note="validation failed"; build_validation=$((build_validation + 1));; + build_validation=$((build_validation + 1));; 2) - build_icon="❌"; build_note="config error"; build_config=$((build_config + 1));; + build_config=$((build_config + 1));; 1) - build_icon="❌"; build_note="runtime error"; build_runtime=$((build_runtime + 1));; + build_runtime=$((build_runtime + 1));; *) - build_icon="❌"; build_note="exit $build_ec"; build_other=$((build_other + 1));; + build_other=$((build_other + 1));; esac - : else build_skipped=$((build_skipped + 1)) fi @@ -269,7 +256,6 @@ while IFS= read -r -d '' cfg; do if [[ $gen_ec -eq 100 ]]; then gen_col="⏭️ cached"; fi if [[ $gen_ec -ne 0 && $gen_ec -ne 2 && $gen_ec -ne 100 ]]; then gen_col="❌ runtime"; fi - build_col="$build_icon" case "$build_ec" in 0) build_col="✅" ;; 3) build_col="⚠️ validation" ;; @@ -314,4 +300,8 @@ echo " • Generate: ✅ $gen_ok | ⏭️ $gen_cached (cached) | ❌ $gen_fa echo " • Build: ✅ $build_ok | ⚠️ $build_validation (validation) | ❌ $build_runtime (runtime) | ❌ $build_config (config) | ⏭️ $build_skipped (skipped) | ❓ $build_other (other)" echo "======================" +if (( build_ok != total )); then + echo "Batch failed: $((total - build_ok)) config(s) did not build successfully." >&2 + exit 1 +fi echo "Done. ✨" diff --git a/dev/STREAMLINING.md b/dev/STREAMLINING.md new file mode 100644 index 0000000..73ee62c --- /dev/null +++ b/dev/STREAMLINING.md @@ -0,0 +1,363 @@ +# TopoGen streamlining + +Objective: simplify the topology pipeline, make its configuration and graph +contracts explicit, fix correctness gaps, and improve measured performance. +This is one cleanup with implementation gates, not a sequence of releases. + +Baseline: `68fd680`; `make check-ci` passes (315 tests). Official Census inputs +and reproducible baseline artifacts are in ignored `data/` and `output/`. +Before-change profiles and all four sample scenarios are saved in +`output/streamline/before/`. Dragonfly assembly expands NetGraph DSL 28 times +for 27 site adjacencies; capacity counting accounts for over half of profiled +assembly time. + +## Gates + +1. **One network expansion for assembly.** Generate demands once; retain their + identity through sizing and emission. Expand the complete network once, + including node rules; index concrete links by site edge. Use that index for + capacity allocation and optics. Never group distinct capacity budgets by a + shared adjacency family. Verify heterogeneous capacities, striping, custom + blueprints, and semantic equivalence of the four examples. +2. **Explicit configuration and execution context.** Separate serializable + settings from paths and export context. Remove hidden config attributes, + no-op controls, duplicated link parsing, and fallback logic that conceals + invalid state. Keep CLI errors actionable and only report success after + validation. Document intentional contract changes. +3. **Coherent geographic graph contract.** Fix nearest-neighbor adjacency + dependence on metro IDs; make path multiplicity honest throughout discovery, + serialization, site construction, and risks. Replace whole-road-graph scans + per corridor with path/index traversal; preserve geographic provenance. + Verify on adversarial small graphs and the downloaded national datasets. +4. **Validation and delivery.** Preserve node rules in audits, avoid repeated + expansion where the same network suffices, and keep construction, hardware, + capacity and failure-policy checks. Run `make check-ci`, real generation and + build for all example designs, before/after semantic comparisons and + repeatable timings. Update README and Unreleased changelog with the final + contracts, evidence and limitations. No push is part of this task. + +Progress is recorded below as evidence becomes available. Existing behavior is +not a correctness oracle where a regression demonstrates a bug. Timing claims +must separate computation from optional map rendering and network tile access. + + +## Completed evidence + +All four gates are complete. The package now has one explicit corridor graph +contract and one assembly path; removed APIs and formats have no compatibility +adapter. Required migration steps are in [README](../README.md#current-contracts-and-migration). + +- Assembly generates traffic once and expands the complete device DSL once, + including node rules. Capacities and endpoint optics use the actual concrete + links for each site edge. Heterogeneous PoP budgets, asymmetric optics, missing + reverse assignments and caller link attributes have regression coverage. +- Typed `RunContext` owns output paths; configuration has no hidden export state. + Schema checks reject removed controls and formats. Library parsing is shared; + malformed libraries and computation/render failures propagate. CLI validation + precedes atomic YAML replacement, including `--print`. +- Corridor identity survives contraction, k-nearest pairing, k-path extraction, + JSON and site expansion. Risks use the owning path's length and reject unknown + owners. Tests cover parallel routes of different lengths, ID-order dependence, + JSON round trips and shared risks. Road copies and scans were removed from + contraction and corridor extraction where ownership/indexing suffices. +- `make check-ci`: **324 passed**, 87.61% coverage; Ruff formatting/lint, Pyright + and all four example schemas pass. Pyright reports one existing dependency + typing warning for Contextily (no type stub), with zero errors. Full log: + `output/streamline/final-ci.log`. Real NetGraph parsing and reduced workflows + execute in integration tests. Published changelog history is unchanged. + +### Repeatable assembly measurement + +Recorded for the initial cleanup, before the follow-up review below. These +timings were not rerun after the audit corrections. + +Local Python 3.13.13, NetGraph 0.24.0, NetGraph-Core 0.11.0. Compare baseline +`68fd680` in an isolated checkout with this implementation, using the **same +frozen geographic graph**, corresponding example configs and one virtualenv. +One warmup, seven timed iterations; median milliseconds. Optional maps and +artifact I/O are excluded. This is an assembly measurement, not a claim about +national geographic generation or NetGraph simulation throughput. + +| Example | Device nodes / links | Before, ms | After, ms | Build speedup | +| --- | ---: | ---: | ---: | ---: | +| small_baseline | 13 / 27 | 32.2 | 30.1 | 1.07x | +| small_clos | 127 / 756 | 55.3 | 37.4 | 1.48x | +| small_dragonfly | 133 / 739 | 83.5 | 43.9 | 1.90x | +| small_dragonfly_custom | 103 / 612 | 64.2 | 39.9 | 1.61x | + +Evidence: `output/streamline/compare_assembly.py`, `benchmark-before/` and +`benchmark-after/` under `output/streamline/`. Every iteration passed scenario +validation. All four parsed scenario documents are equal after accounting for +explicit `:path0` adjacency identities and newly preserved `euclidean_km` / +`detour_ratio` attributes. Comparison script and result: +`verify_semantics.py`, `semantic-comparison.json` in the same directory. +Corrected geographic routing can intentionally change distances when regenerating +from Census inputs; the frozen-graph comparison isolates assembly changes. + +### Real-data artifacts + +Official Census ZIPs, provenance and integrity/schema/CRS checks are in `data/` +and `data/SOURCES.json`: 2,645 urban areas, 17,522 primary-road features and 56 +state/territory boundaries. All four examples were generated from these data, +then built and validated. Each has 5 metros and 8 corridors. Current artifacts +are in `output/` and `output/streamline/final/`; old outputs are retained under +`output/streamline/before/`. + +```bash +venv/bin/python -m topogen generate examples/small_baseline.yml -o output +venv/bin/python -m topogen build examples/small_baseline.yml -o output +``` + +The same commands succeed for `small_clos`, `small_dragonfly` and +`small_dragonfly_custom`. Maps and blueprint diagrams were rendered; the +baseline geographic/site maps and Clos diagram were visually inspected. +The real example builds with maps took about 12 seconds per CLI invocation; +Contextily tile access and rendering dominate compared with assembly alone. +Full Monte Carlo workflows on these real artifacts were not run. No remote CI, +commit, push or release was performed. + + +## Follow-up correctness review + +The earlier passing suite did not establish absence of defects. Added behavioral +witnesses before fixes; failures and verification logs are in `output/recheck/`. +No compatibility path or format was added during this review. + +- **DC demand accounting:** separate demand sets were summed, regex selectors + bypassed checks, and DSL rules were used instead of actual external link + capacity. The audit now resolves demands through NetGraph, checks every set + independently, excludes internal/disabled links and reports unmatched demands. + Dictionary-only validation now explicitly covers metadata/references, while + capacity auditing belongs to the NetGraph-backed path. +- **Hose fitting:** impossible margins could silently become a different profile. + Both fitting stages now share one vectorized fitter with feasibility and + convergence checks; exactly saturated margins use their unique star solution. +- **Disabled rounding:** gravity and hose rounded to two decimals even with + rounding disabled, erasing small demands. They now preserve unquantized values. +- **Numeric/config contracts:** integer strings above 2^53 lost precision through + float conversion, and NaN/infinity could bypass schema bounds. Exact parsing + and recursive finite-number checks reject these cases. +- **Geographic contracts:** projected metre units are now required; degree/foot + CRS values previously reached metre-based calculations. Duplicate metro names + could merge site identities; names/IDs and coordinate values are now checked. + PoP coordinates are cross-checked even when a metro has no DC group. +- **Validation robustness:** malformed network/attribute/reference shapes report + issues instead of aborting the validator. +- **Component inventory:** hardware overridden in a blueprint remained a library + dependency. Inventory now scans finalized blueprints and reuses that section. +- **Cleanup:** removed an unused graph-filter argument, a silent role-map + substitution and duplicated IPF loops; corrected library and audit API docs. + +Final `make check-ci`: **341 passed**, **87.77% coverage**, Ruff and all four +example schemas pass, Pyright has zero errors and the existing Contextily stub +warning. Added 17 regression cases in `tests/test_review_regressions.py`. +`git diff --check` passes; published changelog history is preserved. + +Additional behavioral evidence: + +- Contraction preserves the first five shortest simple-path lengths on 163 + connected random graphs (`contraction-differential.txt`). +- Sixty seeded hose profiles at scales 1e-6, 1 and 1e6 preserve every DC total + within 1e-8 relative error (`hose-numerics.txt`). +- All four real-data examples are rebuilt and validated with no change to their + parsed scenario documents. All four workflow steps execute on each example + with iteration counts reduced to 3; MSD alpha is positive (1.6875). Results and + the reproducible runner are `real-workflow-results.json` and + `run-real-scenarios.py` in `output/recheck/`. + +This verifies local construction, validation and reduced workflow execution. +Full Monte Carlo runs and remote CI were not run. Changes remain uncommitted; +no push or release was performed. + + +## Full-source simplification and performance audit (completed) + +Objective: inspect all package code and support scripts for straightforward +control flow, duplication, defects and unnecessary computation. Preserve no +legacy compatibility paths. Previous test success is a baseline, not completion. + +Coverage gates: + +1. Traffic models: shared pair selection/rounding/emission, deterministic PRNG, + disabled-debug overhead, bounded fitting and conservation witnesses. +2. Site graph and sizing: endpoint/stripe selection, distinct corridor identities, + repeated expansion/I/O, topology modes, capacity and serialization contracts. +3. Configuration and libraries: one validation boundary, exact conversion, + overrides/references, coherent public loading API. +4. Geographic pipeline: metro selection, road anchoring/contraction, corridor + discovery/risks, serialization and representative real-data execution. +5. Remaining modules: CLI/context, audits, visualization/helpers, entry points, + packaging/dev/CI/workspace scripts and test adequacy. +6. Final evidence: full current checks, adversarial witnesses, before/after + measurements on identical inputs, real examples and package build. + +Inspection and performance artifacts are in `output/full-review/`; the 341-test +result in `output/recheck/` is the starting baseline. Current completion evidence +for every coverage gate is recorded below. + +### Full-source audit completion + +All six coverage gates are complete for the current local source. This is a +behavioral/code audit, not a proof that no possible defect remains. No legacy +adapter or compatibility re-export was introduced. + +| Coverage gate | Inspected scope and outcome | +| --- | --- | +| Traffic | All three models, common inventory, partner selection, fitting, quantization, PRNG and output. Shared emission replaces duplicate branches; debug formatting is conditional. | +| Sites and sizing | All adjacency families, roles, stripe selection, expansion, capacity routing, risk ownership and serialization. Construction, sizing and network serialization have separate modules. Ordered metro-pair demands share one routing operation and one capacity update rule. | +| Config and libraries | Every dataclass/schema section and all four libraries. Schema owns scalar bounds; exact key conversion and cross-field checks remain explicit. Libraries include nested blueprint dependencies, detect cycles, and are finalized once before stripe selection. Workflow definitions load once per assembly. | +| Geography | UAC filtering/overrides, naming, coordinates, road loading/snapping, contraction, anchoring, nearest-neighbor pairs, corridor extraction/risk and JSON. The four national-data examples were regenerated and compared with saved graphs. | +| Remaining package and support | CLI/context/logging/public exports, all validation audits, maps/blueprint diagrams, Makefile, batch/analyzer/workspace scripts, packaging and CI configuration. Tests were migrated to current internal APIs; no compatibility wrappers. | +| Verification | Full local CI target, baseline-failing witnesses, traffic differential/performance checks, real generation/build/validation/reduced workflow runs, package build/metadata checks and installed-wheel smoke test. | + +Additional defects isolated and fixed during this audit: + +- Gravity top-K selection depended on alphabetical metro-name order; jitter used + the global PRNG. Zero-share classes could also enter invalid normalization. +- Uniform traffic expanded across devices, introducing intra-DC traffic and + changing site shares with blueprint size. Explicit `group_pairwise` emission + now matches DC-level sizing. +- Striping ignored `one_to_one` and custom endpoint matches, collapsed equal + device basenames from distinct groups and treated literal names as regexes. + Stripe rules now address exact full paths and merge attributes per device. + Disabled corridor risks no longer leak into emitted adjacencies. +- Config priority conversion truncated fractions; normalized duplicates and + repeated metro overrides overwrote previous entries. Unknown stripe keys and + unmatched forced metros were silently ignored. They now fail explicitly. +- Configured link cost was ignored for geometric links. It now acts as a minimum + routing cost while `distance_km` retains the geographic value. +- Sizing accepted partial endpoint patterns and non-finite/negative volume. + Those inputs are rejected before routing. +- Nested blueprint definitions omitted their dependencies. Both build and stripe + selection now use the same finalized library with recursive dependencies. +- Blueprint diagrams implemented a second, permissive DSL parser; dictionary + selectors created phantom groups. Both panels now use actual expanded devices, + with group counts and summed link capacities. +- The UAC boundary mask assumed NAD83 regardless of source CRS. Metro coordinates + were NumPy scalars, causing direct `generate -> build` Python calls to fail YAML + emission even though a JSON round trip succeeded. The loader now returns native + floats and uses the actual source CRS for filtering. +- Role audits accepted `None`, boolean and numeric roles as strings; chassis + capacity audits accepted NaN/infinity. These malformed values now produce issues. +- Batch builds returned exit 0 after stage failures. The analyzer called k-core + routines on unsupported MultiGraphs. Both have current-contract behavior; + vertex analysis uses simple adjacency and preserves reported corridor counts. + +Final evidence (all paths below are relative to `output/full-review/`): + +- `final-check-ci.log`: **375 passed**, **89.52% coverage**; Ruff and four config + validations pass; Pyright has **0 errors**, with one existing missing Contextily + stub warning. `git diff --check` and shell syntax checks pass. +- The `*-before-tests.log` files retain failing witnesses; `real-examples-before.log` + records the direct Python/YAML scalar failure before its fix. +- `traffic-parity.log`: **72 profiles** cover gravity/hose, multiple inventory + sizes and all rounding policies. Non-volume fields match exactly; directional + volumes match within 1e-12 relative / 1e-10 absolute tolerance. +- `traffic-before-bench.json` and `traffic-final-bench.json`: seven-run local + medians on identical inputs, logging disabled, allocation peak measured + separately. Gravity (80 DCs, 18,960 entries): **90.97 -> 6.50 ms** and + **29,040,021 -> 9,586,183 bytes**. Hose (40 DCs, 3 samples, 14,040 entries): + **12.56 -> 5.33 ms** and **9,911,842 -> 6,815,101 bytes**. These are generator + microbenchmarks, not end-to-end or cross-machine performance claims. +- `real/results.json`: all four examples regenerate **5 metros / 8 corridors**; + device/link counts are **13/27**, **127/756**, **133/739**, **103/612**. + Four workflow steps run on each with Monte Carlo iterations reduced to **3**; + MSD alpha remains **1.6875**. Full Monte Carlo runs were not performed. +- `network-parity.json`: all four expanded networks match the saved baseline + exactly in device attrs and link capacities/costs/risks/hardware. The scenario + text changes only by explicit `logic: or` on single-condition role selectors; + the real examples use hose, so the uniform contract change is exercised by + separate heterogeneous-DC regression tests. +- Fresh maps and blueprint diagrams are under `real/`; the final Clos diagram + was visually inspected for counts, capacities, group layout and readable labels. +- `package-build.log`, `package-check.log`, `pip-check.log`, `wheel-smoke.log`: + wheel/sdist build, Twine metadata checks and dependency checks pass. Importing + the installed wheel (outside the source package path) loads its schema and + builds/validates the real baseline. + +Released changelog history remains byte-for-byte unchanged. Work is local on +`explain-topology-pipeline-2cd9ea05`, based on `68fd680`; changes remain uncommitted. +No push, remote CI, release or deployment was performed. + +### Contextily typing follow-up (2026-10-04) + +Added `typings/contextily/__init__.pyi` for the `add_basemap` arguments used by +TopoGen, checked against installed Contextily 1.7.1. This is an explicitly limited +static description, with no runtime wrapper, `Any` escape hatch or diagnostic +suppression. Pyright uses the explicit `typings` stub path. + +`make lint` now reports **0 errors / 0 warnings**. A separate temporary type probe +accepts the real call and rejects incorrect alpha, attribution and Axes types +(`output/full-review/contextily-typing/probe-result.json`). No runtime source was +changed; the earlier 375-test result remains the latest full-suite run. + +### Code and architecture recheck (2026-10-04) + +This follow-up supersedes the earlier test count. It checks current production +modules, repository callers, tests, comments/docstrings, CLI/batch scripts and +example configuration. GeoPandas, Matplotlib, Contextily and its local stub remain. + +Removed twelve helpers whose only repository consumers were their own tests: +the four point/transform/bearing functions, three MetroCluster convenience +methods, TopologyConfig.summary, and four singular/list/filter component APIs. +The effective component library remains available through get_builtin_components; +topogen info remains the configuration summary command. These are intentional +API removals with no compatibility aliases, documented in README and Unreleased. + +Removed the unused corridor_path_ids edge tags and three redundant CorridorPath +fields. Scenario construction now reads the corridor MultiGraph directly instead +of copying every edge to an intermediate list of dictionaries. The shared site +edge identity lives in naming rather than forcing serialization to depend on +device expansion. Sizing resolves the Core flow-placement enum directly and +combines capacity adjustment, aggregate reporting and PoP egress accumulation. +Removed unused loggers, an empty TYPE_CHECKING block, obsolete type/noqa +suppression comments and dead batch-runner state. Production Python is smaller by +253 lines relative to this review's starting snapshot. + +Two misleading diagnostics were reproduced before changes: + +- A DC-to-PoP YAML rule received an “Inter-metro corridors” heading because the + comment inserter searched the next 15 lines. Deleted this text heuristic; + link_type attributes remain authoritative. Witness: before-build.log. +- Largest-component filtering legitimately removed a metro anchor, but raised + a RuntimeError alleging a contraction bug. It now raises ValueError identifying + highway filtering and the relevant settings. The new regression fails against + the starting source (anchor-before.log) and passes after correction. + +Corrected descriptions of full-mesh PoP adjacency with ring-arc costs, +representative points versus centroids, hose same-metro weighting, component +filtering and geometry cleanup counts. Reused the single dissolved CONUS geometry +without dissolving it again. Duplicate snapped road segments have the same length +by construction, so their unreachable shorter-edge branch was removed. + +CLI tests no longer restore global print or mock sys.exit with another SystemExit +implementation. They now require the expected exceptions. The quiet-mode test +uses valid CLI syntax, checks successful dispatch, and demonstrates that the same +stub prints when quiet mode is absent. Previously it passed on argparse rejecting +the unsupported -c option. Ruff now checks unused noqa directives. The two SciPy +imports retain narrow missing-stub suppressions; removing them was checked and +does produce real missing-stub diagnostics. Contextily needs no suppression. + +Evidence is under output/architecture-review/: + +- final-check-ci.log: 348 tests pass, coverage 89.69%, all four configuration + schemas validate. The count is 375 minus 28 cases belonging to deleted helpers + plus one new anchor-filtering regression. No remaining behavior tests were + removed. An optional unused-type-ignore check exposed a warning inside + Pyright's bundled typeshed; that option was not retained. Final lint is recorded + separately in final-lint.log. +- focused.log: 61 geographic, sizing and assembly checks pass. +- cli-check.log: 18 CLI and batch checks pass, including failure propagation. +- verify_examples.py, example-parity.json: actual national geographic inputs + regenerate the exact saved graph (5 metros, 8 corridors). All four example + geographic configurations were first checked equal, then each scenario rebuilt + and validated. Parsed YAML and expanded node/link attributes, capacities, costs, + risks and hardware exactly match the pre-review snapshots. Device/link counts: + baseline 13/27, Clos 127/756, Dragonfly 133/739, custom Dragonfly 103/612. +- A production symbol-reference scan found no further unreferenced functions + beyond the PyYAML ignore_aliases override, which is invoked by the serializer. + This is repository-level evidence, not proof about unknown external consumers. + +No new workflow simulation, tile download or performance benchmark was needed for +this pass. Shell syntax and git diff --check pass; released changelog history is +unchanged. This review was completed before committing and opening the PR. diff --git a/dev/analyze_integrated_graph.py b/dev/analyze_integrated_graph.py index bf4abea..444fa40 100644 --- a/dev/analyze_integrated_graph.py +++ b/dev/analyze_integrated_graph.py @@ -1,301 +1,90 @@ -"""Integrated graph analysis helper. +"""Report corridor counts and metro vertex connectivity from current graph JSON. -This script loads an integrated (corridor-level) graph JSON and prints: - -- Metros ordered by number of adjacencies (degree) -- Global node connectivity and a minimum vertex cut -- A high-degree vertex cut of the same cardinality when possible -- Auxiliary metrics (k-core size, articulation points if any) - -Usage: - python dev/analyze_integrated_graph.py baseline_integrated_graph.json +Usage: python dev/analyze_integrated_graph.py output/small_baseline_integrated_graph.json +Parallel corridors count toward reported degree. Vertex cuts and k-cores operate +on the simple adjacency graph, since parallel paths do not add metro neighbors. """ from __future__ import annotations -from dataclasses import dataclass from itertools import combinations from pathlib import Path -from typing import Iterable import networkx as nx -import pandas as pd from topogen.integrated_graph import load_from_json -@dataclass(frozen=True, slots=True) -class MetroInfo: - """Basic metro information used for reporting. - - Attributes: - node_key: NetworkX node key (coordinate tuple). - metro_id: Metro ID string if available. - name: Sanitized metro name if available, else metro_id or coord string. - degree: Unweighted degree in the corridor graph. - """ - - node_key: tuple[float, float] - metro_id: str - name: str - degree: int - - -def _format_int(x: int) -> str: - return f"{x:,}" - - -def _format_float(x: float, decimals: int = 2) -> str: - return f"{x:,.{decimals}f}" - - -def _is_disconnected_after_removal(graph: nx.Graph, nodes_to_remove: Iterable) -> bool: - """Return whether removing nodes empties or disconnects the graph.""" - - remaining = graph.copy() - remaining.remove_nodes_from(nodes_to_remove) - if remaining.number_of_nodes() == 0: - return True - return not nx.is_connected(remaining) - - -def _find_high_degree_cut( - graph: nx.Graph, - k: int, - search_pool_size: int = 20, -) -> list[tuple[float, float]]: - """Try k-node cuts from the highest-degree ``search_pool_size`` nodes. - - Return the first cut found, or [] if none. Requires up to C(pool, k) - connectivity checks. - """ - - if k <= 0: - return [] - - degree_pairs: list[tuple[tuple[float, float], int]] = [ - (n, d) - for n, d in list(graph.degree()) # type: ignore[misc] - ] - degree_pairs_sorted = sorted(degree_pairs, key=lambda t: t[1], reverse=True) - pool_limit = min(search_pool_size, len(degree_pairs_sorted)) - pool_nodes = [n for n, _d in degree_pairs_sorted[:pool_limit]] - - for combo in combinations(pool_nodes, k): - if _is_disconnected_after_removal(graph, combo): - return list(combo) +def _find_high_degree_cut(graph: nx.Graph, k: int, pool_size: int = 20) -> list: + """Search up to C(pool_size, k) cuts, testing views without copying the graph.""" + pool = sorted(graph, key=lambda node: (-graph.degree(node), node))[:pool_size] + for cut in combinations(pool, k): + remaining = graph.subgraph(graph.nodes - set(cut)) + if len(remaining) <= 1 or not nx.is_connected(remaining): + return list(cut) return [] -def _greedy_disconnect( - graph: nx.Graph, order: list[tuple[float, float]] -) -> list[tuple[float, float]]: - """Remove nodes in order until disconnected, then discard redundant removals.""" - - removed: list[tuple[float, float]] = [] - for node in order: - removed.append(node) - if _is_disconnected_after_removal(graph, removed): - break - - # Post-shrink: try to drop redundant nodes while preserving disconnection - changed = True - while changed: - changed = False - for n in list(removed): - test = [x for x in removed if x != n] - if _is_disconnected_after_removal(graph, test): - removed = test - changed = True - return removed - - -def _collect_metros(graph: nx.Graph) -> list[MetroInfo]: - """Build metro records, using metro ID or coordinates when the name is absent.""" - - infos: list[MetroInfo] = [] - for node, data in graph.nodes(data=True): - metro_id = str(data.get("metro_id", "")) - name_attr = data.get("name") - if isinstance(name_attr, str) and name_attr: - name = name_attr - elif metro_id: - name = metro_id - else: - name = f"({node[0]:.0f},{node[1]:.0f})" - deg_val: int = graph.degree(node) # type: ignore[assignment] - infos.append( - MetroInfo( - node_key=(float(node[0]), float(node[1])), - metro_id=metro_id, - name=name, - degree=deg_val, - ) - ) - return infos - - -def _build_degree_frame(metros: list[MetroInfo]) -> pd.DataFrame: - df = pd.DataFrame( - { - "node_key": [m.node_key for m in metros], - "metro_id": [m.metro_id for m in metros], - "name": [m.name for m in metros], - "degree": [m.degree for m in metros], - } - ) - df = df.sort_values(["degree", "name"], ascending=[False, True]).reset_index( - drop=True - ) - return df - - def analyze_graph(json_path: Path, top_n: int | None = None) -> None: - """Print corridor connectivity, node rankings, and candidate vertex cuts.""" - - graph, crs = load_from_json(json_path) - - # Basic shape and connectivity - num_nodes = graph.number_of_nodes() - num_edges = graph.number_of_edges() - is_conn = nx.is_connected(graph) - kconn = nx.node_connectivity(graph) if num_nodes > 1 else 0 - - print(f"File: {json_path}") - print(f"CRS: {crs}") + corridors, crs = load_from_json(json_path) + if not corridors: + raise ValueError("Cannot analyze an empty corridor graph") + graph = nx.Graph(corridors) + degrees = dict(corridors.degree()) + order = sorted(graph, key=lambda node: (-degrees[node], graph.nodes[node]["name"])) + connectivity = nx.node_connectivity(graph) if len(graph) > 1 else 0 + print(f"File: {json_path}\nCRS: {crs}") print( - f"Graph: nodes={_format_int(num_nodes)}, edges={_format_int(num_edges)}, connected={is_conn}" + f"Graph: nodes={len(graph):,}, corridors={corridors.number_of_edges():,}, connected={nx.is_connected(graph)}" ) - print(f"Global node connectivity (k): {kconn}") - - # Degree ranking - metros = _collect_metros(graph) - df = _build_degree_frame(metros) - - limit = top_n if isinstance(top_n, int) and top_n > 0 else len(df) - print("\nMetros ordered by number of adjacencies (degree):") - for rank, (_idx, row) in enumerate(df.head(limit).iterrows(), start=1): - name = row["name"] - degree = row["degree"] - mid = row["metro_id"] - print(f" {rank:>3}. {name:35s} degree={degree:2d} id={mid}") + print(f"Global node connectivity (k): {connectivity}") - # Minimum vertex cut (size k) - min_cut_nodes: list[tuple[float, float]] = [] - if kconn > 0: - from networkx.algorithms.connectivity import cuts as nx_cuts + def report(title: str, nodes: list) -> None: + print(f"\n{title}:") + for node in nodes: + attrs = graph.nodes[node] + print( + f" {attrs['name']:35s} degree={degrees[node]:2d} id={attrs['metro_id']}" + ) - min_cut_set = nx_cuts.minimum_node_cut(graph) # global min cut - min_cut_nodes = list(min_cut_set) - deg_map: dict[tuple[float, float], int] = { - n: graph.degree(n) - for n in graph.nodes # type: ignore[dict-item] - } - min_cut_nodes = sorted( - min_cut_nodes, key=lambda n: deg_map.get(n, 0), reverse=True + report("Metros ordered by corridor degree", order[:top_n]) + if connectivity: + cut = nx.minimum_node_cut(graph) + report( + f"Minimum vertex cut (size {len(cut)})", + [node for node in order if node in cut], ) - - if min_cut_nodes: - print(f"\nMinimum vertex cut (size {len(min_cut_nodes)}):") - deg_map_print: dict[tuple[float, float], int] = { - n: graph.degree(n) - for n in graph.nodes # type: ignore[dict-item] - } - for n in min_cut_nodes: - data = graph.nodes[n] - name = str(data.get("name", data.get("metro_id", str(n)))) - print(f" - {name} (degree={deg_map_print.get(n, 0)})") - else: - print("\nMinimum vertex cut: none (graph disconnected or trivial)") - - # High-degree cut search of size k - if kconn > 0: - hd_cut = _find_high_degree_cut(graph, k=kconn, search_pool_size=20) - if hd_cut: - print( - f"\nHigh-degree vertex cut of size {kconn} (within top-20 by degree):" + high_degree = _find_high_degree_cut(graph, connectivity) + if high_degree: + report( + f"High-degree vertex cut (size {connectivity}, top-20 pool)", + high_degree, ) - deg_map_hd: dict[tuple[float, float], int] = { - n: graph.degree(n) - for n in graph.nodes # type: ignore[dict-item] - } - for n in hd_cut: - data = graph.nodes[n] - name = str(data.get("name", data.get("metro_id", str(n)))) - print(f" - {name} (degree={deg_map_hd.get(n, 0)})") else: - # Fallback: greedy removal until disconnected - degree_pairs2: list[tuple[tuple[float, float], int]] = [ - (n, d) - for n, d in list(graph.degree()) # type: ignore[misc] - ] - ordered_nodes = [ - n for n, _d in sorted(degree_pairs2, key=lambda t: t[1], reverse=True) - ] - greedy = _greedy_disconnect(graph, ordered_nodes) - print( - f"\nNo size-{kconn} cut found among top-20 degrees. Greedy disconnection set size={len(greedy)}:" - ) - deg_map_hd2: dict[tuple[float, float], int] = { - n: graph.degree(n) - for n in graph.nodes # type: ignore[dict-item] - } - for n in greedy: - data = graph.nodes[n] - name = str(data.get("name", data.get("metro_id", str(n)))) - print(f" - {name} (degree={deg_map_hd2.get(n, 0)})") - - # Articulation points and k-core for additional context - arts = list(nx.articulation_points(graph)) if num_nodes > 0 else [] - if arts: - print(f"\nArticulation points (cut vertices): {len(arts)}") - deg_map2: dict[tuple[float, float], int] = { - n: graph.degree(n) - for n in graph.nodes # type: ignore[dict-item] - } - arts_sorted = sorted(arts, key=lambda n: deg_map2.get(n, 0), reverse=True) - show_ap = min(20, len(arts_sorted)) - for n in arts_sorted[:show_ap]: - data = graph.nodes[n] - name = str(data.get("name", data.get("metro_id", str(n)))) - print(f" - {name} (degree={deg_map2.get(n, 0)})") - - core_num = nx.core_number(graph) if num_nodes > 0 else {} - if core_num: - max_core_k = max(core_num.values()) - in_max_core = [n for n, k in core_num.items() if k == max_core_k] - print(f"\nMax k-core: k={max_core_k}, size={len(in_max_core)}") - # Show up to 20 members sorted by degree - deg_map3: dict[tuple[float, float], int] = { - n: graph.degree(n) - for n in graph.nodes # type: ignore[dict-item] - } - in_max_core_sorted = sorted( - in_max_core, key=lambda n: deg_map3.get(n, 0), reverse=True + print(f"\nNo size-{connectivity} cut found in the top-20 pool.") + else: + print("\nMinimum vertex cut: none (graph disconnected or trivial)") + articulation = set(nx.articulation_points(graph)) + if articulation: + report( + "Articulation points", [node for node in order if node in articulation][:20] ) - show_core = min(20, len(in_max_core_sorted)) - for n in in_max_core_sorted[:show_core]: - data = graph.nodes[n] - name = str(data.get("name", data.get("metro_id", str(n)))) - print(f" - {name} (degree={deg_map3.get(n, 0)})") + cores = nx.core_number(graph) + maximum = max(cores.values()) + members = [node for node in order if cores[node] == maximum] + report(f"Max k-core (k={maximum}, size={len(members)})", members[:20]) def main() -> None: import argparse - parser = argparse.ArgumentParser( - description="Analyze an integrated corridor graph JSON" - ) - parser.add_argument("json_path", type=Path, help="Path to integrated graph JSON") - parser.add_argument( - "--top", - type=int, - default=None, - help="Only print top-N metros by degree (default: all)", - ) + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("json_path", type=Path) + parser.add_argument("--top", type=int, default=None) args = parser.parse_args() - - analyze_graph(args.json_path, top_n=args.top) + if args.top is not None and args.top <= 0: + parser.error("--top must be positive") + analyze_graph(args.json_path, args.top) if __name__ == "__main__": diff --git a/examples/small_baseline.yml b/examples/small_baseline.yml index dabfe5d..5177408 100644 --- a/examples/small_baseline.yml +++ b/examples/small_baseline.yml @@ -1,7 +1,4 @@ output: - scenario_metadata: - title: "Small Test: East Coast, Baseline" - formatting: json_indent: 2 yaml_anchors: true @@ -44,7 +41,6 @@ build: alpha_dc_to_pop: 1.2 beta_intra_pop: 0.8 flow_placement: EQUAL_BALANCED - edge_select: ALL_MIN_COST respect_min_base_capacity: true components: @@ -54,13 +50,16 @@ components: spine: SpineRouter dc: "" optics: - core|core: 800G-ZR+ - leaf|leaf: 800G-ZR+ - core|leaf: 800G-ZR+ - leaf|dc: 800G-ZR+ - core|dc: 800G-ZR+ - leaf|spine: 800G-DR4 - spine|leaf: 1600G-2xDR4 + core->core: 800G-ZR+ + leaf->leaf: 800G-ZR+ + core->leaf: 800G-ZR+ + leaf->core: 800G-ZR+ + leaf->dc: 800G-ZR+ + dc->leaf: 800G-ZR+ + core->dc: 800G-ZR+ + dc->core: 800G-ZR+ + leaf->spine: 800G-DR4 + spine->leaf: 1600G-2xDR4 failure_policies: assignments: @@ -126,10 +125,7 @@ highway_processing: min_edge_length_km: 0.0 snap_precision_m: 150.0 highway_classes: ["S1100"] - min_cycle_nodes: 3 filter_largest_component: true - validation_sample_size: 5 - corridors: k_paths: 1 k_nearest: 6 diff --git a/examples/small_clos.yml b/examples/small_clos.yml index 98600e6..108ba5f 100644 --- a/examples/small_clos.yml +++ b/examples/small_clos.yml @@ -1,7 +1,4 @@ output: - scenario_metadata: - title: "Small Test: East Coast, Clos_L16_S4" - formatting: json_indent: 2 yaml_anchors: true @@ -44,7 +41,6 @@ build: alpha_dc_to_pop: 1.2 beta_intra_pop: 0.8 flow_placement: EQUAL_BALANCED - edge_select: ALL_MIN_COST respect_min_base_capacity: true components: @@ -54,13 +50,16 @@ components: spine: SpineRouter dc: "" optics: - core|core: 800G-ZR+ - leaf|leaf: 800G-ZR+ - core|leaf: 800G-ZR+ - leaf|dc: 800G-ZR+ - core|dc: 800G-ZR+ - leaf|spine: 800G-DR4 - spine|leaf: 1600G-2xDR4 + core->core: 800G-ZR+ + leaf->leaf: 800G-ZR+ + core->leaf: 800G-ZR+ + leaf->core: 800G-ZR+ + leaf->dc: 800G-ZR+ + dc->leaf: 800G-ZR+ + core->dc: 800G-ZR+ + dc->core: 800G-ZR+ + leaf->spine: 800G-DR4 + spine->leaf: 1600G-2xDR4 failure_policies: assignments: @@ -126,10 +125,7 @@ highway_processing: min_edge_length_km: 0.0 snap_precision_m: 150.0 highway_classes: ["S1100"] - min_cycle_nodes: 3 filter_largest_component: true - validation_sample_size: 5 - corridors: k_paths: 1 k_nearest: 6 diff --git a/examples/small_dragonfly.yml b/examples/small_dragonfly.yml index aa79acd..a892a47 100644 --- a/examples/small_dragonfly.yml +++ b/examples/small_dragonfly.yml @@ -1,7 +1,4 @@ output: - scenario_metadata: - title: "Small Test: East Coast, Dragonfly_A3H2G7" - formatting: json_indent: 2 yaml_anchors: true @@ -44,7 +41,6 @@ build: alpha_dc_to_pop: 1.2 beta_intra_pop: 0.8 flow_placement: EQUAL_BALANCED - edge_select: ALL_MIN_COST respect_min_base_capacity: true components: @@ -54,13 +50,16 @@ components: spine: SpineRouter dc: "" optics: - core|core: 800G-ZR+ - leaf|leaf: 800G-ZR+ - core|leaf: 800G-ZR+ - leaf|dc: 800G-ZR+ - core|dc: 800G-ZR+ - leaf|spine: 800G-DR4 - spine|leaf: 1600G-2xDR4 + core->core: 800G-ZR+ + leaf->leaf: 800G-ZR+ + core->leaf: 800G-ZR+ + leaf->core: 800G-ZR+ + leaf->dc: 800G-ZR+ + dc->leaf: 800G-ZR+ + core->dc: 800G-ZR+ + dc->core: 800G-ZR+ + leaf->spine: 800G-DR4 + spine->leaf: 1600G-2xDR4 failure_policies: assignments: @@ -126,10 +125,7 @@ highway_processing: min_edge_length_km: 0.0 snap_precision_m: 150.0 highway_classes: ["S1100"] - min_cycle_nodes: 3 filter_largest_component: true - validation_sample_size: 5 - corridors: k_paths: 1 k_nearest: 6 diff --git a/examples/small_dragonfly_custom.yml b/examples/small_dragonfly_custom.yml index 89d09b0..bb75ad9 100644 --- a/examples/small_dragonfly_custom.yml +++ b/examples/small_dragonfly_custom.yml @@ -1,7 +1,4 @@ output: - scenario_metadata: - title: "Small Test: East Coast, DragonFly_CustomG4" - formatting: json_indent: 2 yaml_anchors: true @@ -44,7 +41,6 @@ build: alpha_dc_to_pop: 1.2 beta_intra_pop: 0.8 flow_placement: EQUAL_BALANCED - edge_select: ALL_MIN_COST respect_min_base_capacity: true components: @@ -54,13 +50,16 @@ components: spine: SpineRouter dc: "" optics: - core|core: 800G-ZR+ - leaf|leaf: 800G-ZR+ - core|leaf: 800G-ZR+ - leaf|dc: 800G-ZR+ - core|dc: 800G-ZR+ - leaf|spine: 800G-DR4 - spine|leaf: 1600G-2xDR4 + core->core: 800G-ZR+ + leaf->leaf: 800G-ZR+ + core->leaf: 800G-ZR+ + leaf->core: 800G-ZR+ + leaf->dc: 800G-ZR+ + dc->leaf: 800G-ZR+ + core->dc: 800G-ZR+ + dc->core: 800G-ZR+ + leaf->spine: 800G-DR4 + spine->leaf: 1600G-2xDR4 failure_policies: assignments: @@ -126,10 +125,7 @@ highway_processing: min_edge_length_km: 0.0 snap_precision_m: 150.0 highway_classes: ["S1100"] - min_cycle_nodes: 3 filter_largest_component: true - validation_sample_size: 5 - corridors: k_paths: 1 k_nearest: 6 diff --git a/pyproject.toml b/pyproject.toml index b0681ff..29cd5fb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -103,7 +103,7 @@ line-length = 88 indent-width = 4 [tool.ruff.lint] -select = ["E4", "E7", "E9", "F", "B", "I"] # core + Bugbear + isort +select = ["E4", "E7", "E9", "F", "B", "I", "RUF100"] # core + Bugbear + isort + unused noqa ignore = ["E501"] # allow long strings and comments fixable = ["ALL"] @@ -116,6 +116,8 @@ skip-magic-trailing-comma = false # Pyright / Pylance [tool.pyright] +include = ["topogen"] +stubPath = "typings" typeCheckingMode = "standard" pythonVersion = "3.11" venvPath = "." diff --git a/tests/integration/test_blueprint_diagram_integration.py b/tests/integration/test_blueprint_diagram_integration.py index 4712d28..ef900bb 100644 --- a/tests/integration/test_blueprint_diagram_integration.py +++ b/tests/integration/test_blueprint_diagram_integration.py @@ -1,52 +1,25 @@ -from __future__ import annotations - from pathlib import Path -from topogen.visualization import export_blueprint_diagram - - -class _Node: - def __init__(self, name: str) -> None: - self.name = name - +from ngraph import Link, Network, Node -class _Link: - def __init__(self, src, dst, cap: float) -> None: - self.source = src - self.target = dst - self.capacity = cap - - -def _make_stub_net(): - class _Net: - pass - - net = _Net() - net.nodes = { - 1: _Node("metro1/dc1/A1"), - 2: _Node("metro1/dc1/B1"), - 3: _Node("metro2/dc3/X1"), - } - net.links = { - 1: _Link(net.nodes[1], net.nodes[2], 10.0), - 2: _Link(net.nodes[1], net.nodes[3], 20.0), - } - return net +from topogen.visualization import export_blueprint_diagram def test_export_blueprint_diagram_smoke(tmp_path: Path) -> None: - # One intra-group mesh and one inter-group link. - bp = { - "nodes": { - "G1": {"count": 2}, - "G2": {"count": 1}, - }, - "links": [ - {"source": "G1", "target": "G1", "pattern": "mesh"}, - {"source": "G1", "target": "G2", "pattern": "uplink"}, - ], - } - net = _make_stub_net() - out = tmp_path / "bp.jpg" - export_blueprint_diagram("UnitBP", bp, net, "metro1/dc1", out) - assert out.exists() and out.stat().st_size > 1000 + network = Network() + for name in ( + "metro1/dc1/a/r1", + "metro1/dc1/a/r2", + "metro1/dc1/b/r1", + "metro2/dc1/c/r1", + ): + network.add_node(Node(name, attrs={"role": "core"})) + for source, target in ( + ("metro1/dc1/a/r1", "metro1/dc1/a/r2"), + ("metro1/dc1/a/r1", "metro1/dc1/b/r1"), + ("metro1/dc1/a/r1", "metro2/dc1/c/r1"), + ): + network.add_link(Link(source, target, capacity=10)) + output = tmp_path / "bp.jpg" + export_blueprint_diagram("UnitBP", network, "metro1/dc1", output) + assert output.exists() and output.stat().st_size > 1000 diff --git a/tests/integration/test_netgraph_scenarios.py b/tests/integration/test_netgraph_scenarios.py index 046eea7..bde2052 100644 --- a/tests/integration/test_netgraph_scenarios.py +++ b/tests/integration/test_netgraph_scenarios.py @@ -11,14 +11,15 @@ from topogen.blueprints_lib import get_builtin_blueprints from topogen.config import TopologyConfig -from topogen.scenario_builder import build_scenario +from topogen.corridors import corridor_risk_name +from topogen.scenario import build_scenario from topogen.validation import validate_scenario_yaml REPO = Path(__file__).resolve().parents[2] def _graph(): - graph = nx.Graph() + graph = nx.MultiGraph() for index, name in enumerate(["alpha", "beta", "gamma"]): graph.add_node( name, @@ -28,6 +29,7 @@ def _graph(): x=index * 300_000.0, y=0.0, radius_km=20.0, + name_orig=name, ) for source, target in [("alpha", "beta"), ("beta", "gamma"), ("alpha", "gamma")]: graph.add_edge( @@ -87,6 +89,15 @@ def test_example_scenario_runs_with_current_netgraph(example, monkeypatch): strict=True, ): graph.nodes[node]["name"] = name + for source, target, key, data in graph.edges(keys=True, data=True): + data["risk_groups"] = [ + corridor_risk_name( + "corridor_risk", + graph.nodes[source]["name"], + graph.nodes[target]["name"], + key, + ) + ] scenario_yaml = build_scenario(graph, config) assert not validate_scenario_yaml( scenario_yaml, diff --git a/tests/integration/test_site_graph_render_smoke.py b/tests/integration/test_site_graph_render_smoke.py index 4269e91..45accb8 100644 --- a/tests/integration/test_site_graph_render_smoke.py +++ b/tests/integration/test_site_graph_render_smoke.py @@ -18,6 +18,7 @@ def test_export_site_graph_map_smoke(tmp_path: Path) -> None: center_y=0.0, radius_m=10_000.0, metro_idx=1, + site_kind="dc", metro_name="M1", ) G.add_node( @@ -28,6 +29,7 @@ def test_export_site_graph_map_smoke(tmp_path: Path) -> None: center_y=0.0, radius_m=10_000.0, metro_idx=1, + site_kind="dc", metro_name="M1", ) G.add_edge( diff --git a/tests/integration/test_validation_optics_checks.py b/tests/integration/test_validation_optics_checks.py index d4520f7..4e1d2e1 100644 --- a/tests/integration/test_validation_optics_checks.py +++ b/tests/integration/test_validation_optics_checks.py @@ -23,15 +23,10 @@ def link(a: str, b: str, cap: float, aid: str, hw: dict | None = None): return net, link -def test_conflicting_optics_mapping_detected(): +def test_legacy_optics_mapping_is_rejected(): net, _ = _net_two_roles() - # No hardware; mapping-only path exercised - net.links = {} - # Create conflict by mixing unordered and ordered declarations for the same ordered pair - d = {"components": {"optics": {"core|agg": "X", "core-agg": "Y"}}} - issues = check_link_optics(net, d, {}) - # Conflict should be recorded for (core,agg) - assert any("conflicting values for roles (core,agg)" in s for s in issues) + issues = check_link_optics(net, {"core|agg": "X"}, {}) + assert any("role->role" in issue for issue in issues) def test_missing_hardware_and_unknown_and_capacity_shortfall(): @@ -63,9 +58,7 @@ def test_missing_hardware_and_unknown_and_capacity_shortfall(): net.links = {"l1": l_missing_src, "l2": l_unknown_src, "l3": l_cap_short} comp_lib = {"O": {"capacity": 100.0}} # Provide a mapping so the function does not early-return - issues = check_link_optics( - net, {"components": {"optics": {"core-agg": "O"}}}, comp_lib - ) + issues = check_link_optics(net, {"core->agg": "O", "agg->core": "O"}, comp_lib) assert any("missing hardware on source end" in s for s in issues) assert any("unknown component on source end" in s for s in issues) assert any("hardware capacity shortfall" in s for s in issues) @@ -77,7 +70,7 @@ def test_missing_mapping_paths_when_no_hardware_present(): l_plain = make_link("A", "B", 10.0, "plain") net.links = {"e": l_plain} # Mapping exists only for (core,agg); this will require source-end hardware - d = {"components": {"optics": {"core-agg": "O"}}} + d = {"core->agg": "O", "agg->core": "O"} issues = check_link_optics(net, d, {"O": {"capacity": 100}}) assert any( "missing source-end mapping" in s diff --git a/tests/integration/test_validation_port_budget.py b/tests/integration/test_validation_port_budget.py index 11a6439..25eda53 100644 --- a/tests/integration/test_validation_port_budget.py +++ b/tests/integration/test_validation_port_budget.py @@ -19,7 +19,10 @@ def _net_for_ports(): target="B", capacity=400.0, attrs={ - "hardware": {"source": {"component": "O"}, "target": {"component": "O"}} + "hardware": { + "source": {"component": "O", "count": 4}, + "target": {"component": "O", "count": 4}, + } }, ) net = SimpleNamespace(nodes={"A": nA, "B": nB}, links={"l1": l1}) @@ -28,16 +31,14 @@ def _net_for_ports(): def test_audit_port_budget_no_issue_when_ports_sufficient(): net = _net_for_ports() - scenario = {"components": {"optics": {"core|core": "O"}}} comp_lib = {"P": {"ports": 8}, "O": {"capacity": 100, "ports": 1}} - issues = audit_port_budget(net, scenario, comp_lib) + issues = audit_port_budget(net, comp_lib) assert issues == [] def test_audit_port_budget_detects_shortage(): net = _net_for_ports() - scenario = {"components": {"optics": {"core|core": "O"}}} # Only 2 ports available per node → shortage (need 4) comp_lib = {"P": {"ports": 2}, "O": {"capacity": 100, "ports": 1}} - issues = audit_port_budget(net, scenario, comp_lib) + issues = audit_port_budget(net, comp_lib) assert issues and "requires 4 ports" in issues[0] diff --git a/tests/integration/test_visualization_maps.py b/tests/integration/test_visualization_maps.py index 7d4a613..f10838c 100644 --- a/tests/integration/test_visualization_maps.py +++ b/tests/integration/test_visualization_maps.py @@ -4,13 +4,14 @@ import networkx as nx import numpy as np +import pytest import topogen.geo_utils as geoutils import topogen.visualization as viz class _ConusStub: - def plot(self, ax=None, **kwargs): # type: ignore[no-untyped-def] + def plot(self, ax=None, **kwargs): return None @@ -24,7 +25,7 @@ def __init__( self.radius_km = radius_km -def _patch_context_and_conus(monkeypatch): # type: ignore[no-untyped-def] +def _patch_context_and_conus(monkeypatch): # Avoid network calls and GIS dependencies for basemap and CONUS monkeypatch.setattr(viz.cx, "add_basemap", lambda *args, **kwargs: None) # export_* functions import create_conus_mask from topogen.geo_utils at call time @@ -33,7 +34,7 @@ def _patch_context_and_conus(monkeypatch): # type: ignore[no-untyped-def] ) -def test_export_cluster_map_success(tmp_path: Path, monkeypatch) -> None: # type: ignore[no-untyped-def] +def test_export_cluster_map_success(tmp_path: Path, monkeypatch) -> None: _patch_context_and_conus(monkeypatch) out = tmp_path / "clusters.jpg" # Create a dummy file so path.exists() passes @@ -46,7 +47,7 @@ def test_export_cluster_map_success(tmp_path: Path, monkeypatch) -> None: # typ def test_export_integrated_graph_map_corridor_graph( tmp_path: Path, monkeypatch -) -> None: # type: ignore[no-untyped-def] +) -> None: _patch_context_and_conus(monkeypatch) out = tmp_path / "integrated_corridor.jpg" conus = tmp_path / "conus.boundary" @@ -56,7 +57,7 @@ def test_export_integrated_graph_map_corridor_graph( _Metro("A", "A_name", (0.0, 0.0), 10.0), _Metro("B", "B_name", (2.0, 0.0), 8.0), ] - G = nx.Graph() + G = nx.MultiGraph() # corridor edges with metadata specifying metro ids (straight line mode) G.add_edge("A_node", "B_node", edge_type="corridor", metro_a="A", metro_b="B") @@ -66,7 +67,7 @@ def test_export_integrated_graph_map_corridor_graph( assert out.exists() and out.stat().st_size > 1000 -def test_export_integrated_graph_map_real_geometry(tmp_path: Path, monkeypatch) -> None: # type: ignore[no-untyped-def] +def test_export_integrated_graph_map_real_geometry(tmp_path: Path, monkeypatch) -> None: _patch_context_and_conus(monkeypatch) out = tmp_path / "integrated_geom.jpg" conus = tmp_path / "conus.boundary" @@ -76,7 +77,7 @@ def test_export_integrated_graph_map_real_geometry(tmp_path: Path, monkeypatch) _Metro("A", "A_name", (0.0, 0.0), 10.0), _Metro("B", "B_name", (2.0, 0.0), 8.0), ] - G = nx.Graph() + G = nx.MultiGraph() # corridor edges providing explicit geometry G.add_edge( "A_node", @@ -93,7 +94,7 @@ def test_export_integrated_graph_map_real_geometry(tmp_path: Path, monkeypatch) def test_export_integrated_graph_map_full_graph_from_tags( tmp_path: Path, monkeypatch -) -> None: # type: ignore[no-untyped-def] +) -> None: _patch_context_and_conus(monkeypatch) out = tmp_path / "integrated_full.jpg" conus = tmp_path / "conus.boundary" @@ -109,7 +110,6 @@ def test_export_integrated_graph_map_full_graph_from_tags( # Full highway graph style: corridor tag on edges with list of pairs G.add_edge("mA", "mB", corridor=[{"metro_a": "A", "metro_b": "B"}]) - viz.export_integrated_graph_map( - metros, G, out, conus, "EPSG:5070", use_real_geometry=False, dpi=120 - ) - assert out.exists() and out.stat().st_size > 1000 + with pytest.raises(TypeError, match="MultiGraph"): + viz.export_integrated_graph_map(metros, G, out, conus, "EPSG:5070") + assert not out.exists() diff --git a/tests/test_assembly_contract.py b/tests/test_assembly_contract.py new file mode 100644 index 0000000..c3e060b --- /dev/null +++ b/tests/test_assembly_contract.py @@ -0,0 +1,110 @@ +"""Behavioral contracts shared by sizing, device expansion and hardware.""" + +import math + +import networkx as nx +import yaml +from ngraph.dsl.blueprints.expand import expand_network_dsl + +from topogen.config import TopologyConfig +from topogen.scenario import build_scenario + + +def test_optics_follow_each_site_edge_capacity(): + """Different PoP egress budgets in one metro need different optic counts.""" + graph = nx.MultiGraph() + for i, name in enumerate(("alpha", "beta")): + graph.add_node( + (i * 100_000.0, 0.0), + node_type="metro", + name=name, + metro_id=name, + x=i * 100_000.0, + y=0.0, + radius_km=10.0, + name_orig=name, + ) + graph.add_edge((0.0, 0.0), (100_000.0, 0.0), edge_type="corridor", length_km=100.0) + cfg = TopologyConfig() + defaults = cfg.build.build_defaults + defaults.pop_per_metro = 3 + defaults.dc_regions_per_metro = 1 + defaults.inter_metro_link.mode = "one_to_one" + defaults.inter_metro_link.attrs = {"owner": "fixture"} + defaults.intra_metro_link.role_pairs = ["core|core"] + defaults.inter_metro_link.role_pairs = ["core|core"] + defaults.dc_to_pop_link.role_pairs = ["dc|core"] + cfg.build.build_overrides = {"beta": {"pop_per_metro": 2}} + cfg.build.tm_sizing.enabled = True + cfg.traffic.enabled = True + cfg.traffic.model = "uniform" + cfg.traffic.mw_per_dc_region = 100.0 + cfg.traffic.gbps_per_mw = 200.0 + cfg.components.hw_component = {"core": "CoreRouter", "dc": ""} + cfg.components.optics = { + "core->core": "800G-ZR+", + "dc->core": "800G-ZR+", + "core->dc": "800G-ZR+", + } + + scenario = yaml.safe_load(build_scenario(graph, cfg)) + network = expand_network_dsl(scenario) + wan = [ + link + for link in network.links.values() + if link.attrs["link_type"] == "inter_metro_corridor" + ] + assert wan and all(link.attrs["owner"] == "fixture" for link in wan) + capacities = {link.capacity for link in network.links.values()} + assert len(capacities) > 1 + for link in network.links.values(): + for endpoint in ("source", "target"): + hardware = link.attrs["hardware"][endpoint] + component = scenario["components"][hardware["component"]] + expected = math.ceil(link.capacity / component["capacity"]) + assert hardware["count"] == expected, ( + link.source, + link.target, + link.capacity, + hardware, + ) + + +def test_directional_optics_are_resolved_for_each_endpoint(): + import pytest + + from topogen.naming import site_edge_id + from topogen.scenario.expansion import resolve_network + + graph = nx.MultiGraph() + graph.add_edge("a", "b", key="wan", base_capacity=2400) + edge_id = site_edge_id("a", "b", "wan") + scenario = { + "components": {"small": {"capacity": 800}, "large": {"capacity": 1600}}, + "network": { + "nodes": { + "a": {"attrs": {"role": "leaf"}}, + "b": {"attrs": {"role": "spine"}}, + }, + "links": [ + { + "source": "a", + "target": "b", + "capacity": 1, + "attrs": {"site_edge": edge_id}, + } + ], + }, + } + network = resolve_network( + graph, scenario, {"leaf->spine": "small", "spine->leaf": "large"} + ) + expected = { + "source": {"component": "small", "count": 3}, + "target": {"component": "large", "count": 2}, + } + assert graph["a"]["b"]["wan"]["hardware"] == expected + assert scenario["network"]["links"][0]["attrs"]["hardware"] == expected + assert next(iter(network.links.values())).attrs["hardware"] == expected + with pytest.raises(ValueError, match="no target-end optic mapping"): + resolve_network(graph, scenario, {"leaf->spine": "small"}) diff --git a/tests/test_audit_streamlining.py b/tests/test_audit_streamlining.py new file mode 100644 index 0000000..7faaacb --- /dev/null +++ b/tests/test_audit_streamlining.py @@ -0,0 +1,21 @@ +"""Malformed custom hardware and roles must not bypass scenario audits.""" + +import pytest +from ngraph import Network, Node + +from topogen.validation.audits.hw_capacity import check_node_hw_capacity +from topogen.validation.audits.node_role import check_node_roles + + +@pytest.mark.parametrize("role", [None, False, 12, ""]) +def test_role_must_be_a_nonempty_string(role): + network = Network() + network.add_node(Node("r", attrs={"role": role})) + assert check_node_roles(network) + + +@pytest.mark.parametrize("capacity", [float("nan"), float("inf"), -1]) +def test_invalid_chassis_capacity_is_reported(capacity): + network = Network() + network.add_node(Node("r", attrs={"hardware": {"component": "C", "count": 1}})) + assert check_node_hw_capacity(network, {"C": {"capacity": capacity}}) diff --git a/tests/test_build_integration.py b/tests/test_build_integration.py index 714bf17..f8d1ea0 100644 --- a/tests/test_build_integration.py +++ b/tests/test_build_integration.py @@ -8,7 +8,7 @@ from topogen.blueprints_lib import get_builtin_blueprints from topogen.config import TopologyConfig -from topogen.scenario_builder import build_scenario +from topogen.scenario import build_scenario from topogen.workflows_lib import get_builtin_workflows @@ -17,7 +17,7 @@ class TestBuildIntegration: @pytest.fixture def sample_integrated_graph(self): - graph = nx.Graph() + graph = nx.MultiGraph() denver = (100.0, 200.0) slc = (150.0, 250.0) @@ -31,15 +31,17 @@ def sample_integrated_graph(self): x=100.0, y=200.0, radius_km=50.0, + name_orig="Denver", ) graph.add_node( slc, - node_type="metro+highway", + node_type="metro", name="Salt Lake City", metro_id="metro_002", x=150.0, y=250.0, radius_km=40.0, + name_orig="Salt Lake City", ) graph.add_node( phoenix, @@ -49,13 +51,9 @@ def sample_integrated_graph(self): x=75.0, y=150.0, radius_km=45.0, + name_orig="Phoenix", ) - highway1 = (125.0, 225.0) - highway2 = (90.0, 180.0) - graph.add_node(highway1, node_type="highway") - graph.add_node(highway2, node_type="highway") - graph.add_edge(denver, slc, length_km=530.0, capacity=400, edge_type="corridor") graph.add_edge( denver, phoenix, length_km=890.0, capacity=400, edge_type="corridor" @@ -64,10 +62,6 @@ def sample_integrated_graph(self): slc, phoenix, length_km=650.0, capacity=400, edge_type="corridor" ) - graph.add_edge(denver, highway1, length_km=25.0) - graph.add_edge(highway1, slc, length_km=30.0) - graph.add_edge(phoenix, highway2, length_km=20.0) - return graph @pytest.fixture @@ -328,7 +322,7 @@ def test_empty_graph_handling(self): config.build.build_defaults.site_blueprint = "SingleRouter" config.build.build_overrides = {} - empty_graph = nx.Graph() + empty_graph = nx.MultiGraph() config.workflows.assignments.default = next( iter(get_builtin_workflows().keys()) ) @@ -361,7 +355,7 @@ def test_single_metro_graph(self): } } - graph = nx.Graph() + graph = nx.MultiGraph() metro = (100.0, 200.0) graph.add_node( metro, @@ -371,6 +365,7 @@ def test_single_metro_graph(self): x=100.0, y=200.0, radius_km=50.0, + name_orig="Denver", ) config.workflows.assignments.default = next( diff --git a/tests/test_cli.py b/tests/test_cli.py index c9c26ea..9a2cb3a 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -5,15 +5,18 @@ from __future__ import annotations -import builtins as py_builtins import importlib import logging from argparse import Namespace +from contextlib import ExitStack +from functools import partial from io import StringIO from pathlib import Path from types import SimpleNamespace from unittest.mock import patch +import pytest + def _invoke_main(argv: list[str], *, stub_subcommand: bool = False): """Run the CLI and capture exit code, stdout, selected command, and log level. @@ -27,65 +30,39 @@ def _invoke_main(argv: list[str], *, stub_subcommand: bool = False): called: dict[str, bool] = {"build": False, "generate": False, "info": False} level_holder: dict[str, int | None] = {"level": None} - def _noop_cmd(_args): # type: ignore[no-untyped-def] - print("noop") - - patchers = [] + def record(command, _args): + called[command] = True + print("stub command output") - patchers.append( - patch( - "topogen.log_config.set_global_log_level", - side_effect=lambda lvl: level_holder.__setitem__("level", lvl), + out = SimpleNamespace(code=0, stdout="", called=None, level=None) + with ( + ExitStack() as patches, + patch("sys.stdout", new_callable=StringIO) as buf, + patch("sys.argv", ["topogen"] + argv), + ): + patches.enter_context( + patch( + "topogen.log_config.set_global_log_level", + side_effect=lambda lvl: level_holder.__setitem__("level", lvl), + ) ) - ) - - if stub_subcommand: - patchers.extend( - [ - patch.object( - cli, - "build_command", - side_effect=lambda a: called.__setitem__("build", True), - ), - patch.object( - cli, - "generate_command", - side_effect=lambda a: called.__setitem__("generate", True), - ), - patch.object( - cli, - "info_command", - side_effect=lambda a: called.__setitem__("info", True), - ), - ] + if stub_subcommand: + for command in called: + patches.enter_context( + patch.object( + cli, f"{command}_command", side_effect=partial(record, command) + ) + ) + try: + cli.main() + except SystemExit as exc: + out.code = exc.code + out.stdout = buf.getvalue() + out.level = level_holder["level"] + out.called = next( + (name for name, was_called in called.items() if was_called), None ) - for p in patchers: - p.start() - - out = SimpleNamespace(code=0, stdout="", called=None, level=None) - saved_print = py_builtins.print - try: - with ( - patch("sys.stdout", new_callable=StringIO) as buf, - patch("sys.argv", ["topogen"] + argv), - ): - try: - cli.main() - except SystemExit as e: - out.code = int(getattr(e, "code", 0) or 0) - out.stdout = buf.getvalue() - out.level = level_holder["level"] - for name, was_called in called.items(): - if was_called: - out.called = name - break - finally: - # Restore global print in case --quiet modified it - py_builtins.print = saved_print - for p in reversed(patchers): - p.stop() - return out @@ -107,8 +84,13 @@ def test_default_log_level_is_info(): def test_quiet_suppresses_print_output(): - res = _invoke_main(["--quiet", "info", "-c", "config.yml"], stub_subcommand=True) + res = _invoke_main(["--quiet", "info", "config.yml"], stub_subcommand=True) + assert res.code == 0 + assert res.called == "info" assert res.stdout == "" + visible = _invoke_main(["info", "config.yml"], stub_subcommand=True) + assert visible.code == 0 + assert "stub command output" in visible.stdout def test_subcommand_dispatch_build_generate_info(): @@ -126,44 +108,33 @@ def test_timer_context_manager_success_and_error(): s = buf.getvalue() assert "Unit test op" in s - with patch("sys.stdout", new_callable=StringIO): - try: - with Timer("Failing op"): - raise RuntimeError("boom") - except RuntimeError: - pass + with ( + patch("sys.stdout", new_callable=StringIO) as buf, + pytest.raises(RuntimeError, match="boom"), + Timer("Failing op"), + ): + raise RuntimeError("boom") + assert "failed after" in buf.getvalue() def test__load_config_file_not_found_exits_with_code_2(tmp_path): from topogen.cli import _load_config - with patch("sys.stdout", new_callable=StringIO): - pass - missing = tmp_path / "does_not_exist.yml" - with patch( - "sys.exit", side_effect=lambda code=0: (_ for _ in ()).throw(SystemExit(code)) - ) as _: - try: - _load_config(missing) - except SystemExit as e: - assert int(e.code or 0) == 2 + with pytest.raises(SystemExit) as exc: + _load_config(missing) + assert exc.value.code == 2 def test__load_config_generic_error_exits_with_code_2(): import topogen.cli as cli - importlib.reload(cli) - - with patch.object(cli.TopologyConfig, "from_yaml", side_effect=ValueError("bad")): - with patch( - "sys.exit", - side_effect=lambda code=0: (_ for _ in ()).throw(SystemExit(code)), - ): - try: - cli._load_config(Path("config.yml")) # type: ignore[name-defined] - except SystemExit as e: # noqa: F841 - assert int(e.code or 0) == 2 + with ( + patch.object(cli.TopologyConfig, "from_yaml", side_effect=ValueError("bad")), + pytest.raises(SystemExit) as exc, + ): + cli._load_config(Path("config.yml")) + assert exc.value.code == 2 def test_build_command_success_print_and_non_print(): @@ -176,7 +147,12 @@ def test_build_command_success_print_and_non_print(): patch.object(cli, "_run_pipeline", return_value="YAML"), patch("sys.stdout", new_callable=StringIO) as buf, ): - args = Namespace(config="config.yml", output="config_scenario.yml", print=True) + args = Namespace( + config="config.yml", + output="config_scenario.yml", + print=True, + debug_dir=None, + ) cli.build_command(args) out = buf.getvalue() assert "GENERATED SCENARIO YAML" in out @@ -187,7 +163,12 @@ def test_build_command_success_print_and_non_print(): patch.object(cli, "_run_pipeline", return_value="YAML"), patch("sys.stdout", new_callable=StringIO) as buf, ): - args = Namespace(config="config.yml", output="config_scenario.yml", print=False) + args = Namespace( + config="config.yml", + output="config_scenario.yml", + print=False, + debug_dir=None, + ) cli.build_command(args) out = buf.getvalue() assert "SUCCESS! Generated topology" in out @@ -206,40 +187,26 @@ def test_build_command_failure_exit_codes(): ]: with ( patch.object(cli, "_run_pipeline", side_effect=exc), - patch( - "sys.exit", - side_effect=lambda code=0: (_ for _ in ()).throw(SystemExit(code)), - ) as _, + pytest.raises(SystemExit) as error, ): - try: - cli.build_command( - Namespace(config="c.yml", output="o.yaml", print=False) + cli.build_command( + Namespace( + config="c.yml", output="o.yaml", print=False, debug_dir=None ) - except SystemExit as e: - assert int(e.code or 0) == expected + ) + assert error.value.code == expected -def test__run_pipeline_missing_integrated_graph_exits_1(): +def test_run_pipeline_missing_integrated_graph(tmp_path): + from topogen import RunContext, TopologyConfig from topogen.cli import _run_pipeline - with patch( - "sys.exit", side_effect=lambda code=0: (_ for _ in ()).throw(SystemExit(code)) - ): - # Force the integrated graph sentinel to be treated as missing regardless of repo state - original_exists = Path.exists - - def fake_exists(self: Path) -> bool: # type: ignore[no-redef] - if str(self).endswith("_integrated_graph.json"): - return False - return original_exists(self) - - with patch("pathlib.Path.exists", new=fake_exists): - try: - # Provide a minimal TopologyConfig-like object with _source_path used by _run_pipeline - fake_cfg = Namespace(_source_path=Path("config.yml")) - _run_pipeline(fake_cfg, Path("config_scenario.yml"), print_yaml=False) # type: ignore[name-defined] - except SystemExit as e: - assert int(e.code or 0) == 1 + with pytest.raises(FileNotFoundError, match="Run topogen generate first"): + _run_pipeline( + TopologyConfig(), + tmp_path / "scenario.yml", + context=RunContext(tmp_path, "config"), + ) def test_generate_command_success_and_failure(): @@ -251,20 +218,15 @@ def test_generate_command_success_and_failure(): patch.object(cli, "_load_config", return_value=Namespace()), patch.object(cli, "_run_generation", return_value=None), ): - cli.generate_command(Namespace(config="config.yml")) + cli.generate_command(Namespace(config="config.yml", output=None)) with ( patch.object(cli, "_load_config", return_value=Namespace()), patch.object(cli, "_run_generation", side_effect=RuntimeError("boom")), - patch( - "sys.exit", - side_effect=lambda code=0: (_ for _ in ()).throw(SystemExit(code)), - ), + pytest.raises(SystemExit) as exc, ): - try: - cli.generate_command(Namespace(config="config.yml")) - except SystemExit as e: - assert int(e.code or 0) == 1 + cli.generate_command(Namespace(config="config.yml", output=None)) + assert exc.value.code == 1 def test_info_command_prints_status(tmp_path): @@ -278,7 +240,11 @@ def test_info_command_prints_status(tmp_path): # tiger intentionally missing fake_cfg = Namespace( - data_sources=Namespace(uac_polygons=str(uac), tiger_roads=str(tiger)), + data_sources=Namespace( + uac_polygons=str(uac), + tiger_roads=str(tiger), + conus_boundary=str(tmp_path / "boundary.zip"), + ), projection=Namespace(target_crs="EPSG:5070"), clustering=Namespace(metro_clusters=1), ) @@ -287,7 +253,7 @@ def test_info_command_prints_status(tmp_path): patch.object(cli, "_load_config", return_value=fake_cfg), patch("sys.stdout", new_callable=StringIO) as buf, ): - cli.info_command(Namespace(config="config.yml")) + cli.info_command(Namespace(config="config.yml", output=None)) out = buf.getvalue() assert "TopoGen Configuration" in out assert "UAC polygons:" in out diff --git a/tests/test_components_config.py b/tests/test_components_config.py index be39377..32a9ee6 100644 --- a/tests/test_components_config.py +++ b/tests/test_components_config.py @@ -1,182 +1,32 @@ -"""Tests for component configuration functionality.""" - -from pathlib import Path -from tempfile import NamedTemporaryFile +"""Supported component assignment settings and obsolete-key rejection.""" import pytest import yaml -from topogen.config import ComponentAssignment, ComponentsConfig, TopologyConfig - - -class TestComponentsConfig: - def test_component_assignment_defaults(self): - assignment = ComponentAssignment() - - assert assignment.hw_component == "" - assert assignment.optics == "" - - def test_component_assignment_with_values(self): - assignment = ComponentAssignment(hw_component="SpineChassis", optics="400G-LR4") - - assert assignment.hw_component == "SpineChassis" - assert assignment.optics == "400G-LR4" - - def test_components_config_defaults(self): - config = ComponentsConfig() - - assert isinstance(config.assignments.spine, ComponentAssignment) - assert isinstance(config.assignments.leaf, ComponentAssignment) - assert isinstance(config.assignments.core, ComponentAssignment) - - def test_parse_empty_components_config(self): - config_data = { - "data_sources": { - "uac_polygons": "test.zip", - "tiger_roads": "test.zip", - "conus_boundary": "test.zip", - }, - "projection": {"target_crs": "EPSG:5070"}, - "clustering": {"metro_clusters": 25}, - "highway_processing": {}, - "corridors": {}, - "validation": {}, - "output": {"scenario_metadata": {}, "formatting": {}}, - "components": {}, - } - - with NamedTemporaryFile(mode="w", suffix=".yml", delete=False) as f: - yaml.safe_dump(config_data, f) - config_path = Path(f.name) - - try: - config = TopologyConfig.from_yaml(config_path) - assert isinstance(config.components, ComponentsConfig) - finally: - config_path.unlink() - - def test_parse_components_ignores_library(self): - config_data = { - "data_sources": { - "uac_polygons": "test.zip", - "tiger_roads": "test.zip", - "conus_boundary": "test.zip", - }, - "projection": {"target_crs": "EPSG:5070"}, - "clustering": {"metro_clusters": 25}, - "highway_processing": {}, - "corridors": {}, - "validation": {}, - "output": {"scenario_metadata": {}, "formatting": {}}, - "components": {"library": {"CustomChassis": {"component_type": "chassis"}}}, - } - - with NamedTemporaryFile(mode="w", suffix=".yml", delete=False) as f: - yaml.safe_dump(config_data, f) - config_path = Path(f.name) - - try: - config = TopologyConfig.from_yaml(config_path) - assert hasattr(config.components, "assignments") - finally: - config_path.unlink() - - def test_parse_components_with_assignments(self): - config_data = { - "data_sources": { - "uac_polygons": "test.zip", - "tiger_roads": "test.zip", - "conus_boundary": "test.zip", - }, - "projection": {"target_crs": "EPSG:5070"}, - "clustering": {"metro_clusters": 25}, - "highway_processing": {}, - "corridors": {}, - "validation": {}, - "output": {"scenario_metadata": {}, "formatting": {}}, - "components": { - "assignments": { - "spine": {"hw_component": "SpineChassis", "optics": "400G-LR4"}, - "leaf": {"hw_component": "LeafChassis", "optics": "400G-SR8"}, - "core": {"hw_component": "CoreRouter", "optics": "100G-LR4"}, - } - }, - } - - with NamedTemporaryFile(mode="w", suffix=".yml", delete=False) as f: - yaml.safe_dump(config_data, f) - config_path = Path(f.name) - - try: - config = TopologyConfig.from_yaml(config_path) - - assert config.components.assignments.spine.hw_component == "SpineChassis" - assert config.components.assignments.spine.optics == "400G-LR4" - assert config.components.assignments.leaf.hw_component == "LeafChassis" - assert config.components.assignments.leaf.optics == "400G-SR8" - assert config.components.assignments.core.hw_component == "CoreRouter" - assert config.components.assignments.core.optics == "100G-LR4" - finally: - config_path.unlink() - - def test_parse_components_invalid_assignments_type(self): - config_data = { - "data_sources": { - "uac_polygons": "test.zip", - "tiger_roads": "test.zip", - "conus_boundary": "test.zip", - }, - "projection": {"target_crs": "EPSG:5070"}, - "clustering": {"metro_clusters": 25}, - "highway_processing": {}, - "corridors": {}, - "validation": {}, - "output": {"scenario_metadata": {}, "formatting": {}}, - "components": { - "assignments": "invalid_type" # Should be dict - }, - } +from topogen.config import ComponentsConfig, TopologyConfig - with NamedTemporaryFile(mode="w", suffix=".yml", delete=False) as f: - yaml.safe_dump(config_data, f) - config_path = Path(f.name) - try: - with pytest.raises( - ValueError, match="'components.assignments' must be a dictionary" - ): - TopologyConfig.from_yaml(config_path) - finally: - config_path.unlink() +def test_component_defaults(): + assert ComponentsConfig().hw_component == {} + assert ComponentsConfig().optics == {} - def test_parse_components_none_library(self): - config_yaml = """ -data_sources: - uac_polygons: test.zip - tiger_roads: test.zip - conus_boundary: test.zip -projection: - target_crs: EPSG:5070 -clustering: - metro_clusters: 25 -highway_processing: {} -corridors: {} -validation: {} -output: - scenario_metadata: {} - formatting: {} -components: - library: # This becomes None in YAML - assignments: {} -""" +def test_component_maps(tmp_path): + config = yaml.safe_load(open("examples/small_baseline.yml")) + config["components"] = { + "hw_component": {"core": "CoreRouter"}, + "optics": {"core->dc": "400G-LR4", "dc->core": "400G-LR4"}, + } + path = tmp_path / "config.yml" + path.write_text(yaml.safe_dump(config)) + actual = TopologyConfig.from_yaml(path).components + assert actual.hw_component == config["components"]["hw_component"] + assert actual.optics == config["components"]["optics"] - with NamedTemporaryFile(mode="w", suffix=".yml", delete=False) as f: - f.write(config_yaml) - config_path = Path(f.name) - try: - config = TopologyConfig.from_yaml(config_path) - assert hasattr(config.components, "assignments") - finally: - config_path.unlink() +@pytest.mark.parametrize("key", ["assignments", "library"]) +def test_obsolete_component_controls_rejected(key): + config = yaml.safe_load(open("examples/small_baseline.yml")) + config["components"] = {key: {}} + with pytest.raises(ValueError, match=key): + TopologyConfig._from_dict(config) diff --git a/tests/test_components_lib.py b/tests/test_components_lib.py index e7698a6..96fea0f 100644 --- a/tests/test_components_lib.py +++ b/tests/test_components_lib.py @@ -2,15 +2,7 @@ from __future__ import annotations -import pytest - -from topogen.components_lib import ( - get_builtin_component, - get_builtin_components, - get_components_by_role, - get_components_by_type, - list_builtin_component_names, -) +from topogen.components_lib import get_builtin_components class TestComponentsLib: @@ -28,72 +20,6 @@ def test_get_builtin_components(self): assert "capex" in comp assert "power_watts" in comp - def test_get_builtin_component_valid(self): - names = list_builtin_component_names() - assert names - component = get_builtin_component(names[0]) - - assert isinstance(component, dict) - assert isinstance(component.get("component_type"), str) - assert isinstance(component.get("description"), str) - assert isinstance(component.get("capex"), (int, float)) - assert isinstance(component.get("power_watts"), (int, float)) - - def test_get_builtin_component_invalid(self): - with pytest.raises(KeyError) as exc_info: - get_builtin_component("NonExistentComponent") - - assert "Component 'NonExistentComponent' not found" in str(exc_info.value) - - def test_list_builtin_component_names(self): - names = list_builtin_component_names() - - assert isinstance(names, list) - assert len(names) > 0 - assert all(isinstance(name, str) for name in names) - - def test_get_components_by_type_chassis(self): - chassis_components = get_components_by_type("chassis") - - assert isinstance(chassis_components, dict) - assert len(chassis_components) > 0 - - for _name, comp in chassis_components.items(): - assert comp["component_type"] == "chassis" - - def test_get_components_by_type_optic(self): - optic_components = get_components_by_type("optic") - - assert isinstance(optic_components, dict) - assert len(optic_components) > 0 - - for _name, comp in optic_components.items(): - assert comp["component_type"] == "optic" - - def test_get_components_by_type_nonexistent(self): - result = get_components_by_type("nonexistent") - assert result == {} - - def test_get_components_by_role_spine(self): - spine_components = get_components_by_role("spine") - - assert isinstance(spine_components, dict) - # Minimal library may not have explicit spine components - for _name, comp in spine_components.items(): - assert comp.get("attrs", {}).get("role") == "spine" - - def test_get_components_by_role_core(self): - core_components = get_components_by_role("core") - - assert isinstance(core_components, dict) - - for _name, comp in core_components.items(): - assert comp.get("attrs", {}).get("role") == "core" - - def test_get_components_by_role_nonexistent(self): - result = get_components_by_role("nonexistent") - assert result == {} - def test_component_structure_consistency(self): components = get_builtin_components() diff --git a/tests/test_components_scenario_builder.py b/tests/test_components_scenario_builder.py index 0409c6a..486d4e1 100644 --- a/tests/test_components_scenario_builder.py +++ b/tests/test_components_scenario_builder.py @@ -6,7 +6,7 @@ from topogen.config import ( TopologyConfig, ) -from topogen.scenario_builder import ( +from topogen.scenario.libraries import ( _build_blueprints_section, _build_components_section, ) @@ -22,10 +22,11 @@ def create_test_config(self) -> TopologyConfig: "dc": "CoreRouter", } config.components.optics = { - "core-core": "800G-ZR+", - # Add a couple to exercise inclusion; actual adjacency tests are elsewhere - "leaf-spine": "800G-DR4", - "spine-leaf": "1600G-2xDR4", + "core->core": "800G-ZR+", + "leaf->spine": "800G-DR4", + "spine->leaf": "800G-DR4", + "core->leaf": "1600G-2xDR4", + "leaf->core": "1600G-2xDR4", } return config @@ -33,7 +34,9 @@ def test_build_components_section_basic(self): config = self.create_test_config() used_blueprints = {"SingleRouter", "FullMesh4"} - components = _build_components_section(config, used_blueprints) + components = _build_components_section( + config, _build_blueprints_section(used_blueprints, config) + ) assert isinstance(components, dict) assert len(components) > 0 @@ -56,7 +59,9 @@ def test_build_components_section_with_role_assignments(self): assert clos_name is not None, "No Clos-style blueprint found" used_blueprints = {clos_name} - components = _build_components_section(config, used_blueprints) + components = _build_components_section( + config, _build_blueprints_section(used_blueprints, config) + ) assert isinstance(components, dict) assert "CoreRouter" in components @@ -77,7 +82,9 @@ def test_build_components_section_uses_merged_library(self): ) assert clos_name is not None, "No Clos-style blueprint found" used_blueprints = {clos_name} - components = _build_components_section(config, used_blueprints) + components = _build_components_section( + config, _build_blueprints_section(used_blueprints, config) + ) assert "CoreRouter" in components def test_build_components_section_missing_component_warning(self): @@ -85,10 +92,12 @@ def test_build_components_section_missing_component_warning(self): config.components.hw_component["spine"] = "NonExistentChassis" - used_blueprints = {"Clos_64_256"} + used_blueprints = {"SingleRouter"} - components = _build_components_section(config, used_blueprints) - assert isinstance(components, dict) + with pytest.raises(ValueError, match="Unknown components"): + _build_components_section( + config, _build_blueprints_section(used_blueprints, config) + ) def test_build_blueprints_section_basic(self): config = self.create_test_config() @@ -191,7 +200,9 @@ def test_components_section_includes_only_referenced(self): config = self.create_test_config() used_blueprints = {"SingleRouter"} - components = _build_components_section(config, used_blueprints) + components = _build_components_section( + config, _build_blueprints_section(used_blueprints, config) + ) assert "UnusedChassis1" not in components assert "UnusedChassis2" not in components @@ -204,7 +215,9 @@ def test_empty_used_blueprints(self): config = self.create_test_config() used_blueprints = set() - components = _build_components_section(config, used_blueprints) + components = _build_components_section( + config, _build_blueprints_section(used_blueprints, config) + ) blueprints = _build_blueprints_section(used_blueprints, config) assert isinstance(components, dict) diff --git a/tests/test_config.py b/tests/test_config.py index 87eeb66..e7dd194 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -27,9 +27,7 @@ def test_config_from_yaml(tmp_path: Path) -> None: "min_edge_length_km": 0.05, "snap_precision_m": 10.0, "highway_classes": ["S1100", "S1200"], - "min_cycle_nodes": 3, "filter_largest_component": True, - "validation_sample_size": 5, }, "validation": { "max_metro_highway_distance_km": 10.0, @@ -44,11 +42,6 @@ def test_config_from_yaml(tmp_path: Path) -> None: "max_corridor_distance_km": 1000.0, }, "output": { - "scenario_metadata": { - "title": "Test Topology", - "description": "Test description", - "version": "1.0", - }, "formatting": { "json_indent": 2, }, @@ -97,7 +90,7 @@ def test_export_clusters_config() -> None: "highway_processing": {}, "validation": {}, "corridors": {}, - "output": {"scenario_metadata": {}, "formatting": {}}, + "output": {"formatting": {}}, } from tempfile import NamedTemporaryFile as _NTF @@ -124,9 +117,8 @@ def test_dc_regions_config_defaults() -> None: assert config.build.build_defaults.dc_to_pop_link.cost == 1 assert config.build.build_defaults.dc_to_pop_link.attrs["link_type"] == "dc_to_pop" - assert hasattr(config.components.assignments, "dc") - assert config.components.assignments.dc.hw_component == "" - assert config.components.assignments.dc.optics == "" + assert config.components.hw_component == {} + assert config.components.optics == {} def test_config_validation_missing_files() -> None: @@ -152,14 +144,6 @@ def test_config_validation_invalid_params() -> None: config.validate() -def test_config_summary() -> None: - config = TopologyConfig() - summary = config.summary() - - assert "TOPOLOGY GENERATOR CONFIGURATION" in summary - assert "Metro Clusters: ~30" in summary - - def test_config_file_not_found() -> None: non_existent_path = Path("non_existent_config.yml") diff --git a/tests/test_config_streamlining.py b/tests/test_config_streamlining.py new file mode 100644 index 0000000..0d8afd0 --- /dev/null +++ b/tests/test_config_streamlining.py @@ -0,0 +1,47 @@ +"""Strict configuration must reject ambiguous and ignored options.""" + +from pathlib import Path + +import pytest +import yaml + +from topogen.config import TopologyConfig + + +def config_dict(): + return yaml.safe_load(Path("examples/small_baseline.yml").read_text()) + + +@pytest.mark.parametrize("key", [0.5, True]) +def test_fractional_or_boolean_priority_is_not_truncated(key): + data = config_dict() + data["traffic"]["priority_ratios"] = {key: 1} + with pytest.raises(ValueError): + TopologyConfig._from_dict(data) + + +def test_duplicate_normalized_priority_is_rejected(): + data = config_dict() + data["traffic"]["priority_ratios"] = {0: 0.5, "0": 1.0} + with pytest.raises(ValueError, match="Duplicate"): + TopologyConfig._from_dict(data) + + +def test_duplicate_metro_override_is_rejected(): + data = config_dict() + data["build"]["build_overrides"] = [ + {"metros": ["New York"], "pop_per_metro": 2}, + {"metros": ["new-york"], "pop_per_metro": 3}, + ] + with pytest.raises(ValueError, match="Duplicate"): + TopologyConfig._from_dict(data) + + +def test_unknown_striping_option_is_rejected(): + data = config_dict() + data["build"]["build_defaults"]["inter_metro_link"]["striping"] = { + "width": 1, + "widht": 2, + } + with pytest.raises(ValueError): + TopologyConfig._from_dict(data) diff --git a/tests/test_corridor_discovery.py b/tests/test_corridor_discovery.py index ebb1942..a4492ca 100644 --- a/tests/test_corridor_discovery.py +++ b/tests/test_corridor_discovery.py @@ -17,8 +17,18 @@ def test_basic_corridor_discovery(self): graph = nx.Graph() # Highway path: A --- B --- C - graph.add_edge((0.0, 0.0), (500.0, 0.0), length_km=500.0) - graph.add_edge((500.0, 0.0), (1000.0, 0.0), length_km=500.0) + graph.add_edge( + (0.0, 0.0), + (500.0, 0.0), + length_km=500.0, + geometry=[(0.0, 0.0), (500.0, 0.0)], + ) + graph.add_edge( + (500.0, 0.0), + (1000.0, 0.0), + length_km=500.0, + geometry=[(500.0, 0.0), (1000.0, 0.0)], + ) metros = [ MetroCluster("metro1", "metro-a", "Metro A", "001", 100.0, 0.0, 0.0, 25.0), @@ -47,6 +57,7 @@ def test_basic_corridor_discovery(self): anchor_point, edge_type="metro_anchor", length_km=0.0, + geometry=[metro.node_key, anchor_point], ) config = CorridorsConfig() @@ -71,8 +82,18 @@ def test_basic_corridor_discovery(self): def test_corridor_discovery_no_path(self): graph = nx.Graph() - graph.add_edge((0.0, 0.0), (100.0, 0.0), length_km=100.0) - graph.add_edge((2000.0, 0.0), (2100.0, 0.0), length_km=100.0) # Disconnected + graph.add_edge( + (0.0, 0.0), + (100.0, 0.0), + length_km=100.0, + geometry=[(0.0, 0.0), (100.0, 0.0)], + ) + graph.add_edge( + (2000.0, 0.0), + (2100.0, 0.0), + length_km=100.0, + geometry=[(2000.0, 0.0), (2100.0, 0.0)], + ) # Disconnected metros = [ MetroCluster("metro1", "metro-a", "Metro A", "001", 100.0, 0.0, 0.0, 25.0), @@ -98,6 +119,7 @@ def test_corridor_discovery_no_path(self): anchor_point, edge_type="metro_anchor", length_km=0.0, + geometry=[metro.node_key, anchor_point], ) config = CorridorsConfig() @@ -118,7 +140,7 @@ def test_corridor_distance_limiting(self): for i in range(10): start = (i * 500000.0, 0.0) # 500km intervals in EPSG:5070 meters end = ((i + 1) * 500000.0, 0.0) - graph.add_edge(start, end, length_km=500.0) + graph.add_edge(start, end, length_km=500.0, geometry=[start, end]) metros = [ MetroCluster("metro1", "metro-a", "Metro A", "001", 100.0, 0.0, 0.0, 25.0), @@ -144,6 +166,7 @@ def test_corridor_distance_limiting(self): anchor_point, edge_type="metro_anchor", length_km=0.0, + geometry=[metro.node_key, anchor_point], ) config = CorridorsConfig() @@ -159,12 +182,32 @@ def test_multiple_paths_discovery(self): graph = nx.Graph() # Path 1: A --- B --- D - graph.add_edge((0.0, 0.0), (500.0, 100.0), length_km=510.0) - graph.add_edge((500.0, 100.0), (1000.0, 0.0), length_km=510.0) + graph.add_edge( + (0.0, 0.0), + (500.0, 100.0), + length_km=510.0, + geometry=[(0.0, 0.0), (500.0, 100.0)], + ) + graph.add_edge( + (500.0, 100.0), + (1000.0, 0.0), + length_km=510.0, + geometry=[(500.0, 100.0), (1000.0, 0.0)], + ) # Path 2: A --- C --- D (shorter) - graph.add_edge((0.0, 0.0), (500.0, -100.0), length_km=510.0) - graph.add_edge((500.0, -100.0), (1000.0, 0.0), length_km=510.0) + graph.add_edge( + (0.0, 0.0), + (500.0, -100.0), + length_km=510.0, + geometry=[(0.0, 0.0), (500.0, -100.0)], + ) + graph.add_edge( + (500.0, -100.0), + (1000.0, 0.0), + length_km=510.0, + geometry=[(500.0, -100.0), (1000.0, 0.0)], + ) metros = [ MetroCluster("metro1", "metro-a", "Metro A", "001", 100.0, 0.0, 0.0, 25.0), @@ -190,6 +233,7 @@ def test_multiple_paths_discovery(self): anchor_point, edge_type="metro_anchor", length_km=0.0, + geometry=[metro.node_key, anchor_point], ) config = CorridorsConfig() @@ -220,8 +264,8 @@ def test_corridor_distance_equals_path_length(self): B = (60000.0, 80000.0) # detour up (arbitrary coords, units in meters) C = (100000.0, 0.0) - graph.add_edge(A, B, length_km=100.0) - graph.add_edge(B, C, length_km=100.0) + graph.add_edge(A, B, length_km=100.0, geometry=[A, B]) + graph.add_edge(B, C, length_km=100.0, geometry=[B, C]) metros = [ MetroCluster( @@ -250,6 +294,7 @@ def test_corridor_distance_equals_path_length(self): anchor_point, edge_type="metro_anchor", length_km=0.0, + geometry=[metro.node_key, anchor_point], ) config = CorridorsConfig() @@ -289,8 +334,8 @@ def test_path_length_filtering_over_euclidean(self): B = (400000.0, 300000.0) # arbitrary detour coordinate (meters) C = (800000.0, 0.0) - graph.add_edge(A, B, length_km=600.0) - graph.add_edge(B, C, length_km=600.0) + graph.add_edge(A, B, length_km=600.0, geometry=[A, B]) + graph.add_edge(B, C, length_km=600.0, geometry=[B, C]) metros = [ MetroCluster( @@ -319,6 +364,7 @@ def test_path_length_filtering_over_euclidean(self): anchor_point, edge_type="metro_anchor", length_km=0.0, + geometry=[metro.node_key, anchor_point], ) # Euclidean ~800 km; path = 1200 km @@ -349,6 +395,7 @@ def test_basic_corridor_graph_extraction(self): x=100.0, y=200.0, radius_km=25.0, + name_orig="metro-a", ) full_graph.add_node( metro2_coords, @@ -358,6 +405,7 @@ def test_basic_corridor_graph_extraction(self): x=200.0, y=300.0, radius_km=25.0, + name_orig="metro-b", ) full_graph.add_edge( @@ -373,6 +421,7 @@ def test_basic_corridor_graph_extraction(self): } ], risk_groups=["corridor_risk_metro-a_metro-b"], + geometry=[(150.0, 220.0), (180.0, 280.0)], ) metros = [ @@ -380,21 +429,16 @@ def test_basic_corridor_graph_extraction(self): MetroCluster("002", "metro-b", "Metro B", "002", 100.0, 200.0, 300.0, 25.0), ] - # Populate corridor path registry and tag edges with path id + # Populate the corridor path registry. path_id = ("001", "002", 0) full_graph.graph["corridor_paths"] = { path_id: CorridorPath( - metros=("001", "002"), - path_index=0, - nodes=[metro1_coords, metro2_coords], - edges=[(metro1_coords, metro2_coords)], + edges=[((150.0, 220.0), (180.0, 280.0))], segment_ids=[], length_km=141.4, geometry=[metro1_coords, metro2_coords], ) } - e = full_graph[(150.0, 220.0)][(180.0, 280.0)] - e["corridor_path_ids"] = {path_id} corridor_graph = extract_corridor_graph(full_graph, metros) @@ -404,7 +448,7 @@ def test_basic_corridor_graph_extraction(self): assert len(corridor_graph.edges) == 1 - edge_data = corridor_graph[metro1_coords][metro2_coords] + edge_data = corridor_graph[metro1_coords][metro2_coords][0] assert edge_data["edge_type"] == "corridor" assert edge_data["length_km"] == 141.4 assert edge_data["metro_a"] == "001" @@ -419,17 +463,25 @@ def test_basic_corridor_graph_extraction(self): # With length_km=141.4 and euclidean_km≈0.1414, detour ratio ≈ 1000 assert 900 < edge_data["detour_ratio"] < 1100 - def test_corridor_graph_aggregates_shortest_distance(self): + def test_corridor_graph_preserves_all_distances(self): full_graph = nx.Graph() metro1_coords = (0.0, 0.0) metro2_coords = (100.0, 100.0) full_graph.add_node( - metro1_coords, node_type="metro", name="metro1", metro_id="001" + metro1_coords, + node_type="metro", + name="metro1", + metro_id="001", + name_orig="metro1", ) full_graph.add_node( - metro2_coords, node_type="metro", name="metro2", metro_id="002" + metro2_coords, + node_type="metro", + name="metro2", + metro_id="002", + name_orig="metro2", ) full_graph.add_edge( @@ -466,31 +518,24 @@ def test_corridor_graph_aggregates_shortest_distance(self): pid_long = ("001", "002", 0) pid_short = ("001", "002", 1) reg[pid_long] = CorridorPath( - metros=("001", "002"), - path_index=0, - nodes=[metro1_coords, metro2_coords], edges=[((10.0, 10.0), (20.0, 20.0))], segment_ids=[], length_km=150.0, geometry=[metro1_coords, metro2_coords], ) reg[pid_short] = CorridorPath( - metros=("001", "002"), - path_index=1, - nodes=[metro1_coords, metro2_coords], edges=[((30.0, 30.0), (40.0, 40.0))], segment_ids=[], length_km=120.0, geometry=[metro1_coords, metro2_coords], ) full_graph.graph["corridor_paths"] = reg - full_graph[(10.0, 10.0)][(20.0, 20.0)]["corridor_path_ids"] = {pid_long} - full_graph[(30.0, 30.0)][(40.0, 40.0)]["corridor_path_ids"] = {pid_short} corridor_graph = extract_corridor_graph(full_graph, metros) - edge_data = corridor_graph[metro1_coords][metro2_coords] - assert edge_data["length_km"] == 120.0 # Shorter distance + edge_data = corridor_graph[metro1_coords][metro2_coords][0] + assert edge_data["length_km"] == 150.0 + assert corridor_graph[metro1_coords][metro2_coords][1]["length_km"] == 120.0 def test_corridor_graph_preserves_risk_groups(self): full_graph = nx.Graph() @@ -499,10 +544,18 @@ def test_corridor_graph_preserves_risk_groups(self): metro2_coords = (100.0, 100.0) full_graph.add_node( - metro1_coords, node_type="metro", name="metro1", metro_id="001" + metro1_coords, + node_type="metro", + name="metro1", + metro_id="001", + name_orig="metro1", ) full_graph.add_node( - metro2_coords, node_type="metro", name="metro2", metro_id="002" + metro2_coords, + node_type="metro", + name="metro2", + metro_id="002", + name_orig="metro2", ) full_graph.add_edge( @@ -540,21 +593,16 @@ def test_corridor_graph_preserves_risk_groups(self): pid = ("001", "002", 0) full_graph.graph["corridor_paths"] = { pid: CorridorPath( - metros=("001", "002"), - path_index=0, - nodes=[metro1_coords, metro2_coords], edges=[((10.0, 10.0), (20.0, 20.0)), ((30.0, 30.0), (40.0, 40.0))], segment_ids=[], length_km=100.0, geometry=[metro1_coords, metro2_coords], ) } - full_graph[(10.0, 10.0)][(20.0, 20.0)]["corridor_path_ids"] = {pid} - full_graph[(30.0, 30.0)][(40.0, 40.0)]["corridor_path_ids"] = {pid} corridor_graph = extract_corridor_graph(full_graph, metros) - edge_data = corridor_graph[metro1_coords][metro2_coords] + edge_data = corridor_graph[metro1_coords][metro2_coords][0] risk_groups = set(edge_data["risk_groups"]) expected_risk_groups = { "corridor_risk_metro1_metro2", diff --git a/tests/test_end_to_end_risk_groups.py b/tests/test_end_to_end_risk_groups.py index 4b03f37..b0568c5 100644 --- a/tests/test_end_to_end_risk_groups.py +++ b/tests/test_end_to_end_risk_groups.py @@ -18,7 +18,7 @@ assign_risk_groups as assign_risk_groups_to_corridors, ) from topogen.metro_clusters import MetroCluster -from topogen.scenario_builder import build_scenario +from topogen.scenario import build_scenario from topogen.workflows_lib import get_builtin_workflows @@ -27,12 +27,32 @@ def test_complete_risk_group_workflow(self): highway_graph = nx.Graph() # Highway backbone: A --- B --- C --- D - highway_graph.add_edge((0.0, 0.0), (500.0, 0.0), length_km=500.0) # A-B - highway_graph.add_edge((500.0, 0.0), (1000.0, 0.0), length_km=500.0) # B-C - highway_graph.add_edge((1000.0, 0.0), (1500.0, 0.0), length_km=500.0) # C-D + highway_graph.add_edge( + (0.0, 0.0), + (500.0, 0.0), + length_km=500.0, + geometry=[(0.0, 0.0), (500.0, 0.0)], + ) # A-B + highway_graph.add_edge( + (500.0, 0.0), + (1000.0, 0.0), + length_km=500.0, + geometry=[(500.0, 0.0), (1000.0, 0.0)], + ) # B-C + highway_graph.add_edge( + (1000.0, 0.0), + (1500.0, 0.0), + length_km=500.0, + geometry=[(1000.0, 0.0), (1500.0, 0.0)], + ) # C-D # Branch from B: B --- E - highway_graph.add_edge((500.0, 0.0), (500.0, 500.0), length_km=500.0) # B-E + highway_graph.add_edge( + (500.0, 0.0), + (500.0, 500.0), + length_km=500.0, + geometry=[(500.0, 0.0), (500.0, 500.0)], + ) # B-E metros = [ MetroCluster( @@ -145,9 +165,6 @@ def test_complete_risk_group_workflow(self): # metro1-metro2 via two edges total 1000km pid_12 = ("metro1", "metro2", 0) registry[pid_12] = CorridorPath( - metros=("metro1", "metro2"), - path_index=0, - nodes=[metros[0].node_key, metros[1].node_key], edges=[((0.0, 0.0), (500.0, 0.0)), ((500.0, 0.0), (1000.0, 0.0))], segment_ids=[], length_km=1000.0, @@ -156,9 +173,6 @@ def test_complete_risk_group_workflow(self): # metro2-metro3 via one edge 500km pid_23 = ("metro2", "metro3", 0) registry[pid_23] = CorridorPath( - metros=("metro2", "metro3"), - path_index=0, - nodes=[metros[1].node_key, metros[2].node_key], edges=[((1000.0, 0.0), (1500.0, 0.0))], segment_ids=[], length_km=500.0, @@ -167,22 +181,12 @@ def test_complete_risk_group_workflow(self): # metro2-metro4 via 2 edges ~707km pid_24 = ("metro2", "metro4", 0) registry[pid_24] = CorridorPath( - metros=("metro2", "metro4"), - path_index=0, - nodes=[metros[1].node_key, metros[3].node_key], edges=[((500.0, 0.0), (1000.0, 0.0)), ((500.0, 0.0), (500.0, 500.0))], segment_ids=[], length_km=707.0, geometry=[metros[1].node_key, metros[3].node_key], ) highway_graph.graph["corridor_paths"] = registry - highway_graph[(0.0, 0.0)][(500.0, 0.0)]["corridor_path_ids"] = {pid_12} - highway_graph[(500.0, 0.0)][(1000.0, 0.0)]["corridor_path_ids"] = { - pid_12, - pid_24, - } - highway_graph[(1000.0, 0.0)][(1500.0, 0.0)]["corridor_path_ids"] = {pid_23} - highway_graph[(500.0, 0.0)][(500.0, 500.0)]["corridor_path_ids"] = {pid_24} corridor_graph = extract_corridor_graph(highway_graph, metros) @@ -260,7 +264,10 @@ def test_metro_radius_exclusion_integration(self): highway_graph.add_edge(*close_edge, length_km=20.0) highway_graph.add_edge( - (1200000.0, 1000.0), (2000000.0, 1000.0), length_km=800.0 + (1200000.0, 1000.0), + (2000000.0, 1000.0), + length_km=800.0, + geometry=[(1200000.0, 1000.0), (2000000.0, 1000.0)], ) metros = [ @@ -281,6 +288,7 @@ def test_metro_radius_exclusion_integration(self): x=metro.centroid_x, y=metro.centroid_y, radius_km=metro.radius_km, + name_orig=metro.name, ) highway_graph[close_edge[0]][close_edge[1]]["corridor"] = [ @@ -322,19 +330,13 @@ def test_metro_radius_exclusion_integration(self): pid = ("metro1", "metro2", 0) highway_graph.graph["corridor_paths"] = { pid: CorridorPath( - metros=("metro1", "metro2"), - path_index=0, - nodes=[metros[0].node_key, metros[1].node_key], - edges=[(close_edge[0], close_edge[1])], + edges=[close_edge, ((1200000.0, 1000.0), (2000000.0, 1000.0))], segment_ids=[], length_km=1010.0, geometry=[metros[0].node_key, metros[1].node_key], ) } # Tag only far edge with path membership to keep its risks - highway_graph[(1200000.0, 1000.0)][(2000000.0, 1000.0)]["corridor_path_ids"] = { - pid - } corridor_graph = extract_corridor_graph(highway_graph, metros) @@ -395,6 +397,7 @@ def test_risk_group_naming_consistency_end_to_end(self): node_type="metro", name=metro.name, metro_id=metro.metro_id, + name_orig=metro.name, ) highway_graph.add_edge( @@ -454,45 +457,35 @@ def test_risk_group_naming_consistency_end_to_end(self): pid_23_1 = ("23527", "43912", 1) highway_graph.graph["corridor_paths"] = { pid_12_0: CorridorPath( - metros=("01171", "23527"), - path_index=0, - nodes=[metros[0].node_key, metros[1].node_key], edges=[((100000.0, 0.0), (900000.0, 0.0))], segment_ids=[], length_km=1000.0, geometry=[metros[0].node_key, metros[1].node_key], ), pid_12_1: CorridorPath( - metros=("01171", "23527"), - path_index=1, - nodes=[metros[0].node_key, metros[1].node_key], edges=[((500000.0, 0.0), (1500000.0, 0.0))], segment_ids=[], length_km=1000.0, geometry=[metros[0].node_key, metros[1].node_key], ), + ("23527", "43912", 0): CorridorPath( + edges=[((1100000.0, 0.0), (1900000.0, 0.0))], + segment_ids=[], + length_km=1000.0, + geometry=[metros[1].node_key, metros[2].node_key], + ), pid_23_1: CorridorPath( - metros=("23527", "43912"), - path_index=1, - nodes=[metros[1].node_key, metros[2].node_key], - edges=[((500000.0, 0.0), (1500000.0, 0.0))], + edges=[ + ((500000.0, 0.0), (1500000.0, 0.0)), + ((1100000.0, 0.0), (1900000.0, 0.0)), + ], segment_ids=[], length_km=1000.0, geometry=[metros[1].node_key, metros[2].node_key], ), } - highway_graph[(100000.0, 0.0)][(900000.0, 0.0)]["corridor_path_ids"] = { - pid_12_0 - } - highway_graph[(1100000.0, 0.0)][(1900000.0, 0.0)]["corridor_path_ids"] = { - pid_23_1 - } # Shared edge carries both second paths - highway_graph[(500000.0, 0.0)][(1500000.0, 0.0)]["corridor_path_ids"] = { - pid_12_1, - pid_23_1, - } corridor_graph = extract_corridor_graph(highway_graph, metros) @@ -526,6 +519,8 @@ def test_risk_group_naming_consistency_end_to_end(self): denver_kc_link = None for link in corridor_links: link_attrs = link["attrs"] + if not link_attrs["adjacency_id"].endswith(":path1"): + continue if ( link_attrs["source_metro"] == "denver-aurora" and link_attrs["target_metro"] == "kansas-city" diff --git a/tests/test_failure_policies_workflows_config.py b/tests/test_failure_policies_workflows_config.py index c5a412f..d07367f 100644 --- a/tests/test_failure_policies_workflows_config.py +++ b/tests/test_failure_policies_workflows_config.py @@ -1,331 +1,42 @@ -"""Tests for failure policies and workflows configuration parsing.""" - -import tempfile -from pathlib import Path -from typing import Any, Dict +"""Selection settings must represent behavior the pipeline actually applies.""" import pytest +import yaml -from topogen.config import ( - FailurePoliciesConfig, - FailurePolicyAssignments, - TopologyConfig, - WorkflowAssignments, - WorkflowsConfig, -) -from topogen.scenario.policies import _build_workflow_section - - -class TestFailurePolicyAssignments: - def test_default_values(self): - assignments = FailurePolicyAssignments() - assert assignments.default == "single_random_link_failure" - assert assignments.scenario_overrides == {} - - def test_custom_values(self): - overrides = {"test_scenario": {"failure_policy": "custom_policy"}} - assignments = FailurePolicyAssignments( - default="dual_random_link_failure", scenario_overrides=overrides - ) - assert assignments.default == "dual_random_link_failure" - assert assignments.scenario_overrides == overrides - - -class TestFailurePoliciesConfig: - def test_default_values(self): - config = FailurePoliciesConfig() - assert isinstance(config.assignments, FailurePolicyAssignments) - assert config.assignments.default == "single_random_link_failure" - - def test_custom_values(self): - assignments = FailurePolicyAssignments(default="custom_policy") - config = FailurePoliciesConfig(assignments=assignments) - assert config.assignments == assignments - - -class TestWorkflowAssignments: - def test_default_values(self): - assignments = WorkflowAssignments() - assert assignments.default == "design_analysis_brief" - assert assignments.scenario_overrides == {} - - def test_custom_values(self): - overrides = {"test_scenario": {"workflow": "custom_workflow"}} - assignments = WorkflowAssignments( - default="fast_network_analysis", scenario_overrides=overrides - ) - assert assignments.default == "fast_network_analysis" - assert assignments.scenario_overrides == overrides - - -class TestWorkflowsConfig: - def test_default_values(self): - config = WorkflowsConfig() - assert isinstance(config.assignments, WorkflowAssignments) - assert config.assignments.default == "design_analysis_brief" - - def test_custom_values(self): - assignments = WorkflowAssignments(default="custom_workflow") - config = WorkflowsConfig(assignments=assignments) - assert config.assignments == assignments - - -class TestConfigurationParsing: - def create_temp_config(self, config_dict: Dict[str, Any]) -> Path: - """Create a temporary configuration file.""" - import yaml - - # Add all required sections if not present - if "data_sources" not in config_dict: - config_dict["data_sources"] = { - "uac_polygons": "data/tl_2020_us_uac20.zip", - "tiger_roads": "data/tl_2024_us_primaryroads.zip", - "conus_boundary": "data/cb_2024_us_state_500k.zip", - } - - if "projection" not in config_dict: - config_dict["projection"] = {"target_crs": "EPSG:5070"} - - if "clustering" not in config_dict: - config_dict["clustering"] = { - "metro_clusters": 30, - "max_uac_radius_km": 100.0, - "export_clusters": False, - "export_integrated_graph": False, - "coordinate_precision": 1, - "area_precision": 2, - } - - if "highway_processing" not in config_dict: - config_dict["highway_processing"] = { - "min_edge_length_km": 0.05, - "snap_precision_m": 10.0, - "highway_classes": ["S1100", "S1200"], - "min_cycle_nodes": 3, - "filter_largest_component": True, - "validation_sample_size": 5, - } - - if "corridors" not in config_dict: - config_dict["corridors"] = { - "k_paths": 1, - "k_nearest": 3, - "max_edge_km": 600.0, - "max_corridor_distance_km": 1000.0, - "risk_groups": { - "enabled": True, - "group_prefix": "corridor_risk", - "exclude_metro_radius_shared": True, - }, - } - - if "validation" not in config_dict: - config_dict["validation"] = { - "max_metro_highway_distance_km": 10.0, - "require_connected": True, - "max_degree_threshold": 1000, - "high_degree_warning": 20, - "min_largest_component_fraction": 0.5, - } - - if "output" not in config_dict: - config_dict["output"] = { - "scenario_metadata": { - "title": "Continental US Backbone Topology", - "description": "Generated backbone topology based on population density and highway infrastructure", - "version": "1.0", - }, - "formatting": { - "json_indent": 2, - }, - } - - with tempfile.NamedTemporaryFile(mode="w", suffix=".yml", delete=False) as f: - yaml.safe_dump(config_dict, f) - return Path(f.name) - - def test_empty_failure_policies_section(self): - config_dict = {"failure_policies": {"assignments": {}}} - - config_path = self.create_temp_config(config_dict) - try: - config = TopologyConfig.from_yaml(config_path) - - assert isinstance(config.failure_policies, FailurePoliciesConfig) - assert ( - config.failure_policies.assignments.default - == "single_random_link_failure" - ) - finally: - config_path.unlink() +from topogen.config import TopologyConfig - def test_empty_workflows_section(self): - config_dict = {"workflows": {"assignments": {}}} - config_path = self.create_temp_config(config_dict) - try: - config = TopologyConfig.from_yaml(config_path) - - assert isinstance(config.workflows, WorkflowsConfig) - assert config.workflows.assignments.default == "design_analysis_brief" - assert _build_workflow_section(config)[0]["type"] == "NetworkStats" - finally: - config_path.unlink() - - def test_failure_policies_assignments_parsing(self): - config_dict = { - "failure_policies": { - "assignments": { - "default": "dual_random_link_failure", - "scenario_overrides": { - "test_scenario": {"failure_policy": "custom_policy"} - }, - } - } - } - - config_path = self.create_temp_config(config_dict) - try: - config = TopologyConfig.from_yaml(config_path) - - assert ( - config.failure_policies.assignments.default - == "dual_random_link_failure" - ) - assert ( - "test_scenario" - in config.failure_policies.assignments.scenario_overrides - ) - assert ( - config.failure_policies.assignments.scenario_overrides["test_scenario"][ - "failure_policy" - ] - == "custom_policy" - ) - finally: - config_path.unlink() - - def test_workflows_assignments_parsing(self): - config_dict = { - "workflows": { - "assignments": { - "default": "fast_network_analysis", - "scenario_overrides": { - "test_scenario": {"workflow": "custom_workflow"} - }, - } - } - } - - config_path = self.create_temp_config(config_dict) - try: - config = TopologyConfig.from_yaml(config_path) - - assert config.workflows.assignments.default == "fast_network_analysis" - assert "test_scenario" in config.workflows.assignments.scenario_overrides - assert ( - config.workflows.assignments.scenario_overrides["test_scenario"][ - "workflow" - ] - == "custom_workflow" - ) - finally: - config_path.unlink() - - def test_none_values_handling(self): - config_dict = { - "failure_policies": {"assignments": {"scenario_overrides": None}}, - "workflows": {"assignments": {"scenario_overrides": None}}, - } - - config_path = self.create_temp_config(config_dict) - try: - config = TopologyConfig.from_yaml(config_path) - - assert config.failure_policies.assignments.scenario_overrides == {} - assert config.workflows.assignments.scenario_overrides == {} - finally: - config_path.unlink() - - def test_missing_sections(self): - config_dict = {} - - config_path = self.create_temp_config(config_dict) - try: - config = TopologyConfig.from_yaml(config_path) - - assert isinstance(config.failure_policies, FailurePoliciesConfig) - assert ( - config.failure_policies.assignments.default - == "single_random_link_failure" - ) - assert isinstance(config.workflows, WorkflowsConfig) - assert config.workflows.assignments.default == "design_analysis_brief" - assert _build_workflow_section(config)[0]["type"] == "NetworkStats" - finally: - config_path.unlink() - - def test_invalid_failure_policies_type(self): - config_dict = {"failure_policies": "not a dict"} - - config_path = self.create_temp_config(config_dict) - try: - with pytest.raises( - ValueError, - match="'failure_policies' configuration section must be a dictionary", - ): - TopologyConfig.from_yaml(config_path) - finally: - config_path.unlink() - - def test_invalid_workflows_type(self): - config_dict = {"workflows": "not a dict"} - - config_path = self.create_temp_config(config_dict) - try: - with pytest.raises( - ValueError, - match="'workflows' configuration section must be a dictionary", - ): - TopologyConfig.from_yaml(config_path) - finally: - config_path.unlink() - - def test_invalid_library_type(self): - config_dict = {"failure_policies": {"assignments": "not a dict"}} - - config_path = self.create_temp_config(config_dict) - try: - with pytest.raises( - ValueError, match="'failure_policies.assignments' must be a dictionary" - ): - TopologyConfig.from_yaml(config_path) - finally: - config_path.unlink() - - def test_invalid_assignments_type(self): - config_dict = {"workflows": {"assignments": "not a dict"}} - - config_path = self.create_temp_config(config_dict) - try: - with pytest.raises( - ValueError, match="'workflows.assignments' must be a dictionary" - ): - TopologyConfig.from_yaml(config_path) - finally: - config_path.unlink() - - def test_complete_configuration(self): - config_dict = { - "failure_policies": {"assignments": {"default": "custom_failure"}}, - "workflows": {"assignments": {"default": "custom_workflow"}}, - } - - config_path = self.create_temp_config(config_dict) - try: - config = TopologyConfig.from_yaml(config_path) - - assert config.failure_policies.assignments.default == "custom_failure" - assert config.workflows.assignments.default == "custom_workflow" - finally: - config_path.unlink() +@pytest.mark.parametrize( + "section, default", + [ + ("failure_policies", "single_random_link_failure"), + ("workflows", "design_analysis_brief"), + ], +) +def test_policy_selection(section, default): + raw = yaml.safe_load(open("examples/small_baseline.yml")) + raw.pop(section, None) + assert ( + getattr(TopologyConfig._from_dict(raw), section).assignments.default == default + ) + raw[section] = {"assignments": {"default": "custom"}} + assert ( + getattr(TopologyConfig._from_dict(raw), section).assignments.default == "custom" + ) + + +@pytest.mark.parametrize("section", ["failure_policies", "workflows"]) +@pytest.mark.parametrize( + "value", + [ + "not a mapping", + {"assignments": "not a mapping"}, + {"assignments": {"scenario_overrides": {}}}, + {"library": {}}, + ], +) +def test_invalid_or_ignored_selection_controls_rejected(section, value): + raw = yaml.safe_load(open("examples/small_baseline.yml")) + raw[section] = value + with pytest.raises(ValueError, match=section): + TopologyConfig._from_dict(raw) diff --git a/tests/test_failure_policies_workflows_scenario_builder.py b/tests/test_failure_policies_workflows_scenario_builder.py index 2a99038..b811f9f 100644 --- a/tests/test_failure_policies_workflows_scenario_builder.py +++ b/tests/test_failure_policies_workflows_scenario_builder.py @@ -1,81 +1,74 @@ """Tests for failure policies and workflows in scenario builder.""" -from unittest.mock import Mock - import networkx as nx import pytest import yaml -from topogen.config import FailurePoliciesConfig, TopologyConfig, WorkflowsConfig -from topogen.scenario_builder import ( +from topogen.config import TopologyConfig +from topogen.scenario import build_scenario +from topogen.scenario.policies import ( _build_failure_policy_set_section, _build_workflow_section, - build_scenario, ) from topogen.workflows_lib import get_builtin_workflows class TestBuildFailurePolicySetSection: def test_default_policy_only(self): - config = Mock(spec=TopologyConfig) - config.failure_policies = Mock(spec=FailurePoliciesConfig) - config.failure_policies.assignments = Mock() + config = TopologyConfig() config.failure_policies.assignments.default = "single_random_link_failure" - result = _build_failure_policy_set_section(config) + result = _build_failure_policy_set_section( + config, _build_workflow_section(config) + ) assert isinstance(result, dict) assert "single_random_link_failure" in result assert "modes" in result["single_random_link_failure"] def test_custom_policy_in_library(self): - config = Mock(spec=TopologyConfig) - config.failure_policies = Mock(spec=FailurePoliciesConfig) - config.failure_policies.assignments = Mock() + config = TopologyConfig() config.failure_policies.assignments.default = "single_random_link_failure" - result = _build_failure_policy_set_section(config) + result = _build_failure_policy_set_section( + config, _build_workflow_section(config) + ) assert "single_random_link_failure" in result def test_custom_and_default_policies(self): - config = Mock(spec=TopologyConfig) - config.failure_policies = Mock(spec=FailurePoliciesConfig) - config.failure_policies.assignments = Mock() + config = TopologyConfig() config.failure_policies.assignments.default = "single_random_link_failure" - result = _build_failure_policy_set_section(config) + result = _build_failure_policy_set_section( + config, _build_workflow_section(config) + ) assert "single_random_link_failure" in result def test_custom_policy_overrides_builtin(self): - config = Mock(spec=TopologyConfig) - config.failure_policies = Mock(spec=FailurePoliciesConfig) - config.failure_policies.assignments = Mock() + config = TopologyConfig() config.failure_policies.assignments.default = "single_random_link_failure" - result = _build_failure_policy_set_section(config) + result = _build_failure_policy_set_section( + config, _build_workflow_section(config) + ) assert "single_random_link_failure" in result def test_unknown_default_policy(self): - config = Mock(spec=TopologyConfig) - config.failure_policies = Mock(spec=FailurePoliciesConfig) - config.failure_policies.library = {} - config.failure_policies.assignments = Mock() + config = TopologyConfig() config.failure_policies.assignments.default = "unknown_policy" with pytest.raises( ValueError, match="Default failure policy 'unknown_policy' not found" ): - _build_failure_policy_set_section(config) + _build_failure_policy_set_section(config, _build_workflow_section(config)) class TestBuildWorkflowSection: def test_default_workflow_builtin(self): - config = Mock(spec=TopologyConfig) - config.workflows = Mock(spec=WorkflowsConfig) - config.workflows.assignments = Mock() + config = TopologyConfig() # Pick any available built-in workflow to avoid brittle name pinning config.workflows.assignments.default = next( iter(get_builtin_workflows().keys()) @@ -89,9 +82,7 @@ def test_default_workflow_builtin(self): def test_custom_workflow_in_library(self): """The selected workflow is a list of steps from the merged library.""" - config = Mock(spec=TopologyConfig) - config.workflows = Mock(spec=WorkflowsConfig) - config.workflows.assignments = Mock() + config = TopologyConfig() config.workflows.assignments.default = next( iter(get_builtin_workflows().keys()) ) @@ -101,9 +92,7 @@ def test_custom_workflow_in_library(self): assert isinstance(result, list) def test_custom_workflow_priority(self): - config = Mock(spec=TopologyConfig) - config.workflows = Mock(spec=WorkflowsConfig) - config.workflows.assignments = Mock() + config = TopologyConfig() config.workflows.assignments.default = next( iter(get_builtin_workflows().keys()) ) @@ -113,9 +102,7 @@ def test_custom_workflow_priority(self): assert isinstance(result, list) def test_unknown_default_workflow(self): - config = Mock(spec=TopologyConfig) - config.workflows = Mock(spec=WorkflowsConfig) - config.workflows.assignments = Mock() + config = TopologyConfig() config.workflows.assignments.default = "unknown_workflow" with pytest.raises( @@ -126,64 +113,13 @@ def test_unknown_default_workflow(self): class TestBuildScenarioIntegration: def create_mock_config(self): - config = Mock(spec=TopologyConfig) - - config.build = Mock() - config.build.build_defaults = Mock() - config.build.build_defaults.pop_per_metro = 2 - config.build.build_defaults.site_blueprint = "SingleRouter" - config.build.build_defaults.dc_regions_per_metro = 2 - config.build.build_defaults.dc_region_blueprint = "DCRegion" - - config.build.build_defaults.intra_metro_link = Mock() - config.build.build_defaults.intra_metro_link.capacity = 400 - config.build.build_defaults.intra_metro_link.cost = 1 - config.build.build_defaults.intra_metro_link.attrs = { - "link_type": "intra_metro" - } - - config.build.build_defaults.inter_metro_link = Mock() - config.build.build_defaults.inter_metro_link.capacity = 100 - config.build.build_defaults.inter_metro_link.cost = 1 - config.build.build_defaults.inter_metro_link.attrs = { - "link_type": "inter_metro_corridor" - } - - config.build.build_defaults.dc_to_pop_link = Mock() - config.build.build_defaults.dc_to_pop_link.capacity = 400 - config.build.build_defaults.dc_to_pop_link.cost = 1 - config.build.build_defaults.dc_to_pop_link.attrs = {"link_type": "dc_to_pop"} - - config.build.build_overrides = {} - - config.components = Mock() - config.components.assignments = Mock() - config.components.assignments.spine = Mock(hw_component="", optics="") - config.components.assignments.leaf = Mock(hw_component="", optics="") - config.components.assignments.core = Mock(hw_component="", optics="") - config.components.assignments.dc = Mock(hw_component="", optics="") - - config.failure_policies = Mock(spec=FailurePoliciesConfig) - config.failure_policies.assignments = Mock() + config = TopologyConfig() config.failure_policies.assignments.default = "single_random_link_failure" - - config.workflows = Mock(spec=WorkflowsConfig) - config.workflows.assignments = Mock() - # Use first available workflow to avoid brittle expectations - config.workflows.assignments.default = next( - iter(get_builtin_workflows().keys()) - ) - - config.corridors = Mock() - config.corridors.risk_groups = Mock() - config.corridors.risk_groups.enabled = True - config.corridors.risk_groups.group_prefix = "corridor_risk" - config.corridors.risk_groups.exclude_metro_radius_shared = True - + config.workflows.assignments.default = next(iter(get_builtin_workflows())) return config def create_mock_graph(self): - graph = nx.Graph() + graph = nx.MultiGraph() graph.add_node( "metro_1", @@ -193,6 +129,7 @@ def create_mock_graph(self): x=100, y=200, radius_km=20.0, + name_orig="Metro 1", ) graph.add_node( "metro_2", @@ -202,6 +139,7 @@ def create_mock_graph(self): x=300, y=400, radius_km=25.0, + name_orig="Metro 2", ) graph.add_edge("metro_1", "metro_2", edge_type="corridor", length_km=500.5) diff --git a/tests/test_geo_utils.py b/tests/test_geo_utils.py deleted file mode 100644 index ae7acf0..0000000 --- a/tests/test_geo_utils.py +++ /dev/null @@ -1,97 +0,0 @@ -"""Tests for geographic utilities.""" - -import pytest -from shapely.geometry import Point - -from topogen.geo_utils import ( - CONUS_ALBERS, - WGS84, - bearing_to_offset, - get_transformer, - point_at_bearing, - transform_point, -) - - -def test_get_transformer() -> None: - transformer = get_transformer(WGS84, CONUS_ALBERS) - - # Test transformation of a known point (Chicago) - chicago_lon, chicago_lat = -87.6298, 41.8781 - x, y = transformer.transform(chicago_lon, chicago_lat) - - assert isinstance(x, float) - assert isinstance(y, float) - assert abs(x) > 1000 - assert abs(y) > 1000 - - -def test_transform_point() -> None: - # Chicago coordinates - chicago_wgs84 = Point(-87.6298, 41.8781) - - chicago_albers = transform_point(chicago_wgs84, WGS84, CONUS_ALBERS) - - assert chicago_albers.x != chicago_wgs84.x - assert chicago_albers.y != chicago_wgs84.y - - assert abs(chicago_albers.x) > 1000 - assert abs(chicago_albers.y) > 1000 - - -def test_bearing_to_offset() -> None: - # North (0 degrees) - x, y = bearing_to_offset(0, 1000) - assert abs(x) < 1e-10 - assert abs(y - 1000) < 1e-10 - - # East (90 degrees) - x, y = bearing_to_offset(90, 1000) - assert abs(x - 1000) < 1e-10 - assert abs(y) < 1e-10 - - # South (180 degrees) - x, y = bearing_to_offset(180, 1000) - assert abs(x) < 1e-10 - assert abs(y + 1000) < 1e-10 - - # West (270 degrees) - x, y = bearing_to_offset(270, 1000) - assert abs(x + 1000) < 1e-10 - assert abs(y) < 1e-10 - - -def test_point_at_bearing() -> None: - center = Point(0, 0) - - # Point 1000m north - north_point = point_at_bearing(center, 0, 1000) - assert abs(north_point.x) < 1e-10 - assert abs(north_point.y - 1000) < 1e-10 - - # Point 1000m east - east_point = point_at_bearing(center, 90, 1000) - assert abs(east_point.x - 1000) < 1e-10 - assert abs(east_point.y) < 1e-10 - - -@pytest.mark.parametrize( - "bearing,expected_x,expected_y", - [ - (0, 0, 1), # North - (45, 0.707, 0.707), # Northeast (approximate) - (90, 1, 0), # East - (135, 0.707, -0.707), # Southeast - (180, 0, -1), # South - (225, -0.707, -0.707), # Southwest - (270, -1, 0), # West - (315, -0.707, 0.707), # Northwest - ], -) -def test_bearing_directions( - bearing: float, expected_x: float, expected_y: float -) -> None: - x, y = bearing_to_offset(bearing, 1.0) - - assert abs(x - expected_x) < 0.01 - assert abs(y - expected_y) < 0.01 diff --git a/tests/test_geographic_contract.py b/tests/test_geographic_contract.py new file mode 100644 index 0000000..79cc87e --- /dev/null +++ b/tests/test_geographic_contract.py @@ -0,0 +1,125 @@ +"""Geographic paths retain identity, length, geometry, and shared risks.""" + +import networkx as nx +import pytest + +from topogen.blueprints_lib import get_builtin_blueprints +from topogen.config import ( + CorridorsConfig, + FormattingConfig, + RiskGroupsConfig, + TopologyConfig, +) +from topogen.corridors import add_corridors, assign_risk_groups, extract_corridor_graph +from topogen.integrated_graph import ( + _contract_degree2_chains, + load_from_json, + save_to_json, +) +from topogen.metro_clusters import MetroCluster +from topogen.scenario.config import _determine_metro_settings +from topogen.scenario.graph_pipeline import build_site_graph +from topogen.scenario.network import _extract_metros_from_graph +from topogen.scenario.risk import _build_risk_groups_section + + +def metro(identifier, x): + return MetroCluster( + identifier, identifier, identifier, identifier, 1.0, x, 0.0, 0.0 + ) + + +def add_edge(graph, a, b, length): + graph.add_edge(a, b, length_km=length, geometry=[a, b], segment_id=f"{a}:{b}") + + +def test_nearest_neighbor_union_does_not_depend_on_id_order(): + metros = [metro("a", 0), metro("b", 1000), metro("z", 4000)] + graph = nx.Graph() + add_edge(graph, metros[0].node_key, metros[1].node_key, 1) + add_edge(graph, metros[1].node_key, metros[2].node_key, 3) + add_corridors(graph, metros, CorridorsConfig(k_nearest=1)) + assert set(graph.graph["corridor_paths"]) == {("a", "b", 0), ("b", "z", 0)} + + +def test_contraction_preserves_parallel_routes(): + graph = nx.Graph() + a, b = (0.0, 0.0), (4000.0, 0.0) + for y, length in [(1000, 2), (2000, 3), (3000, 4)]: + mid = (2000.0, float(y)) + add_edge(graph, a, mid, length / 2) + add_edge(graph, mid, b, length / 2) + actual = _contract_degree2_chains(graph, {a, b}) + paths = list(nx.shortest_simple_paths(actual, a, b, weight="length_km")) + assert [ + sum(actual[u][v]["length_km"] for u, v in zip(p, p[1:], strict=False)) + for p in paths + ] == [2, 3, 4] + for u, v, data in actual.edges(data=True): + assert {tuple(data["geometry"][0]), tuple(data["geometry"][-1])} == {u, v} + + +def test_parallel_corridors_survive_json_and_site_expansion(tmp_path): + metros = [metro("a", 0), metro("b", 4000)] + a, b = (m.node_key for m in metros) + graph = nx.Graph() + for y, length in [(1000, 5), (2000, 6)]: + mid = (2000.0, float(y)) + add_edge(graph, a, mid, length / 2) + add_edge(graph, mid, b, length / 2) + corridors = CorridorsConfig( + k_paths=2, + k_nearest=1, + risk_groups=RiskGroupsConfig(exclude_metro_radius_shared=False), + ) + add_corridors(graph, metros, corridors) + assign_risk_groups(graph, metros, corridors) + extracted = extract_corridor_graph(graph, metros) + assert extracted.is_multigraph() + assert extracted.number_of_edges(a, b) == 2 + assert sorted(d["risk_groups"] for _, _, d in extracted.edges(data=True)) == [ + ["corridor_risk_a_b"], + ["corridor_risk_a_b_path1"], + ] + path = tmp_path / "corridors.json" + save_to_json(extracted, path, "EPSG:5070", FormattingConfig()) + loaded, crs = load_from_json(path) + assert crs == "EPSG:5070" + assert dict(loaded.edges) == dict(extracted.edges) + config = TopologyConfig() + config.build.build_defaults.pop_per_metro = 1 + config.build.build_defaults.dc_regions_per_metro = 0 + selected = _extract_metros_from_graph(loaded) + sites = build_site_graph( + selected, + _determine_metro_settings(selected, config), + loaded, + get_builtin_blueprints(), + ) + wan = [ + d + for _, _, d in sites.edges(data=True) + if d["link_type"] == "inter_metro_corridor" + ] + assert len(wan) == 2 + assert {d["cost"] for d in wan} == {5, 6} + + assert [ + risk["attrs"]["distance_km"] + for risk in _build_risk_groups_section(loaded, config) + ] == [5, 6] + + +def test_component_filter_reports_removed_metro_anchor(monkeypatch): + from topogen import integrated_graph as pipeline + + metros = [metro("main", 0), metro("island", 100_000)] + roads = nx.Graph() + for left, right in ((0, 1000), (1000, 2000), (2000, 0), (100_000, 101_000)): + add_edge(roads, (float(left), 0.0), (float(right), 0.0), 1.0) + monkeypatch.setattr(pipeline, "load_metro_clusters", lambda **_kwargs: metros) + monkeypatch.setattr(pipeline, "build_highway_graph", lambda **_kwargs: roads) + config = TopologyConfig() + config.highway_processing.min_edge_length_km = 0 + with pytest.raises(ValueError, match="Highway filtering removed.*island"): + pipeline.build_integrated_graph(config) diff --git a/tests/test_highway_graph.py b/tests/test_highway_graph.py index 3120be8..49b0d5c 100644 --- a/tests/test_highway_graph.py +++ b/tests/test_highway_graph.py @@ -47,7 +47,7 @@ def test_grid_snapping_precision(self): ), # Should snap to (0.0, 0.0), (20.0, 0.0) LineString( [(4.0, 6.0), (16.0, 4.0)] - ), # Should snap to (0.0, 0.0), (20.0, 0.0) + ), # Should snap to (0.0, 10.0), (20.0, 0.0) ] edges = list(_iter_snapped_edges(lines, snap_m=10.0)) @@ -88,8 +88,7 @@ def test_build_intersection_graph(self): # Node at (100, 0) should have degree 3 (junction) junction_node = (100.0, 0.0) - if junction_node in G.nodes: - assert len(list(G.neighbors(junction_node))) == 3 + assert G.degree[junction_node] == 3 def test_segment_length_calculation_horizontal_vertical(self): """Segments should have accurate length_km for axis-aligned geometries.""" diff --git a/tests/test_integrated_graph.py b/tests/test_integrated_graph.py index a4af397..67aaaaa 100644 --- a/tests/test_integrated_graph.py +++ b/tests/test_integrated_graph.py @@ -42,11 +42,14 @@ def test_contract_isolated_cycle(self): contracted = _contract_degree2_chains(G) - assert len(contracted.nodes) == 2 - assert len(contracted.edges) == 1 - - edge_data = list(contracted.edges(data=True))[0][2] - assert edge_data["length_km"] == 400.0 # Sum of cycle + assert nx.is_isomorphic(contracted, G) + assert sum(d["length_km"] for _, _, d in contracted.edges(data=True)) == 400.0 + assert ( + nx.shortest_path_length( + contracted, (0.0, 0.0), (100.0, 100.0), weight="length_km" + ) + == 200.0 + ) def test_contract_mixed_topology(self): G = nx.Graph() @@ -122,8 +125,7 @@ def test_keep_largest_component(self): # Small isolated component G.add_edge((100.0, 100.0), (101.0, 100.0), length_km=1.0) - validation_config = ValidationConfig() - largest = _keep_largest_component(G, validation_config) + largest = _keep_largest_component(G) assert len(largest.nodes) == 4 assert len(largest.edges) == 3 @@ -135,8 +137,7 @@ def test_already_connected_graph(self): G.add_edge((0.0, 0.0), (1.0, 0.0), length_km=1.0) G.add_edge((1.0, 0.0), (2.0, 0.0), length_km=1.0) - validation_config = ValidationConfig() - result = _keep_largest_component(G, validation_config) + result = _keep_largest_component(G) assert len(result.nodes) == len(G.nodes) assert len(result.edges) == len(G.edges) @@ -196,10 +197,20 @@ def fake_extract_corridor_graph(full_graph: nx.Graph, metros): a = (1000.0, 2000.0) b = (3000.0, 4000.0) cg.add_node( - a, node_type="metro", metro_id="001", name="metro-a", radius_km=10.0 + a, + node_type="metro", + metro_id="001", + name="metro-a", + radius_km=10.0, + name_orig="metro-a", ) cg.add_node( - b, node_type="metro", metro_id="002", name="metro-b", radius_km=20.0 + b, + node_type="metro", + metro_id="002", + name="metro-b", + radius_km=20.0, + name_orig="metro-b", ) cg.add_edge(a, b, edge_type="corridor", length_km=10.0) return cg diff --git a/tests/test_metro_clusters.py b/tests/test_metro_clusters.py index 71c64de..c7809e4 100644 --- a/tests/test_metro_clusters.py +++ b/tests/test_metro_clusters.py @@ -2,16 +2,18 @@ import math import tempfile +from dataclasses import asdict from pathlib import Path from unittest.mock import patch import geopandas as gpd -import numpy as np import pandas as pd import pytest +import yaml from shapely.geometry import Polygon -from topogen.config import ClusteringConfig, FormattingConfig +from topogen.config import ClusteringConfig +from topogen.context import RunContext from topogen.metro_clusters import MetroCluster, load_metro_clusters @@ -29,19 +31,6 @@ def sample_cluster(self): radius_km=53.4, ) - @pytest.fixture - def another_cluster(self): - return MetroCluster( - metro_id="51445", - name="los-angeles--long-beach--anaheim", - name_orig="Los Angeles--Long Beach--Anaheim, CA", - uac_code="51445", - land_area_km2=4850.3, - centroid_x=-2056789.1, - centroid_y=3926233.5, - radius_km=39.3, - ) - def test_cluster_creation(self, sample_cluster): assert sample_cluster.metro_id == "63217" assert sample_cluster.name == "new-york--newark" @@ -62,61 +51,6 @@ def test_coordinates_property(self, sample_cluster): assert isinstance(coords, tuple) assert len(coords) == 2 - def test_coordinates_array_property(self, sample_cluster): - coords_array = sample_cluster.coordinates_array - assert isinstance(coords_array, np.ndarray) - assert coords_array.shape == (2,) - np.testing.assert_array_equal(coords_array, [586515.2, 4506242.8]) - - def test_distance_to(self, sample_cluster, another_cluster): - distance = sample_cluster.distance_to(another_cluster) - - # Calculate expected distance manually - dx = 586515.2 - (-2056789.1) - dy = 4506242.8 - 3926233.5 - expected = math.sqrt(dx * dx + dy * dy) - - assert abs(distance - expected) < 1e-6 - - def test_distance_to_same_cluster(self, sample_cluster): - distance = sample_cluster.distance_to(sample_cluster) - assert distance == 0.0 - - def test_overlaps_with_far_clusters(self, sample_cluster, another_cluster): - # These clusters are far apart and should not overlap - assert not sample_cluster.overlaps_with(another_cluster) - assert not another_cluster.overlaps_with(sample_cluster) - - def test_overlaps_with_close_clusters(self): - cluster1 = MetroCluster( - metro_id="12345", - name="cluster-1", - name_orig="Cluster 1", - uac_code="12345", - land_area_km2=100.0, - centroid_x=0.0, - centroid_y=0.0, - radius_km=10.0, - ) - - cluster2 = MetroCluster( - metro_id="67890", - name="cluster-2", - name_orig="Cluster 2", - uac_code="67890", - land_area_km2=200.0, - centroid_x=15000.0, # 15 km away - centroid_y=0.0, - radius_km=10.0, - ) - - # Clusters with 20km combined radius and 15km separation should overlap - assert cluster1.overlaps_with(cluster2) - assert cluster2.overlaps_with(cluster1) - - def test_overlaps_with_same_cluster(self, sample_cluster): - assert sample_cluster.overlaps_with(sample_cluster) - class TestLoadMetroClusters: @pytest.fixture @@ -180,17 +114,18 @@ def temp_uac_file(self, mock_uac_data): def test_load_basic_functionality(self, temp_uac_file, mock_uac_data): clustering_config = ClusteringConfig(max_uac_radius_km=100.0) - formatting_config = FormattingConfig() + clusters = load_metro_clusters( uac_path=temp_uac_file, k=3, target_crs="EPSG:3857", clustering_config=clustering_config, - formatting_config=formatting_config, ) assert len(clusters) == 3 assert all(isinstance(c, MetroCluster) for c in clusters) + # The direct generate -> build API must emit ordinary YAML scalars. + yaml.safe_dump([asdict(cluster) for cluster in clusters]) for i in range(len(clusters)): expected_uace = f"UAC{i + 1:02d}" @@ -202,14 +137,13 @@ def test_load_basic_functionality(self, temp_uac_file, mock_uac_data): def test_load_insufficient_areas(self, temp_uac_file): clustering_config = ClusteringConfig() - formatting_config = FormattingConfig() + with pytest.raises(ValueError, match="Only 5 urban areas available"): load_metro_clusters( uac_path=temp_uac_file, k=10, # More than the 5 available target_crs="EPSG:3857", clustering_config=clustering_config, - formatting_config=formatting_config, ) def test_override_selects_largest_match_with_non_numeric_index( @@ -225,7 +159,6 @@ def test_override_selects_largest_match_with_non_numeric_index( clustering_config=ClusteringConfig( max_uac_radius_km=100.0, override_metro_clusters=["Urban Area"] ), - formatting_config=FormattingConfig(), ) assert [cluster.uac_code for cluster in clusters] == ["UAC01"] @@ -233,25 +166,23 @@ def test_load_file_not_found(self): nonexistent_path = Path("nonexistent_file.zip") clustering_config = ClusteringConfig() - formatting_config = FormattingConfig() + with pytest.raises(FileNotFoundError, match="UAC file not found"): load_metro_clusters( uac_path=nonexistent_path, k=3, target_crs="EPSG:3857", clustering_config=clustering_config, - formatting_config=formatting_config, ) def test_radius_calculation(self, temp_uac_file): clustering_config = ClusteringConfig(max_uac_radius_km=100.0) - formatting_config = FormattingConfig() + clusters = load_metro_clusters( uac_path=temp_uac_file, k=2, target_crs="EPSG:3857", clustering_config=clustering_config, - formatting_config=formatting_config, ) expected_radius_1 = math.sqrt(50 / math.pi) # r = sqrt(area/π) @@ -262,13 +193,12 @@ def test_radius_calculation(self, temp_uac_file): def test_numpy_array_indexing(self, temp_uac_file): clustering_config = ClusteringConfig(max_uac_radius_km=100.0) - formatting_config = FormattingConfig() + clusters = load_metro_clusters( uac_path=temp_uac_file, k=1, target_crs="EPSG:3857", clustering_config=clustering_config, - formatting_config=formatting_config, ) assert isinstance(clusters[0].radius_km, float) @@ -276,54 +206,53 @@ def test_numpy_array_indexing(self, temp_uac_file): def test_radius_capping(self, temp_uac_file): clustering_config = ClusteringConfig(max_uac_radius_km=2.0) # Very small cap - formatting_config = FormattingConfig() + clusters = load_metro_clusters( uac_path=temp_uac_file, k=1, target_crs="EPSG:3857", clustering_config=clustering_config, - formatting_config=formatting_config, ) assert clusters[0].radius_km == 2.0 @patch("topogen.metro_clusters._export_cluster_files") - def test_export_files_flag(self, mock_export, temp_uac_file): + def test_export_flag_without_context_does_not_write( + self, mock_export, temp_uac_file + ): clustering_config = ClusteringConfig(export_clusters=True) - formatting_config = FormattingConfig() + clusters = load_metro_clusters( uac_path=temp_uac_file, k=2, target_crs="EPSG:3857", clustering_config=clustering_config, - formatting_config=formatting_config, ) - mock_export.assert_called_once_with(clusters, "EPSG:3857") + assert len(clusters) == 2 + mock_export.assert_not_called() @patch("topogen.metro_clusters._export_cluster_files") def test_no_export_by_default(self, mock_export, temp_uac_file): clustering_config = ClusteringConfig(export_clusters=False) - formatting_config = FormattingConfig() + load_metro_clusters( uac_path=temp_uac_file, k=2, target_crs="EPSG:3857", clustering_config=clustering_config, - formatting_config=formatting_config, ) mock_export.assert_not_called() def test_coordinate_precision(self, temp_uac_file): clustering_config = ClusteringConfig() - formatting_config = FormattingConfig() + clusters = load_metro_clusters( uac_path=temp_uac_file, k=1, target_crs="EPSG:3857", clustering_config=clustering_config, - formatting_config=formatting_config, ) cluster = clusters[0] @@ -351,18 +280,17 @@ def test_conus_filtering(self, mock_conus_mask, temp_uac_file, mock_uac_data): mock_intersects.return_value = pd.Series([True, True, True, False, False]) clustering_config = ClusteringConfig() - formatting_config = FormattingConfig() + clusters = load_metro_clusters( uac_path=temp_uac_file, k=2, target_crs="EPSG:3857", clustering_config=clustering_config, - formatting_config=formatting_config, conus_boundary_path=boundary_path, ) assert len(clusters) == 2 - mock_conus_mask.assert_called_once_with(boundary_path, "EPSG:4269") + mock_conus_mask.assert_called_once_with(boundary_path, "EPSG:4326") @patch("topogen.geo_utils.create_conus_mask") def test_conus_filtering_multiple_geometries(self, mock_conus_mask, temp_uac_file): @@ -381,13 +309,12 @@ def test_conus_filtering_multiple_geometries(self, mock_conus_mask, temp_uac_fil mock_intersects.return_value = pd.Series([True, True, False, False, False]) clustering_config = ClusteringConfig() - formatting_config = FormattingConfig() + clusters = load_metro_clusters( uac_path=temp_uac_file, k=2, target_crs="EPSG:3857", clustering_config=clustering_config, - formatting_config=formatting_config, conus_boundary_path=boundary_path, ) @@ -461,7 +388,12 @@ def test_export_files_basic(self, mock_export_map, sample_clusters): mock_path_instance.mkdir(parents=True, exist_ok=True) with patch.object(gpd.GeoDataFrame, "to_file") as mock_to_file: - _export_cluster_files(sample_clusters, "EPSG:3857") + _export_cluster_files( + sample_clusters, + "EPSG:3857", + RunContext(Path(temp_dir), "test"), + Path("boundaries.zip"), + ) mock_to_file.assert_called_once() @@ -483,6 +415,26 @@ def test_export_visualization_error_handling( with patch.object(gpd.GeoDataFrame, "to_file"): with pytest.raises(Exception, match="Mock error"): - _export_cluster_files(sample_clusters, "EPSG:3857") + _export_cluster_files( + sample_clusters, + "EPSG:3857", + RunContext(Path(temp_dir), "test"), + Path("boundaries.zip"), + ) mock_export_map.assert_called_once() + + +def test_unmatched_override_does_not_fall_back_to_largest(tmp_path, monkeypatch): + path = tmp_path / "metros.zip" + path.touch() + frame = gpd.GeoDataFrame( + {"NAME20": ["Actual Metro"], "UACE20": ["1"], "ALAND20": [1000]}, + geometry=[Polygon([(0, 0), (0, 1), (1, 1), (0, 0)])], + crs="EPSG:5070", + ) + monkeypatch.setattr(gpd, "read_file", lambda *a, **kw: frame) + with pytest.raises(ValueError, match="matched no metros"): + load_metro_clusters( + path, 1, "EPSG:5070", ClusteringConfig(override_metro_clusters=["Typo"]) + ) diff --git a/tests/test_review_regressions.py b/tests/test_review_regressions.py new file mode 100644 index 0000000..e6eec2b --- /dev/null +++ b/tests/test_review_regressions.py @@ -0,0 +1,232 @@ +"""Behavioral witnesses found during the post-cleanup review.""" + +from collections import defaultdict +from copy import deepcopy + +import networkx as nx +import pytest +import yaml + +from topogen.config import TopologyConfig, _normalize_int +from topogen.scenario import build_scenario +from topogen.traffic_matrix import generate_traffic_matrix +from topogen.validation import validate_scenario_yaml + + +def dc_scenario(): + return { + "network": { + "nodes": { + "metro1/dc1/r": {"attrs": {"role": "dc"}}, + "metro2/dc1/r": {"attrs": {"role": "dc"}}, + "core": {"attrs": {"role": "core"}}, + }, + "links": [ + {"source": "metro1/dc1/r", "target": "core", "capacity": 10}, + {"source": "metro2/dc1/r", "target": "core", "capacity": 10}, + ], + }, + "demands": { + "first": [ + { + "source": "^metro1/dc1/.*", + "target": "^metro2/dc1/.*", + "volume": 8, + "mode": "pairwise", + } + ] + }, + } + + +def test_independent_demand_sets_do_not_sum_capacity(): + data = dc_scenario() + data["demands"]["second"] = deepcopy(data["demands"]["first"]) + assert not validate_scenario_yaml(yaml.safe_dump(data)) + data["demands"]["second"][0]["volume"] = 11 + assert any( + "dc capacity" in issue and "second" in issue + for issue in validate_scenario_yaml(yaml.safe_dump(data)) + ) + + +def test_uniform_selectors_participate_in_dc_capacity_audit(): + data = dc_scenario() + data["demands"]["first"] = [ + { + "source": "(metro[0-9]+/dc[0-9]+)", + "target": "(metro[0-9]+/dc[0-9]+)", + "volume": 30, + "mode": "pairwise", + } + ] + assert any( + "dc capacity" in issue for issue in validate_scenario_yaml(yaml.safe_dump(data)) + ) + + +def test_malformed_network_is_an_issue_not_an_exception(): + assert validate_scenario_yaml("network: []") + + +def test_integer_string_preserves_exact_value(): + assert _normalize_int("9007199254740993", "capacity") == 9007199254740993 + with pytest.raises(ValueError): + _normalize_int("9007199254740992.5", "capacity") + + +def hose_case(powers): + config = TopologyConfig() + config.traffic.model = "hose" + config.traffic.gbps_per_mw = 1.0 + config.traffic.priority_ratios = {0: 1.0} + config.traffic.gravity.mw_per_dc_region_overrides = powers + metros = [ + {"name": name, "x": i * 1000.0, "y": 0.0} for i, name in enumerate(powers) + ] + settings = {name: {"dc_regions_per_metro": 1} for name in powers} + return config, metros, settings + + +def test_infeasible_hose_is_rejected(): + config, metros, settings = hose_case({"a": 10.0, "b": 100.0}) + with pytest.raises(ValueError, match="hose.*(infeasible|converge)"): + generate_traffic_matrix(metros, settings, config) + + +@pytest.mark.parametrize("model", ["hose", "gravity"]) +def test_disabled_rounding_preserves_offered_traffic(model): + config, metros, settings = hose_case({"a": 0.0002, "b": 0.0002, "c": 0.0002}) + config.traffic.model = model + matrices = generate_traffic_matrix(metros, settings, config) + demands = matrices[config.traffic.matrix_name] + assert sum(d["demand"] for d in demands) == pytest.approx(0.0006, abs=1e-9) + if model == "hose": + totals = defaultdict(float) + for demand in demands: + totals[demand["source_path"]] += demand["demand"] + assert all(abs(value - 0.0002) < 1e-9 for value in totals.values()) + + +def test_duplicate_metro_names_are_rejected_before_site_merge(): + graph = nx.MultiGraph() + for i in range(2): + graph.add_node( + (float(i), 0.0), + node_type="metro", + name="same", + name_orig="Same", + metro_id=str(i), + x=float(i), + y=0.0, + radius_km=1.0, + ) + config = TopologyConfig() + config.traffic.enabled = False + with pytest.raises(ValueError, match="[Dd]uplicate.*name"): + build_scenario(graph, config) + + +@pytest.mark.parametrize("value", [float("nan"), float("inf"), float("-inf")]) +def test_nonfinite_config_values_are_rejected(value): + from pathlib import Path + + raw = yaml.safe_load(Path("examples/small_baseline.yml").read_text()) + raw["traffic"]["gravity"]["min_distance_km"] = value + with pytest.raises(ValueError, match="finite"): + TopologyConfig._from_dict(raw) + + +def test_internal_dc_links_do_not_inflate_external_capacity(): + data = dc_scenario() + data["network"]["nodes"]["metro1/dc1/extra"] = {"attrs": {"role": "dc"}} + data["network"]["links"].append( + {"source": "metro1/dc1/r", "target": "metro1/dc1/extra", "capacity": 1000} + ) + data["demands"]["first"][0]["volume"] = 15 + assert any( + "dc capacity" in issue and "metro1/dc1" in issue + for issue in validate_scenario_yaml(yaml.safe_dump(data)) + ) + + +def test_unmatched_demand_cannot_hide_behind_another_valid_demand(): + data = dc_scenario() + bad = deepcopy(data["demands"]["first"][0]) + bad["source"] = "missing-source" + data["demands"]["first"].append(bad) + assert any( + "entry 1" in issue and "expanded" in issue + for issue in validate_scenario_yaml(yaml.safe_dump(data)) + ) + + +@pytest.mark.parametrize("crs", ["EPSG:4326", "EPSG:2263"]) +def test_geographic_or_feet_crs_is_rejected(crs): + from topogen.config import ProjectionConfig + + with pytest.raises(ValueError, match="projected.*metre"): + ProjectionConfig(target_crs=crs) + + +def test_pop_coordinates_are_checked_without_dc_groups(): + from topogen.validation import validate_scenario_dict + + data = { + "network": { + "nodes": { + "metro1/pop[1]": { + "attrs": {"metro_name": "a", "location_x": 1, "location_y": 2} + } + }, + "links": [{"source": "a", "target": "b"}], + } + } + assert any( + "location differs" in issue + for issue in validate_scenario_dict(data, {"a": (5, 6)}) + ) + + +def test_component_inventory_uses_final_blueprint_assignments(tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + library = tmp_path / "lib" + library.mkdir() + (library / "blueprints.yml").write_text( + yaml.safe_dump( + { + "SingleRouter": { + "nodes": { + "core": { + "count": 1, + "attrs": { + "role": "core", + "hardware": { + "component": "ReplacedPlatform", + "count": 1, + }, + }, + } + }, + "links": [], + } + } + ) + ) + graph = nx.MultiGraph() + graph.add_node( + (0.0, 0.0), + node_type="metro", + name="a", + name_orig="A", + metro_id="a", + x=0.0, + y=0.0, + radius_km=1.0, + ) + config = TopologyConfig() + config.traffic.enabled = False + config.components.hw_component = {"core": "CoreRouter"} + scenario = yaml.safe_load(build_scenario(graph, config)) + assert "CoreRouter" in scenario["components"] + assert "ReplacedPlatform" not in scenario["components"] diff --git a/tests/test_risk_groups_functional.py b/tests/test_risk_groups_functional.py index f73bfe4..a10afef 100644 --- a/tests/test_risk_groups_functional.py +++ b/tests/test_risk_groups_functional.py @@ -5,6 +5,7 @@ from topogen.config import CorridorsConfig, RiskGroupsConfig from topogen.corridors import assign_risk_groups as assign_risk_groups_to_corridors from topogen.metro_clusters import MetroCluster +from topogen.naming import metro_slug class TestRiskGroupAssignment: @@ -391,22 +392,20 @@ def test_metro_name_sanitization(self): ] for original, expected in test_cases: - sanitized = MetroCluster._sanitize_metro_name(original) + sanitized = metro_slug(original) assert sanitized == expected, ( f"Failed for {original}: got {sanitized}, expected {expected}" ) def test_sanitization_edge_cases(self): - assert MetroCluster._sanitize_metro_name("Test & City, CA") == "test-city" + assert metro_slug("Test & City, CA") == "test-city" - assert ( - MetroCluster._sanitize_metro_name("Multi -- Space, TX") == "multi-space" - ) + assert metro_slug("Multi -- Space, TX") == "multi-space" very_long_name = ( "Very Long Metro Name That Exceeds Thirty Characters, State--Extra--Parts" ) - sanitized = MetroCluster._sanitize_metro_name(very_long_name) + sanitized = metro_slug(very_long_name) assert len(sanitized) <= 30 assert sanitized == "very-long-metro-name-that-exce" # Truncated at 30 chars @@ -414,7 +413,7 @@ def test_metro_cluster_uses_sanitized_names(self): original_name = "Denver--Aurora, CO" metro = MetroCluster( metro_id="23527", - name=MetroCluster._sanitize_metro_name(original_name), + name=metro_slug(original_name), name_orig=original_name, uac_code="23527", land_area_km2=1669.0, diff --git a/tests/test_runtime_contract.py b/tests/test_runtime_contract.py new file mode 100644 index 0000000..0715dfc --- /dev/null +++ b/tests/test_runtime_contract.py @@ -0,0 +1,128 @@ +"""Current contracts reject silent substitutions and preserve published output.""" + +from copy import deepcopy +from dataclasses import asdict +from pathlib import Path + +import networkx as nx +import pytest +import yaml + +from topogen import RunContext, TopologyConfig +from topogen.cli import _run_pipeline +from topogen.config import _normalize_int +from topogen.integrated_graph import load_from_json +from topogen.scenario import build_scenario +from topogen.scenario.sizing import _resolve_flow_placement + + +def test_failed_validation_preserves_previous_scenario(tmp_path, monkeypatch): + import topogen + import topogen.scenario + import topogen.validation + + context = RunContext(tmp_path, "input") + context.path("integrated_graph.json").write_text("{}") + target = tmp_path / "scenario.yml" + target.write_text("previous valid scenario") + monkeypatch.setattr( + topogen, "load_from_json", lambda path: (nx.MultiGraph(), "EPSG:5070") + ) + monkeypatch.setattr( + topogen.scenario, "build_scenario", lambda *args, **kwargs: "invalid scenario" + ) + monkeypatch.setattr( + topogen.validation, + "validate_scenario_yaml", + lambda *args, **kwargs: ["bad optics"], + ) + with pytest.raises(ValueError, match="bad optics"): + _run_pipeline(TopologyConfig(), target, context=context) + assert target.read_text() == "previous valid scenario" + assert not list(tmp_path.glob(".scenario.yml.*")) + + +def test_library_build_has_no_implicit_output(tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + config = TopologyConfig() + config.traffic.enabled = False + graph = nx.MultiGraph() + graph.add_node( + (0.0, 0.0), + node_type="metro", + name="a", + name_orig="A", + metro_id="001", + x=0.0, + y=0.0, + radius_km=10.0, + ) + before = asdict(config) + result = build_scenario(graph, config) + assert yaml.safe_load(result)["network"]["nodes"] + assert asdict(config) == before + assert list(tmp_path.iterdir()) == [] + + +@pytest.mark.parametrize( + "section, key", + [ + ("highway_processing", "min_cycle_nodes"), + ("highway_processing", "validation_sample_size"), + ("output", "scenario_metadata"), + ("components", "assignments"), + ("components", "library"), + ], +) +def test_removed_settings_are_rejected(section, key): + raw = yaml.safe_load(Path("examples/small_baseline.yml").read_text()) + raw[section][key] = {} + with pytest.raises(ValueError, match=key): + TopologyConfig._from_dict(raw) + + +def test_config_parser_does_not_mutate_input(): + raw = yaml.safe_load(Path("examples/small_baseline.yml").read_text()) + raw["clustering"].pop("override_metro_clusters") + before = deepcopy(raw) + TopologyConfig._from_dict(raw) + assert raw == before + + +@pytest.mark.parametrize("value", [True, 1.5, "1.5", float("nan"), float("inf")]) +def test_integer_quantities_are_not_coerced(value): + with pytest.raises(ValueError, match="Invalid integer"): + _normalize_int(value, "cost") + + +def test_unknown_flow_placement_is_rejected(): + with pytest.raises(ValueError, match="Unknown flow_placement"): + _resolve_flow_placement("PROPORTOINAL") + + +def test_old_graph_format_is_rejected(tmp_path): + path = tmp_path / "old.json" + path.write_text('{"nodes": [], "edges": [], "target_crs": "EPSG:5070"}') + with pytest.raises(ValueError, match="regenerate"): + load_from_json(path) + + +def test_hardware_validation_uses_only_scenario_definitions(): + from topogen.validation import validate_scenario_yaml + + data = { + "network": { + "nodes": { + "a": { + "attrs": { + "role": "core", + "hardware": {"component": "CoreRouter", "count": 1}, + } + }, + "b": {"attrs": {"role": "core"}}, + }, + "links": [{"source": "a", "target": "b", "capacity": 100, "cost": 1}], + } + } + issues = validate_scenario_yaml(yaml.safe_dump(data)) + assert any("unknown component 'CoreRouter'" in issue for issue in issues) diff --git a/tests/test_scenario_risk_groups.py b/tests/test_scenario_risk_groups.py index f697865..f57d791 100644 --- a/tests/test_scenario_risk_groups.py +++ b/tests/test_scenario_risk_groups.py @@ -1,6 +1,7 @@ """Functional tests for risk groups in scenario generation.""" import networkx as nx +import pytest import yaml from topogen.config import ( @@ -10,13 +11,14 @@ RiskGroupsConfig, TopologyConfig, ) -from topogen.scenario_builder import _build_risk_groups_section, build_scenario +from topogen.scenario import build_scenario +from topogen.scenario.risk import _build_risk_groups_section from topogen.workflows_lib import get_builtin_workflows class TestScenarioRiskGroups: def test_risk_groups_section_generation(self): - graph = nx.Graph() + graph = nx.MultiGraph() metro1 = (100.0, 200.0) metro2 = (200.0, 300.0) @@ -26,6 +28,7 @@ def test_risk_groups_section_generation(self): name="denver-aurora", metro_id="23527", radius_km=30.0, + name_orig="denver-aurora", ) graph.add_node( metro2, @@ -33,6 +36,7 @@ def test_risk_groups_section_generation(self): name="kansas-city", metro_id="43912", radius_km=25.0, + name_orig="kansas-city", ) graph.add_edge( @@ -57,7 +61,7 @@ def test_risk_groups_section_generation(self): assert rg["attrs"]["distance_km"] == 500 def test_risk_groups_in_scenario_yaml(self): - graph = nx.Graph() + graph = nx.MultiGraph() metro1 = (100.0, 200.0) metro2 = (200.0, 300.0) @@ -125,7 +129,7 @@ def test_risk_groups_in_scenario_yaml(self): assert "corridor_risk_denver-aurora_kansas-city" in corridor_link["risk_groups"] def test_multiple_risk_groups_per_link(self): - graph = nx.Graph() + graph = nx.MultiGraph() metro1 = (100.0, 200.0) metro2 = (200.0, 300.0) @@ -137,6 +141,7 @@ def test_multiple_risk_groups_per_link(self): x=100.0, y=200.0, radius_km=25.0, + name_orig="metro1", ) graph.add_node( metro2, @@ -146,6 +151,7 @@ def test_multiple_risk_groups_per_link(self): x=200.0, y=300.0, radius_km=25.0, + name_orig="metro2", ) # Add corridor with multiple risk groups (shared infrastructure) @@ -157,11 +163,16 @@ def test_multiple_risk_groups_per_link(self): capacity=400, risk_groups=[ "corridor_risk_metro1_metro2", - "corridor_risk_metro1_metro3", - "corridor_risk_metro2_metro3", + "corridor_risk_metro1_metro2_path1", + "corridor_risk_metro1_metro2_path2", ], ) + for key, length in ((1, 100.0), (2, 200.0)): + graph.add_edge( + metro1, metro2, key=key, edge_type="corridor", length_km=length + ) + config = TopologyConfig() config.build = BuildConfig( build_defaults=BuildDefaults(pop_per_metro=1, site_blueprint="SingleRouter") @@ -178,11 +189,13 @@ def test_multiple_risk_groups_per_link(self): assert len(scenario_data["risk_groups"]) == 3 risk_group_names = {rg["name"] for rg in scenario_data["risk_groups"]} assert "corridor_risk_metro1_metro2" in risk_group_names - assert "corridor_risk_metro1_metro3" in risk_group_names - assert "corridor_risk_metro2_metro3" in risk_group_names - # Distances should be present for all, equal to ceil(150.0)=150 - for rg in scenario_data["risk_groups"]: - assert rg["attrs"]["distance_km"] == 150 + assert "corridor_risk_metro1_metro2_path1" in risk_group_names + assert "corridor_risk_metro1_metro2_path2" in risk_group_names + assert [rg["attrs"]["distance_km"] for rg in scenario_data["risk_groups"]] == [ + 150, + 100, + 200, + ] corridor_links = [ adj @@ -195,15 +208,29 @@ def test_multiple_risk_groups_per_link(self): assert set(link_risk_groups) == risk_group_names def test_risk_groups_disabled(self): - graph = nx.Graph() + graph = nx.MultiGraph() metro1 = (100.0, 200.0) metro2 = (200.0, 300.0) graph.add_node( - metro1, node_type="metro", name="metro1", metro_id="001", radius_km=25.0 + metro1, + node_type="metro", + name="metro1", + metro_id="001", + radius_km=25.0, + name_orig="metro1", + x=100.0, + y=200.0, ) graph.add_node( - metro2, node_type="metro", name="metro2", metro_id="002", radius_km=25.0 + metro2, + node_type="metro", + name="metro2", + metro_id="002", + radius_km=25.0, + name_orig="metro2", + x=200.0, + y=300.0, ) graph.add_edge(metro1, metro2, edge_type="corridor", length_km=100.0) @@ -226,44 +253,16 @@ def test_risk_groups_disabled(self): for adj in adjacency: assert "risk_groups" not in adj - def test_risk_groups_only_on_corridor_edges(self): - graph = nx.Graph() - metro1 = (100.0, 200.0) - metro2 = (200.0, 300.0) - highway = (300.0, 400.0) - - graph.add_node( - metro1, node_type="metro", name="metro1", metro_id="001", radius_km=20.0 - ) - graph.add_node( - metro2, node_type="metro", name="metro2", metro_id="002", radius_km=20.0 - ) - graph.add_node(highway, node_type="highway") # Not a metro - - # Metro-to-metro edge with risk groups - graph.add_edge( - metro1, - metro2, - edge_type="corridor", - risk_groups=["corridor_risk_metro1_metro2"], - ) - - # Metro-to-highway edge with risk groups (should be ignored) - graph.add_edge( - metro1, highway, edge_type="metro_anchor", risk_groups=["should_be_ignored"] - ) - - config = TopologyConfig() - config.corridors = CorridorsConfig() - config.corridors.risk_groups = RiskGroupsConfig(enabled=True) - - risk_groups = _build_risk_groups_section(graph, config) - - assert len(risk_groups) == 1 - assert risk_groups[0]["name"] == "corridor_risk_metro1_metro2" + def test_unknown_risk_owner_is_rejected(self): + graph = nx.MultiGraph() + graph.add_node(0, name="a") + graph.add_node(1, name="b") + graph.add_edge(0, 1, length_km=10.0, risk_groups=["unknown"]) + with pytest.raises(ValueError, match="no owning corridor path"): + _build_risk_groups_section(graph, TopologyConfig()) def test_metro_name_attributes_in_scenario(self): - graph = nx.Graph() + graph = nx.MultiGraph() metro1 = (100.0, 200.0) graph.add_node( diff --git a/tests/test_script_contracts.py b/tests/test_script_contracts.py new file mode 100644 index 0000000..f586227 --- /dev/null +++ b/tests/test_script_contracts.py @@ -0,0 +1,34 @@ +"""Batch commands must propagate failures to their caller.""" + +import os +import subprocess +from pathlib import Path + +import pytest + + +@pytest.mark.parametrize( + "stage,code", + [("generate", 1), ("generate", 2), ("build", 1), ("build", 3), ("none", 0)], +) +def test_batch_exit_status(tmp_path, stage, code): + configs = tmp_path / "configs" + configs.mkdir() + (configs / "one.yml").write_text("{}") + runner = tmp_path / "topogen" + runner.write_text( + f'#!/bin/sh\nif [ "$1" = "{stage}" ]; then exit {code}; fi\nexit 0\n' + ) + runner.chmod(0o755) + result = subprocess.run( + [ + "bash", + str(Path("build.sh").resolve()), + str(configs), + str(tmp_path / "output"), + ], + env={**os.environ, "PATH": f"{tmp_path}:{os.environ['PATH']}"}, + capture_output=True, + text=True, + ) + assert (result.returncode == 0) == (code == 0), result.stdout + result.stderr diff --git a/tests/test_site_graph_sot.py b/tests/test_site_graph_sot.py index e150d8b..9859bce 100644 --- a/tests/test_site_graph_sot.py +++ b/tests/test_site_graph_sot.py @@ -6,12 +6,12 @@ import yaml from topogen.config import TopologyConfig -from topogen.scenario_builder import build_scenario +from topogen.scenario import build_scenario from topogen.workflows_lib import get_builtin_workflows def _single_metro_graph() -> nx.Graph: - g = nx.Graph() + g = nx.MultiGraph() metro = (100.0, 200.0) g.add_node( metro, @@ -21,6 +21,7 @@ def _single_metro_graph() -> nx.Graph: x=100.0, y=200.0, radius_km=50.0, + name_orig="Denver", ) return g diff --git a/tests/test_site_graph_visualization.py b/tests/test_site_graph_visualization.py index c339115..6fb0349 100644 --- a/tests/test_site_graph_visualization.py +++ b/tests/test_site_graph_visualization.py @@ -17,6 +17,7 @@ def test_export_site_graph_map(tmp_path: Path) -> None: G.add_node( "metro1/pop1", metro_idx=1, + metro_name="Metro 1", site_kind="pop", site_ordinal=1, pos_x=1000.0, @@ -28,6 +29,7 @@ def test_export_site_graph_map(tmp_path: Path) -> None: G.add_node( "metro1/dc1", metro_idx=1, + metro_name="Metro 1", site_kind="dc", site_ordinal=1, pos_x=1020.0, diff --git a/tests/test_site_streamlining.py b/tests/test_site_streamlining.py new file mode 100644 index 0000000..94298c3 --- /dev/null +++ b/tests/test_site_streamlining.py @@ -0,0 +1,162 @@ +"""Adjacency modes and stripe identities survive actual blueprint expansion.""" + +import networkx as nx +import pytest +import yaml +from ngraph.dsl.blueprints.expand import expand_network_dsl + +from topogen.config import TopologyConfig +from topogen.scenario import build_scenario + + +def build_case(tmp_path, monkeypatch, blueprint): + monkeypatch.chdir(tmp_path) + (tmp_path / "lib").mkdir() + (tmp_path / "lib/blueprints.yml").write_text(yaml.safe_dump({"Test": blueprint})) + graph = nx.MultiGraph() + for i, name in enumerate(("a", "b")): + graph.add_node( + (i * 1000.0, 0.0), + node_type="metro", + name=name, + name_orig=name, + metro_id=name, + x=i * 1000.0, + y=0.0, + radius_km=1.0, + ) + graph.add_edge((0.0, 0.0), (1000.0, 0.0), length_km=1.0, edge_type="corridor") + config = TopologyConfig() + config.traffic.enabled = False + config.build.build_defaults.dc_regions_per_metro = 0 + config.build.build_defaults.site_blueprint = "Test" + return graph, config + + +def test_striping_respects_one_to_one_site_mode(tmp_path, monkeypatch): + blueprint = { + "nodes": {"r": {"count": 1, "template": "r{n}", "attrs": {"role": "core"}}} + } + graph, config = build_case(tmp_path, monkeypatch, blueprint) + config.build.build_defaults.inter_metro_link.mode = "one_to_one" + config.build.build_defaults.inter_metro_link.striping = { + "mode": "width", + "width": 1, + } + network = expand_network_dsl(yaml.safe_load(build_scenario(graph, config))) + wan = [ + link + for link in network.links.values() + if link.attrs["link_type"] == "inter_metro_corridor" + ] + assert len(wan) == 2 + assert all(link.source.split("/")[1] == link.target.split("/")[1] for link in wan) + + +@pytest.mark.parametrize("template", ["node", "node+{n}"]) +def test_stripe_members_keep_full_paths_and_literal_names( + tmp_path, monkeypatch, template +): + blueprint = { + "nodes": { + "left": { + "count": 1, + "template": template, + "attrs": {"role": "core", "stripe": "a"}, + }, + "right": { + "count": 1, + "template": template, + "attrs": {"role": "core", "stripe": "b"}, + }, + } + } + graph, config = build_case(tmp_path, monkeypatch, blueprint) + config.build.build_defaults.inter_metro_link.striping = { + "mode": "by_attr", + "attribute": "stripe", + } + network = expand_network_dsl(yaml.safe_load(build_scenario(graph, config))) + wan = [ + link + for link in network.links.values() + if link.attrs["link_type"] == "inter_metro_corridor" + ] + assert len(wan) == 4 + assert {network.nodes[link.source].attrs["stripe"] for link in wan} == {"a", "b"} + assert all( + network.nodes[link.source].attrs["stripe"] + == network.nodes[link.target].attrs["stripe"] + for link in wan + ) + + +def test_striping_preserves_custom_endpoint_match(tmp_path, monkeypatch): + blueprint = { + "nodes": { + "left": { + "count": 1, + "template": "r{n}", + "attrs": {"role": "core", "chosen": True}, + }, + "right": { + "count": 1, + "template": "r{n}", + "attrs": {"role": "core", "chosen": False}, + }, + } + } + graph, config = build_case(tmp_path, monkeypatch, blueprint) + link = config.build.build_defaults.inter_metro_link + link.striping = {"mode": "width", "width": 1} + link.match = {"conditions": [{"attr": "chosen", "op": "==", "value": True}]} + network = expand_network_dsl(yaml.safe_load(build_scenario(graph, config))) + wan = [ + link + for link in network.links.values() + if link.attrs["link_type"] == "inter_metro_corridor" + ] + assert wan and all( + network.nodes[link.source].attrs["chosen"] + and network.nodes[link.target].attrs["chosen"] + for link in wan + ) + + +def test_disabled_risks_are_not_emitted_from_saved_graph(tmp_path, monkeypatch): + blueprint = { + "nodes": {"r": {"count": 1, "template": "r{n}", "attrs": {"role": "core"}}} + } + graph, config = build_case(tmp_path, monkeypatch, blueprint) + for _, _, attrs in graph.edges(data=True): + attrs["risk_groups"] = ["corridor_risk_a_b"] + config.corridors.risk_groups.enabled = False + scenario = yaml.safe_load(build_scenario(graph, config)) + assert "risk_groups" not in scenario + assert all("risk_groups" not in link for link in scenario["network"]["links"]) + + +def test_configured_cost_is_a_floor_without_changing_distance(tmp_path, monkeypatch): + graph, config = build_case( + tmp_path, monkeypatch, {"nodes": {"r": {"count": 1, "attrs": {"role": "core"}}}} + ) + config.build.build_defaults.inter_metro_link.cost = 500 + scenario = yaml.safe_load(build_scenario(graph, config)) + links = [ + link + for link in scenario["network"]["links"] + if link["attrs"]["link_type"] == "inter_metro_corridor" + ] + assert all(link["cost"] == 500 for link in links) + assert all(link["attrs"]["distance_km"] < 500 for link in links) + + +def test_nested_blueprint_dependencies_are_included(tmp_path, monkeypatch): + graph, config = build_case( + tmp_path, + monkeypatch, + {"nodes": {"inner": {"blueprint": "SingleRouter", "attrs": {"role": "core"}}}}, + ) + scenario = yaml.safe_load(build_scenario(graph, config)) + assert set(scenario["blueprints"]) == {"Test", "SingleRouter"} + assert len(expand_network_dsl(scenario).nodes) == 4 diff --git a/tests/test_striping_equivalence.py b/tests/test_striping_equivalence.py index f79d2e6..2d334da 100644 --- a/tests/test_striping_equivalence.py +++ b/tests/test_striping_equivalence.py @@ -4,12 +4,12 @@ import yaml from topogen.config import TopologyConfig -from topogen.scenario_builder import build_scenario +from topogen.scenario import build_scenario from topogen.workflows_lib import get_builtin_workflows def _make_sample_integrated_graph() -> nx.Graph: - g = nx.Graph() + g = nx.MultiGraph() # Three metros with radius for ring adjacency a = (10.0, 10.0) @@ -24,6 +24,7 @@ def _make_sample_integrated_graph() -> nx.Graph: x=10.0, y=10.0, radius_km=40.0, + name_orig="Alpha", ) g.add_node( b, @@ -33,6 +34,7 @@ def _make_sample_integrated_graph() -> nx.Graph: x=20.0, y=20.0, radius_km=35.0, + name_orig="Beta", ) g.add_node( c, @@ -42,6 +44,7 @@ def _make_sample_integrated_graph() -> nx.Graph: x=30.0, y=25.0, radius_km=30.0, + name_orig="Gamma", ) # Corridor edges between metros (symmetric undirected) diff --git a/tests/test_striping_semantics.py b/tests/test_striping_semantics.py index effa19a..c5c2504 100644 --- a/tests/test_striping_semantics.py +++ b/tests/test_striping_semantics.py @@ -7,7 +7,7 @@ import yaml from topogen.config import TopologyConfig -from topogen.scenario_builder import build_scenario +from topogen.scenario import build_scenario from topogen.workflows_lib import get_builtin_workflows @@ -36,7 +36,7 @@ def _counts_per_pair(expanded_net, link_type: str) -> dict[tuple[str, str], int] def _make_sample_graph() -> nx.Graph: - g = nx.Graph() + g = nx.MultiGraph() # three metros with ring radii a = (10.0, 10.0) b = (20.0, 20.0) @@ -49,6 +49,7 @@ def _make_sample_graph() -> nx.Graph: x=10.0, y=10.0, radius_km=40.0, + name_orig="Alpha", ) g.add_node( b, @@ -58,6 +59,7 @@ def _make_sample_graph() -> nx.Graph: x=20.0, y=20.0, radius_km=35.0, + name_orig="Beta", ) g.add_node( c, @@ -67,6 +69,7 @@ def _make_sample_graph() -> nx.Graph: x=30.0, y=25.0, radius_km=30.0, + name_orig="Gamma", ) g.add_edge(a, b, length_km=400.0, capacity=3200, edge_type="corridor") g.add_edge(b, c, length_km=600.0, capacity=3200, edge_type="corridor") @@ -95,15 +98,15 @@ def _patch_blueprints(new_entries: dict[str, dict[str, Any]]): removed: list[str] = [] try: for name, defn in new_entries.items(): - if name not in BL._BUILTIN_BLUEPRINTS: # type: ignore[attr-defined] + if name not in BL._BUILTIN_BLUEPRINTS: removed.append(name) - BL._BUILTIN_BLUEPRINTS[name] = defn # type: ignore[attr-defined] + BL._BUILTIN_BLUEPRINTS[name] = defn yield finally: # remove only those we added for name in removed: with contextlib.suppress(Exception): - del BL._BUILTIN_BLUEPRINTS[name] # type: ignore[attr-defined] + del BL._BUILTIN_BLUEPRINTS[name] def test_equivalence_one_device_each_end_width1(): diff --git a/tests/test_tm_sizing_contract.py b/tests/test_tm_sizing_contract.py index 8a3368c..c976c2a 100644 --- a/tests/test_tm_sizing_contract.py +++ b/tests/test_tm_sizing_contract.py @@ -1,7 +1,6 @@ """Traffic sizing must preserve directional load and NetGraph demand semantics.""" import math -from unittest.mock import patch import networkx as nx import pytest @@ -10,8 +9,9 @@ from ngraph.model.demand.builder import build_demand_set from topogen.config import TopologyConfig -from topogen.scenario.graph_pipeline import tm_based_size_capacities -from topogen.scenario.traffic import _build_traffic_matrix_section +from topogen.scenario.sizing import tm_based_size_capacities +from topogen.scenario.traffic import to_demand_sets +from topogen.traffic_matrix import generate_traffic_matrix def _inputs(base_capacity=1.0): @@ -55,11 +55,7 @@ def test_capacity_covers_both_directions(respect_min, base, expected): {"source_path": "^metro1/dc1", "sink_path": "^metro2/dc1", "demand": 1000.0}, {"source_path": "^metro2/dc1", "sink_path": "^metro1/dc1", "demand": 100.0}, ] - with patch( - "topogen.traffic_matrix.generate_traffic_matrix", - return_value={"default": demands}, - ): - tm_based_size_capacities(graph, metros, settings, config) + tm_based_size_capacities(graph, metros, settings, config, {"default": demands}) edge = graph["metro1/pop1"]["metro2/pop1"]["corridor"] assert edge["base_capacity"] == expected assert edge["target_capacity"] == expected @@ -76,7 +72,7 @@ def test_uniform_sizing_matches_netgraph_pairwise_expansion(dc_counts): for dc in range(1, count + 1): network.add_node(Node(f"metro{metro}/dc{dc}/dc")) demand_set = build_demand_set( - _build_traffic_matrix_section(metros, settings, config) + to_demand_sets(generate_traffic_matrix(metros, settings, config)) ) expanded = expand_demands(network, demand_set.get_set("default")) # NetGraph divides the volume among all ordered, distinct DC pairs; @@ -87,7 +83,13 @@ def test_uniform_sizing_matches_netgraph_pairwise_expansion(dc_counts): if d.src_name.startswith("metro1/") and d.dst_name.startswith("metro2/") ) assert expected > 0 - tm_based_size_capacities(graph, metros, settings, config) + tm_based_size_capacities( + graph, + metros, + settings, + config, + generate_traffic_matrix(metros, settings, config), + ) assert graph["metro1/pop1"]["metro2/pop1"]["corridor"][ "base_capacity" ] == math.ceil(expected) @@ -96,12 +98,8 @@ def test_uniform_sizing_matches_netgraph_pairwise_expansion(dc_counts): def test_unknown_sizing_selector_is_not_silently_dropped(): graph, metros, settings, config = _inputs() demands = [{"source_path": "unknown", "sink_path": "^metro2/dc1", "demand": 100.0}] - with patch( - "topogen.traffic_matrix.generate_traffic_matrix", - return_value={"default": demands}, - ): - with pytest.raises(ValueError, match="unsupported.*endpoint"): - tm_based_size_capacities(graph, metros, settings, config) + with pytest.raises(ValueError, match="unsupported.*endpoint"): + tm_based_size_capacities(graph, metros, settings, config, {"default": demands}) def test_sizing_preserves_corridors_between_distinct_pop_pairs(): @@ -114,5 +112,52 @@ def test_sizing_preserves_corridors_between_distinct_pop_pairs(): key="corridor", **graph["metro1/pop1"]["metro2/pop1"]["corridor"], ) - tm_based_size_capacities(graph, metros, settings, config) + tm_based_size_capacities( + graph, + metros, + settings, + config, + generate_traffic_matrix(metros, settings, config), + ) assert [data["base_capacity"] for _, _, data in graph.edges(data=True)] == [5, 5] + + +def test_uniform_volume_is_shared_between_dc_sites_not_devices(): + _, metros, settings, config = _inputs() + network = Network() + for site, count in (("metro1/dc1", 1), ("metro1/dc2", 2), ("metro2/dc1", 3)): + for index in range(count): + network.add_node(Node(f"{site}/r{index}")) + demands = build_demand_set( + to_demand_sets(generate_traffic_matrix(metros, settings, config)) + ) + expanded = expand_demands(network, demands.get_set("default")).demands + outgoing = {} + for demand in expanded: + source = demand.src_name.rsplit("/", 1)[0] + target = demand.dst_name.rsplit("/", 1)[0] + assert source != target + outgoing[source] = outgoing.get(source, 0) + demand.volume + assert outgoing == pytest.approx( + {"metro1/dc1": 10, "metro1/dc2": 10, "metro2/dc1": 10} + ) + + +@pytest.mark.parametrize( + "source", ["^metro1garbage", "^metro1/dc0", "^metro1/dc1|metro2/dc1"] +) +def test_sizing_rejects_partial_endpoint_matches(source): + graph, metros, settings, config = _inputs() + demands = [{"source_path": source, "sink_path": "^metro2/dc1", "demand": 10}] + with pytest.raises(ValueError, match="unsupported.*endpoint"): + tm_based_size_capacities(graph, metros, settings, config, {"default": demands}) + + +@pytest.mark.parametrize("volume", [-1, float("nan"), float("inf")]) +def test_sizing_rejects_invalid_demand_volume(volume): + graph, metros, settings, config = _inputs() + demands = [ + {"source_path": "^metro1/dc1", "sink_path": "^metro2/dc1", "demand": volume} + ] + with pytest.raises(ValueError, match="finite non-negative"): + tm_based_size_capacities(graph, metros, settings, config, {"default": demands}) diff --git a/tests/test_traffic_streamlining.py b/tests/test_traffic_streamlining.py new file mode 100644 index 0000000..e62f158 --- /dev/null +++ b/tests/test_traffic_streamlining.py @@ -0,0 +1,50 @@ +"""Traffic model contracts independent of inventory order and global PRNG.""" + +import random + +from topogen.config import TopologyConfig +from topogen.traffic_matrix import generate_traffic_matrix + + +def traffic_case(names): + config = TopologyConfig() + config.traffic.model = "gravity" + config.traffic.mw_per_dc_region = 10 + config.traffic.gbps_per_mw = 1 + config.traffic.priority_ratios = {0: 1} + metros = [{"name": name, "x": i * 1000.0, "y": 0.0} for i, name in enumerate(names)] + settings = {name: {"dc_regions_per_metro": 1} for name in names} + return config, metros, settings + + +def test_gravity_top_k_works_for_reverse_alphabetic_inventory(): + config, metros, settings = traffic_case(["z", "y", "x"]) + config.traffic.gravity.max_partners_per_dc = 1 + demands = generate_traffic_matrix(metros, settings, config)[ + config.traffic.matrix_name + ] + assert len(demands) == 4 + assert abs(sum(item["demand"] for item in demands) - 30) < 1e-9 + + +def test_gravity_jitter_is_seeded_without_touching_global_prng(): + config, metros, settings = traffic_case(["a", "b", "c", "d"]) + config.traffic.gravity.jitter_stddev = 0.8 + state = random.getstate() + first = generate_traffic_matrix(metros, settings, config) + second = generate_traffic_matrix(metros, settings, config) + assert first == second + assert random.getstate() == state + config.output.scenario_seed += 1 + assert generate_traffic_matrix(metros, settings, config) != first + + +def test_zero_share_class_does_not_require_jitter_normalization(): + config, metros, settings = traffic_case(["a", "b", "c"]) + config.traffic.priority_ratios = {0: 0.0, 1: 1.0} + config.traffic.gravity.jitter_stddev = 0.5 + demands = generate_traffic_matrix(metros, settings, config)[ + config.traffic.matrix_name + ] + assert {demand["priority"] for demand in demands} == {1} + assert abs(sum(demand["demand"] for demand in demands) - 30) < 1e-9 diff --git a/tests/test_validation.py b/tests/test_validation.py index f213d25..15c7b70 100644 --- a/tests/test_validation.py +++ b/tests/test_validation.py @@ -127,7 +127,7 @@ def test_validate_scenario_yaml_isolated_nodes_flagged(): mw_per_dc_region: 50.0 gbps_per_mw: 250.0 """ - # Without adjacency, fallback scenario-level isolation should be reported. + # Metadata-only validation reports site groups without any adjacency rules. issues = validate_scenario_yaml( yaml_text, integrated_graph_path=None, run_ngraph=False ) @@ -135,168 +135,51 @@ def test_validate_scenario_yaml_isolated_nodes_flagged(): def test_dc_capacity_vs_demand_validation(): - data = _minimal_scenario() - # Add one dc_to_pop adjacency with target_capacity 1000 - data["network"]["links"] = [ - { - "source": "metro1/dc1", - "target": "metro1/pop1", - "pattern": "one_to_one", - "capacity": 1000.0, - "cost": 1, - "attrs": { - "link_type": "dc_to_pop", - "source_metro": "Denver", - "target_metro": "Denver", - "target_capacity": 1000.0, - }, - } - ] - # Traffic matrix with a single class that demands 1200 out of the DC and 800 into the DC - data["demands"] = { - "tm": [ - { - "source": "^metro1/dc1/.*", - "target": "^metro1/dc1/.*", - "mode": "pairwise", - "priority": 0, - "volume": 1200.0, - }, - { - "source": "^metro2/dc1/.*", - "target": "^metro1/dc1/.*", - "mode": "pairwise", - "priority": 0, - "volume": 800.0, - }, - ] - } - issues = validate_scenario_dict(data) - # Egress 1200 > capacity 1000 => violation - assert any( - "egress demand" in s and "exceeds adjacency capacity" in s for s in issues - ) - # Ingress 800 <= capacity 1000 => no ingress violation for metro1/dc1 - assert not any("ingress demand 800.0" in s for s in issues) - - -def test_groups_that_expand_to_zero_nodes_are_flagged(monkeypatch): - # Patch DSL expansion to return zero nodes and zero links - class _FakeNet: - def __init__(self) -> None: - self.nodes = {} - self.links = {} - - def _fake_expand(_dsl: dict): # noqa: ANN001 - return _FakeNet() - - monkeypatch.setattr( - "ngraph.dsl.blueprints.expand.expand_network_dsl", _fake_expand, raising=True - ) - - # Patch Scenario.from_yaml to avoid schema errors and isolation noise - class _FakeScenario: - @classmethod - def from_yaml(cls, _y: str): # noqa: ANN001 - return cls() - - class network: # noqa: N801 - name per production API - @staticmethod - def to_strict_multidigraph(add_reverse: bool = True): # noqa: FBT001, FBT002 - class _G: - @staticmethod - def get_nodes(): - return {} - - @staticmethod - def get_edges(): - return {} - - return _G() - - monkeypatch.setattr("ngraph.scenario.Scenario", _FakeScenario, raising=True) - # A group with zero count via an invalid blueprint should be flagged. - # Use a blueprint that exists but set an impossible range in the group path. data = { - "blueprints": {}, "network": { "nodes": { - # Invalid range [1-0] yields zero nodes - "metro1/pop[1-0]": { - "blueprint": "SingleRouter", - "attrs": { - "metro_name": "X", - "metro_name_orig": "X", - "metro_id": 1, - "location_x": 0.0, - "location_y": 0.0, - }, - } + name: {"attrs": {"role": "dc"}} + for name in ("metro1/dc1/r", "metro2/dc1/r") }, - "links": [], + "links": [ + {"source": "metro1/dc1/r", "target": "metro2/dc1/r", "capacity": 1000} + ], + }, + "demands": { + "tm": [ + { + "source": "^metro1/dc1/.*", + "target": "^metro2/dc1/.*", + "mode": "pairwise", + "volume": 1200, + }, + { + "source": "^metro2/dc1/.*", + "target": "^metro1/dc1/.*", + "mode": "pairwise", + "volume": 800, + }, + ] }, - "failures": {}, - "demands": {}, - "workflow": [], } - issues = validate_scenario_yaml( - yaml.safe_dump(data, sort_keys=False), - integrated_graph_path=None, - run_ngraph=True, - ) - assert any("expands to 0 nodes" in s for s in issues) - - -def test_adjacencies_that_expand_to_zero_links_are_flagged(monkeypatch): - # Patch DSL expansion to return zero nodes and zero links - class _FakeNet: - def __init__(self) -> None: - self.nodes = {} - self.links = {} - - def _fake_expand(_dsl: dict): # noqa: ANN001 - return _FakeNet() - - monkeypatch.setattr( - "ngraph.dsl.blueprints.expand.expand_network_dsl", _fake_expand, raising=True + issues = validate_scenario_yaml(yaml.safe_dump(data)) + assert any( + "metro1/dc1 egress demand" in issue and "exceeds adjacency capacity" in issue + for issue in issues ) + assert not any("metro1/dc1 ingress demand" in issue for issue in issues) - # Patch Scenario.from_yaml to avoid schema errors and isolation noise - class _FakeScenario: - @classmethod - def from_yaml(cls, _y: str): # noqa: ANN001 - return cls() - - class network: # noqa: N801 - name per production API - @staticmethod - def to_strict_multidigraph(add_reverse: bool = True): # noqa: FBT001, FBT002 - class _G: - @staticmethod - def get_nodes(): - return {} - @staticmethod - def get_edges(): - return {} +def test_invalid_group_range_is_rejected(): + data = {"network": {"nodes": {"metro1/pop[1-0]": {}}, "links": []}} + issues = validate_scenario_yaml(yaml.safe_dump(data), run_ngraph=True) + assert any("Invalid range '1-0'" in issue for issue in issues) - return _G() - monkeypatch.setattr("ngraph.scenario.Scenario", _FakeScenario, raising=True) - # Construct a scenario with a valid group but an adjacency using nonexistent endpoints +def test_adjacencies_that_expand_to_zero_links_are_flagged(): data = { "network": { - "nodes": { - "metro1/pop[1]": { - "blueprint": "SingleRouter", - "attrs": { - "metro_name": "Y", - "metro_name_orig": "Y", - "metro_id": 1, - "location_x": 0.0, - "location_y": 0.0, - }, - }, - }, + "nodes": {"metro1/pop1": {"attrs": {"role": "core"}}}, "links": [ { "source": "metro1/missing", @@ -306,16 +189,9 @@ def get_edges(): } ], }, - "failures": {}, - "demands": {}, - "workflow": [], } - issues = validate_scenario_yaml( - yaml.safe_dump(data, sort_keys=False), - integrated_graph_path=None, - run_ngraph=True, - ) - assert any("adjacency[0] expands to 0 links" in s for s in issues) + issues = validate_scenario_yaml(yaml.safe_dump(data), run_ngraph=True) + assert any("adjacency[0] expands to 0 links" in issue for issue in issues) def yaml_dump(d: dict) -> str: @@ -343,7 +219,6 @@ def test_node_hardware_presence_audited(): "capacity": 1000.0, "ports": 10, }, - "hw_component": {"core": "CoreRouter"}, }, "network": { "nodes": { @@ -361,12 +236,15 @@ def test_node_hardware_presence_audited(): }, } issues = validate_scenario_yaml( - yaml_dump(data), integrated_graph_path=None, run_ngraph=True + yaml_dump(data), + integrated_graph_path=None, + run_ngraph=True, + hw_component_map={"core": "CoreRouter"}, ) assert any("node hardware:" in s for s in issues) -def test_json_schema_validation_flags_invalid_when_available(monkeypatch): +def test_invalid_network_nodes_type_is_rejected(): invalid = { "network": { # wrong type: nodes should be a mapping, make it a list to fail @@ -389,7 +267,7 @@ def test_json_schema_validation_flags_invalid_when_available(monkeypatch): issues = validate_scenario_yaml( yaml_dump(invalid), integrated_graph_path=None, run_ngraph=True ) - assert any("ngraph schema:" in s for s in issues) + assert any("nodes" in s and "mapping" in s for s in issues) def test_link_optics_presence_audited_unordered_and_directional(): @@ -425,7 +303,7 @@ def test_link_optics_presence_audited_unordered_and_directional(): "800G-DR4": {"component_type": "optic", "capacity": 800.0, "ports": 4}, "1600G-2xDR4": {"component_type": "optic", "capacity": 1600.0, "ports": 8}, # First, unordered only (single 'leaf|spine') should require both ends to be that optic - "optics": {"leaf|spine": "800G-DR4"}, + "optics": {"leaf->spine": "800G-DR4", "spine->leaf": "800G-DR4"}, }, "network": { "nodes": { @@ -443,8 +321,12 @@ def test_link_optics_presence_audited_unordered_and_directional(): }, } # No hardware assigned on links in blueprint -> should flag both ends + optics_map = data["components"].pop("optics") issues = validate_scenario_yaml( - yaml_dump(data), integrated_graph_path=None, run_ngraph=True + yaml_dump(data), + integrated_graph_path=None, + run_ngraph=True, + optics_map=optics_map, ) assert any( "optics: missing hardware required by mapping on source end" in s @@ -455,17 +337,12 @@ def test_link_optics_presence_audited_unordered_and_directional(): for s in issues ) - # Now add directional override alongside unordered: both present means ordered interpretations. - data["components"]["optics"] = { - "leaf|spine": "800G-DR4", - "spine|leaf": "1600G-2xDR4", - } issues2 = validate_scenario_yaml( - yaml_dump(data), integrated_graph_path=None, run_ngraph=True + yaml_dump(data), + optics_map={"leaf->spine": "800G-DR4", "spine->leaf": "1600G-2xDR4"}, ) - # Still missing since blueprint didn't assign; we only assert presence of messages (not exact counts) - assert any("source end" in s for s in issues2) - assert any("target end" in s for s in issues2) + assert any("source end" in issue for issue in issues2) + assert any("target end" in issue for issue in issues2) def test_port_budget_detects_platform_port_overuse(): @@ -499,14 +376,20 @@ def test_port_budget_detects_platform_port_overuse(): # 54.4 Tb/s per link requires ceil(54400/800)=68 modules at each end "capacity": 54_400.0, "cost": 1, - "attrs": {"link_type": "leaf_spine"}, + "attrs": { + "link_type": "leaf_spine", + "hardware": { + end: {"component": "800G-DR4", "count": 68} + for end in ("source", "target") + }, + }, } ], } }, "components": { # Use optics mapping to infer per-end optics when link hardware isn't specified - "optics": {"leaf|spine": "800G-DR4"}, + "optics": {"leaf->spine": "800G-DR4", "spine->leaf": "800G-DR4"}, }, "network": { "nodes": { @@ -523,8 +406,15 @@ def test_port_budget_detects_platform_port_overuse(): "links": [], }, } + from topogen.components_lib import get_builtin_components + + optics_map = data["components"].pop("optics") + data["components"].update(get_builtin_components()) issues = validate_scenario_yaml( - yaml_dump(data), integrated_graph_path=None, run_ngraph=True + yaml_dump(data), + integrated_graph_path=None, + run_ngraph=True, + optics_map=optics_map, ) # Expect a port budget violation referencing LeafRouter (64 ports) needing 68 assert any("hardware ports:" in s and "LeafRouter" in s for s in issues) diff --git a/tests/topogen/scenario/test_config_unit.py b/tests/topogen/scenario/test_config_unit.py index d350030..248af38 100644 --- a/tests/topogen/scenario/test_config_unit.py +++ b/tests/topogen/scenario/test_config_unit.py @@ -1,13 +1,12 @@ from __future__ import annotations -from types import SimpleNamespace - import pytest +from topogen.config import BuildConfig, BuildDefaults, LinkParams, TopologyConfig from topogen.scenario.config import _determine_metro_settings -def _build_cfg() -> SimpleNamespace: +def _build_cfg() -> TopologyConfig: def link( cap, cost, @@ -17,7 +16,7 @@ def link( striping=None, mode="mesh", ): - return SimpleNamespace( + return LinkParams( capacity=cap, cost=cost, attrs=({} if attrs is None else attrs), @@ -27,7 +26,7 @@ def link( mode=mode, ) - defaults = SimpleNamespace( + defaults = BuildDefaults( pop_per_metro=2, site_blueprint="SingleRouter", dc_regions_per_metro=1, @@ -80,8 +79,8 @@ def link( }, } } - return SimpleNamespace( - build=SimpleNamespace(build_defaults=defaults, build_overrides=overrides) + return TopologyConfig( + build=BuildConfig(build_defaults=defaults, build_overrides=overrides) ) diff --git a/tests/topogen/scenario/test_graph_pipeline_unit.py b/tests/topogen/scenario/test_graph_pipeline_unit.py index 6dc8f27..db59c74 100644 --- a/tests/topogen/scenario/test_graph_pipeline_unit.py +++ b/tests/topogen/scenario/test_graph_pipeline_unit.py @@ -1,32 +1,40 @@ from __future__ import annotations -from types import SimpleNamespace -from unittest.mock import patch - import networkx as nx import topogen.scenario.graph_pipeline as gp - - -def _cfg() -> SimpleNamespace: +import topogen.scenario.network as nw +import topogen.scenario.sizing as sz +from topogen.blueprints_lib import get_builtin_blueprints +from topogen.config import ( + BuildConfig, + BuildTmSizingConfig, + TopologyConfig, + TrafficConfig, +) +from topogen.naming import site_edge_id +from topogen.scenario.expansion import resolve_network + + +def _cfg() -> TopologyConfig: # Minimal config object with nested fields used by to_network_sections - return SimpleNamespace( - traffic=SimpleNamespace(enabled=True, mw_per_dc_region=10.0, gbps_per_mw=100.0), - build=SimpleNamespace(tm_sizing=SimpleNamespace(enabled=False)), + return TopologyConfig( + traffic=TrafficConfig(enabled=True, mw_per_dc_region=10.0, gbps_per_mw=100.0), + build=BuildConfig(tm_sizing=BuildTmSizingConfig(enabled=False)), ) -def _tm_sizing_cfg() -> SimpleNamespace: +def _tm_sizing_cfg() -> TopologyConfig: # Config with TM sizing enabled for testing - return SimpleNamespace( - traffic=SimpleNamespace( + return TopologyConfig( + traffic=TrafficConfig( enabled=True, mw_per_dc_region=10.0, gbps_per_mw=100.0, matrix_name="default", ), - build=SimpleNamespace( - tm_sizing=SimpleNamespace( + build=BuildConfig( + tm_sizing=BuildTmSizingConfig( enabled=True, quantum_gbps=3200.0, headroom=1.3, @@ -90,15 +98,15 @@ def test_add_intra_metro_edges_cost_arc() -> None: "intra_metro_link": {"capacity": 3200, "cost": 1}, } } - gp._add_intra_metro_edges(G, metros, metro_settings, _cfg(), idx_map) + gp._add_intra_metro_edges(G, metros, metro_settings, idx_map) assert G.number_of_edges() == 3 - # Longest arc between pop1 and pop3 should be > base cost + # Three equally spaced PoPs have the same shortest ring-arc distance. data = G.get_edge_data("metro1/pop1", "metro1/pop3") any_key = next(iter(data)) assert data[any_key]["cost"] >= 1 -def test_add_inter_metro_edges_one_to_one(monkeypatch) -> None: # type: ignore[no-untyped-def] +def test_add_inter_metro_edges_one_to_one() -> None: G = nx.MultiGraph() metros = [ {"name": "A", "x": 0.0, "y": 0.0, "radius_km": 10.0, "node_key": (0.0, 0.0)}, @@ -115,19 +123,14 @@ def test_add_inter_metro_edges_one_to_one(monkeypatch) -> None: # type: ignore[ G.add_node(gp._site_node_id(1, "pop", p), site_kind="pop") G.add_node(gp._site_node_id(2, "pop", p), site_kind="pop") - # Monkeypatch corridor extractor to return a single corridor edge - monkeypatch.setattr( - gp, - "_extract_corridor_edges", - lambda graph: [ - { - "source": (0.0, 0.0), - "target": (100.0, 0.0), - "length_km": 500.0, - "edge_type": "corridor", - "risk_groups": [], - } - ], + corridors = nx.MultiGraph() + corridors.add_edge( + (0.0, 0.0), + (100.0, 0.0), + key=0, + length_km=500.0, + edge_type="corridor", + risk_groups=[], ) metro_settings = { @@ -145,12 +148,11 @@ def test_add_inter_metro_edges_one_to_one(monkeypatch) -> None: # type: ignore[ } gp._add_inter_metro_edges( G, - metros, metro_settings, - nx.Graph(), - _cfg(), + corridors, idx_map, {(0.0, 0.0): metros[0], (100.0, 0.0): metros[1]}, + gp.StripePlanner(get_builtin_blueprints()), ) assert G.number_of_edges() == 2 for _u, _v, d in G.edges(data=True): @@ -165,7 +167,25 @@ def test_assign_per_link_capacity_splits_by_expansion() -> None: G.add_node(u, site_blueprint="FullMesh4") G.add_node(v, site_blueprint="FullMesh4") G.add_edge(u, v, key="k1", base_capacity=3200.0, match={}) - gp.assign_per_link_capacity(G, _cfg()) + scenario = { + "blueprints": get_builtin_blueprints(), + "network": { + "nodes": {u: {"blueprint": "FullMesh4"}, v: {"blueprint": "FullMesh4"}}, + "links": [ + { + "source": u, + "target": v, + "pattern": "one_to_one", + "capacity": 3200.0, + "attrs": {"site_edge": site_edge_id(u, v, "k1")}, + } + ], + }, + } + network = resolve_network(G, scenario, {}) + concrete = [link for link in network.links.values() if "site_edge" in link.attrs] + assert len(concrete) == 4 + assert sum(link.capacity for link in concrete) == 3200.0 data = G.get_edge_data(u, v)["k1"] assert data["capacity"] == 800.0 @@ -209,7 +229,10 @@ def test_to_network_sections_serializes_groups_and_adjacency() -> None: source_metro="A", target_metro="B", ) - groups, adjacency = gp.to_network_sections(G, metros, settings, _cfg()) + for index, metro in enumerate(metros): + metro["name_orig"] = metro["name"] + metro["metro_id"] = str(index) + groups, adjacency = nw.to_network_sections(G, metros, settings, _cfg()) assert any(path.endswith("/pop[1-1]") for path in groups) assert any(path.endswith("/dc[1-1]") for path in groups) assert any( @@ -295,8 +318,7 @@ def test_tm_sizing_preserves_parallel_edges() -> None: } cfg = _tm_sizing_cfg() - with patch("topogen.traffic_matrix.generate_traffic_matrix", return_value=mock_tm): - gp.tm_based_size_capacities(G, metros, metro_settings, cfg) + sz.tm_based_size_capacities(G, metros, metro_settings, cfg, mock_tm) corridor_edges = [ (u, v, k, d) @@ -383,8 +405,7 @@ def test_tm_sizing_single_edge_baseline() -> None: } cfg = _tm_sizing_cfg() - with patch("topogen.traffic_matrix.generate_traffic_matrix", return_value=mock_tm): - gp.tm_based_size_capacities(G, metros, metro_settings, cfg) + sz.tm_based_size_capacities(G, metros, metro_settings, cfg, mock_tm) corridor_data = G.get_edge_data("metro1/pop1", "metro2/pop1", "corridor:0") assert corridor_data is not None diff --git a/tests/topogen/test_blueprint_viz_unit.py b/tests/topogen/test_blueprint_viz_unit.py index c609bd8..76b0e7f 100644 --- a/tests/topogen/test_blueprint_viz_unit.py +++ b/tests/topogen/test_blueprint_viz_unit.py @@ -1,88 +1,57 @@ -from __future__ import annotations - from typing import Any -from topogen.blueprint_viz import ( - AbstractView, - _extract_vars, - _first_path_component, - _iter_assignments, - _subst, - build_abstract_view, - collect_concrete_site, -) -from topogen.blueprints_lib import get_builtin_blueprints - - -def test_helper_functions() -> None: - assert _first_path_component("/G${g}/G${g}_r${r}") == "G${g}" - assert _first_path_component("G1/G1_r1") == "G1" - assert _extract_vars("G${a}_x$b") == ["a", "b"] - - # zip alignment (truncate to min length) - assigns = _iter_assignments({"a": [1, 2], "b": [3]}, ["a", "b"], "zip") - assert assigns == [{"a": 1, "b": 3}] - # product - assigns = _iter_assignments({"a": [1, 2], "b": [3, 4]}, ["a", "b"], "product") - assert {tuple(sorted(x.items())) for x in assigns} == { - (("a", 1), ("b", 3)), - (("a", 1), ("b", 4)), - (("a", 2), ("b", 3)), - (("a", 2), ("b", 4)), - } - # missing vars → single empty assignment - assert _iter_assignments({"a": []}, ["a"], "zip") == [{}] - - # substitution leaves unknowns intact - assert _subst("G${a}_x${b}", {"a": 1}) == "G1_x${b}" - assert _subst("G$a/r$b", {"a": 1, "b": 2}) == "G1/r2" - +from ngraph.dsl.blueprints.expand import expand_network_dsl -def test_build_abstract_view_and_self_loops() -> None: - bp = { - "nodes": { - "G1": {"count": 2, "attrs": {"role": "agg"}}, - "G2": {"count": 1}, - "G1_r1": {"count": 2}, - "G1_r2": {"count": 2}, - }, - "links": [ - { - "source": "G${g}", - "target": "G${g}_r${r}", - "pattern": "uplink", - "expand": {"vars": {"g": [1, 2], "r": [1, 1]}, "mode": "zip"}, - "attrs": {"target_capacity": 100}, - }, - { - "source": "G1_r${r}", - "target": "G1_r${r}", - "pattern": "mesh", - "expand": {"vars": {"r": [1, 2]}, "mode": "zip"}, - "attrs": {"target_capacity": 50}, - }, - ], - } - av: AbstractView = build_abstract_view(bp) - assert set(av.graph.nodes) >= {"G1", "G2", "G1_r1", "G1_r2"} - assert "role=agg" in av.node_labels["G1"] - # Inter-group edges deduplicated by unordered pair and labeled - edges = list(av.edge_labels.keys()) - assert edges, "expected at least one inter-group edge" - any_label = next(iter(av.edge_labels.values())) - assert "uplink" in any_label and "100" in any_label - loop_groups = {g for (g, _lbl) in av.self_loops} - assert {"G1_r1", "G1_r2"} & loop_groups +from topogen.blueprint_viz import build_abstract_view, collect_concrete_site +from topogen.blueprints_lib import get_builtin_blueprints -def test_builtin_dragonfly_abstract_view_expands_placeholders() -> None: - av = build_abstract_view(get_builtin_blueprints()["Dragonfly_A3H2G7"]) +def test_builtin_dragonfly_abstract_view_expands_placeholders(): + network = expand_network_dsl( + { + "blueprints": get_builtin_blueprints(), + "network": {"nodes": {"metro1/pop1": {"blueprint": "Dragonfly_A3H2G7"}}}, + } + ) + view = build_abstract_view(network, "metro1/pop1") groups = {f"G{i}" for i in range(1, 8)} - assert set(av.graph.nodes) == groups - assert {(u, v) for u, v, _k in av.graph.edges} == { + assert set(view.graph.nodes) == groups + assert set(view.graph.edges()) == { (f"G{u}", f"G{v}") for u in range(1, 8) for v in range(u + 1, 8) } - assert {g for g, _lbl in av.self_loops} == groups + assert {g for g, _ in view.self_loops} == groups + assert all("N=3" in label for label in view.node_labels.values()) + assert set(view.edge_labels.values()) == {"800 Gbps"} + assert set(label for _, label in view.self_loops) == {"2,400 Gbps"} + + +def test_dict_selectors_do_not_create_phantom_groups(): + network = expand_network_dsl( + { + "network": { + "nodes": { + "metro1/pop1/a": {"count": 2, "template": "r{n}"}, + "metro1/pop1/b": {"count": 1, "template": "r{n}"}, + }, + "links": [ + { + "source": {"path": "metro1/pop1/a"}, + "target": {"path": "metro1/pop1/b"}, + "pattern": "mesh", + "capacity": 10, + } + ], + } + } + ) + view = build_abstract_view(network, "metro1/pop1") + assert set(view.graph) == {"a", "b"} + assert list(view.graph.edges()) == [("a", "b")] + assert list(view.edge_labels.values()) == ["20 Gbps"] + assert ( + build_abstract_view(network, "metro1/pop1", include_self_loops=False).self_loops + == [] + ) class _Node: @@ -109,8 +78,8 @@ class _Net: 3: _Node("metro2/dc3/X"), } net.links = { - 1: _Link(net.nodes[1], net.nodes[2], 10.0), - 2: _Link(net.nodes[1], net.nodes[3], 20.0), + 1: _Link(net.nodes[1].name, net.nodes[2].name, 10.0), + 2: _Link(net.nodes[1].name, net.nodes[3].name, 20.0), } ns, pos, links = collect_concrete_site(net, "metro1/dc1") diff --git a/tests/topogen/validation/test_expand_checks.py b/tests/topogen/validation/test_expand_checks.py index 41b5385..3195bbe 100644 --- a/tests/topogen/validation/test_expand_checks.py +++ b/tests/topogen/validation/test_expand_checks.py @@ -2,6 +2,8 @@ from types import SimpleNamespace +from ngraph.dsl.blueprints.expand import expand_network_dsl + from topogen.validation.audits.expand_checks import check_groups_adjacency_blueprints @@ -37,7 +39,7 @@ def test_expand_checks_flags_empty_groups_and_adjacency_and_bp(): } # ng_expand stub: for tagged group expansion, produce zero nodes to trigger issue - def _expand(stub): # type: ignore[no-untyped-def] + def _expand(stub): class Net: def __init__(self): self.nodes = {} @@ -76,3 +78,35 @@ def __init__(self): assert any("group 'g1' expands to 0 nodes" in s for s in issues) assert any("adjacency[0] expands to 0 links" in s for s in issues) assert any("blueprint 'BP' adjacency[0] expands to 0 links" in s for s in issues) + + +def test_expansion_audit_keeps_node_rules(): + """Selectors depending on node rules must see exactly the emitted network.""" + dsl = { + "network": { + "nodes": { + "a": {"attrs": {"role": "core"}}, + "b": {"attrs": {"role": "core"}}, + }, + "node_rules": [{"path": ".*", "attrs": {"stripe": "one"}}], + "links": [ + { + "source": { + "path": "a", + "match": { + "conditions": [ + {"attr": "stripe", "op": "==", "value": "one"} + ] + }, + }, + "target": "b", + "pattern": "one_to_one", + } + ], + }, + } + assert len(expand_network_dsl(dsl).links) == 1 + issues = check_groups_adjacency_blueprints( + dsl, expand_network_dsl, SimpleNamespace(error=lambda *args: None) + ) + assert issues == [] diff --git a/tests/topogen/validation/test_helpers.py b/tests/topogen/validation/test_helpers.py index 85905ea..9f93101 100644 --- a/tests/topogen/validation/test_helpers.py +++ b/tests/topogen/validation/test_helpers.py @@ -2,6 +2,8 @@ import math +import pytest + from topogen.validation.helpers import ( _build_ig_coord_map, _float_or_nan, @@ -9,18 +11,14 @@ ) -def test_build_ig_coord_map_filters_and_extracts() -> None: - ig = { - "nodes": [ - {"node_type": "metro", "name": "A", "x": 1.0, "y": 2.0}, - {"node_type": "metro+highway", "name": "B", "x": 3, "y": 4}, - {"node_type": "highway", "name": "H", "x": 5, "y": 6}, # excluded - {"node_type": "metro", "name": "", "x": 7, "y": 8}, # no name - {"node_type": "metro", "name": "C", "x": "bad", "y": 9}, # bad coords - ] - } - mp = _build_ig_coord_map(ig) - assert mp == {"A": (1.0, 2.0), "B": (3.0, 4.0)} +def test_build_ig_coord_map_extracts_and_rejects_invalid_coordinates(): + assert _build_ig_coord_map( + {"nodes": [{"node_type": "metro", "name": "A", "x": 1, "y": 2}]} + ) == {"A": (1.0, 2.0)} + with pytest.raises(ValueError): + _build_ig_coord_map( + {"nodes": [{"node_type": "metro", "name": "A", "x": "bad", "y": 2}]} + ) def test_float_or_nan() -> None: @@ -34,8 +32,5 @@ def test_node_hw_from_attrs() -> None: # Missing comp comp2, count2 = _node_hw_from_attrs({}) assert comp2 is None and count2 == 0.0 - # Bad count → default 1.0 - comp3, count3 = _node_hw_from_attrs( - {"hardware": {"component": "X", "count": "bad"}} - ) - assert comp3 == "X" and count3 == 1.0 + with pytest.raises(ValueError): + _node_hw_from_attrs({"hardware": {"component": "X", "count": "bad"}}) diff --git a/topogen/__init__.py b/topogen/__init__.py index de49309..f3cce26 100644 --- a/topogen/__init__.py +++ b/topogen/__init__.py @@ -2,10 +2,12 @@ from . import visualization from .config import TopologyConfig +from .context import RunContext from .integrated_graph import build_integrated_graph, load_from_json, save_to_json __all__ = [ "TopologyConfig", + "RunContext", "build_integrated_graph", "load_from_json", "save_to_json", diff --git a/topogen/blueprint_viz.py b/topogen/blueprint_viz.py index ab2649f..85b1846 100644 --- a/topogen/blueprint_viz.py +++ b/topogen/blueprint_viz.py @@ -1,218 +1,81 @@ -"""Build abstract blueprint views and concrete site layouts for rendering. - -Abstract views use blueprint ``nodes``, ``links``, and ``expand.vars``/``expand.mode`` -fields. They aggregate links by group and record intra-group links as labels -and optional self-loop markers. -""" +"""Aggregate and position the devices already expanded by NetGraph.""" from __future__ import annotations -import re +from collections import defaultdict from dataclasses import dataclass -from typing import Any, Iterable import networkx as nx +from ngraph import Network @dataclass(frozen=True, slots=True) class AbstractView: - """Container for the abstract view. - - Attributes: - graph: MultiDiGraph with group nodes and inter-group edges. - node_labels: Mapping of node -> label text to draw. - edge_labels: Mapping of (u, v, key) -> label text to draw. - self_loops: List of (group, label) describing intra-group connectivity - to be rendered as self-loop markers in the abstract view. - """ - graph: nx.MultiDiGraph node_labels: dict[str, str] edge_labels: dict[tuple[str, str, int], str] self_loops: list[tuple[str, str]] -def _first_path_component(selector: str) -> str: - """Return the first path component of a selector. - - Examples: - "/G${g}" -> "G${g}"; "G1/G1_r1" -> "G1". - """ - - s = str(selector or "").strip() - if s.startswith("/"): - s = s[1:] - return s.split("/", 1)[0] - - -# NetGraph placeholder syntax: ``$var`` or ``${var}``. -_VAR_PATTERN = re.compile(r"\$\{([A-Za-z_][A-Za-z0-9_]*)\}|\$([A-Za-z_][A-Za-z0-9_]*)") - - -def _extract_vars(template: str) -> list[str]: - """Return ``$var``/``${var}`` names in ``template`` in order.""" - - return [m.group(1) or m.group(2) for m in _VAR_PATTERN.finditer(template)] - - -def _iter_assignments( - expand_vars: dict[str, Iterable[Any]] | None, - var_names: list[str], - expansion_mode: str | None, -) -> list[dict[str, Any]]: - """Expand variable values by index (default ``zip``) or Cartesian product. - - Zip stops at the shortest input. No variables, or a variable without values, - returns one empty assignment. - """ - - if not var_names: - return [{}] - - ev: dict[str, list[Any]] = { - name: list((expand_vars or {}).get(name, [])) for name in var_names - } - - # If any variable lacks values, nothing can be expanded → single empty assignment - if any(len(v) == 0 for v in ev.values()): - return [{}] - - if (expansion_mode or "zip").lower() == "zip": - # Align by the minimum available length to avoid IndexError if mismatched - n = min(len(v) for v in ev.values()) - result: list[dict[str, Any]] = [] - for i in range(n): - result.append({name: ev[name][i] for name in var_names}) - return result - - from itertools import product - - keys = list(var_names) - vals = [ev[k] for k in keys] - return [dict(zip(keys, comb, strict=False)) for comb in product(*vals)] - - -def _subst(template: str, values: dict[str, Any]) -> str: - """Replace known ``$var``/``${var}`` placeholders, leaving unknown ones intact.""" - - def _replace(match: re.Match[str]) -> str: - val = values.get(match.group(1) or match.group(2)) - return match.group(0) if val is None else str(val) - - return _VAR_PATTERN.sub(_replace, template) - - def build_abstract_view( - blueprint_def: dict[str, Any], *, include_self_loops: bool = True + net: Network, selected_site_path: str, *, include_self_loops: bool = True ) -> AbstractView: - """Build a group-level view from blueprint ``nodes`` and ``links``. + """Aggregate actual devices by blueprint parent, and links by group pair. - Within each link rule, inter-group edges are deduplicated by unordered pair. - Same-group rules become node notes and, if requested, self-loop markers. - Raises ValueError when ``nodes`` is missing or is not a mapping. + Counts and capacity totals come from the same resolved network as the concrete + panel. Nested blueprints, selector dictionaries, and variable expansion need + no second DSL interpreter here. Self-loop labels show internal capacity. """ - - if not isinstance(blueprint_def, dict): - raise ValueError("blueprint_def must be a mapping") - groups = blueprint_def.get("nodes") - if not isinstance(groups, dict): - raise ValueError("blueprint_def must include a 'nodes' mapping") - - abstract = nx.MultiDiGraph() - - node_labels: dict[str, str] = {} - for gname, gdef in groups.items(): - count = int((gdef or {}).get("count", 0)) - role = str(((gdef or {}).get("attrs", {}) or {}).get("role", "")) - lbl = f"{gname}\nN={count}" + (f"\nrole={role}" if role else "") - abstract.add_node(gname) - node_labels[gname] = lbl - - def _append_node_note(group_name: str, text: str) -> None: - base = node_labels.get(group_name, group_name) - if text and text not in base: - node_labels[group_name] = base + f"\n{text}" - - edge_labels: dict[tuple[str, str, int], str] = {} - self_loops: list[tuple[str, str]] = [] - - for adj in blueprint_def.get("links", []) or []: - src_sel = str(adj.get("source", "")) - dst_sel = str(adj.get("target", "")) - pattern = str(adj.get("pattern", "")) - expand_block = adj.get("expand") or {} - expand_vars = expand_block.get("vars") if isinstance(expand_block, dict) else {} - expand_vars = expand_vars or {} - expansion_mode = str( - (expand_block.get("mode") if isinstance(expand_block, dict) else None) - or "zip" - ) - - attrs = adj.get("attrs", {}) or {} - cap_val = attrs.get("target_capacity", adj.get("capacity")) - cap_str = f"{float(cap_val):,.0f}" if cap_val is not None else "" - edge_label_text = (pattern if pattern else "") + ( - f"\n{cap_str}" if cap_str else "" + nodes, _, links = collect_concrete_site(net, selected_site_path) + prefix = selected_site_path + "/" + parents = { + node: node.removeprefix(prefix).rsplit("/", 1)[0] + if "/" in node.removeprefix(prefix) + else "devices" + for node in nodes + } + members: dict[str, list[str]] = defaultdict(list) + for node, parent in parents.items(): + members[parent].append(node) + graph = nx.MultiDiGraph() + labels = {} + for group, names in sorted(members.items()): + roles = sorted( + { + net.nodes[name].attrs["role"] + for name in names + if "role" in net.nodes[name].attrs + } ) - - # Extract only the group part of selectors and expand variables jointly - src_group_tpl = _first_path_component(src_sel) - dst_group_tpl = _first_path_component(dst_sel) - - involved_vars = list( - dict.fromkeys(_extract_vars(src_group_tpl) + _extract_vars(dst_group_tpl)) + graph.add_node(group) + labels[group] = f"{group}\nN={len(names)}" + ( + f"\nrole={','.join(roles)}" if roles else "" ) - - assignments = _iter_assignments(expand_vars, involved_vars, expansion_mode) - - pairs: list[tuple[str, str]] = [] - for assign in assignments: - su = _subst(src_group_tpl, assign) - sv = _subst(dst_group_tpl, assign) - pairs.append((su, sv)) - - if all(u == v for u, v in pairs): - # Intra-group adjacency. Keep label note and optionally emit self-loop. - for u, _v in pairs: - _append_node_note( - u, (pattern if pattern else "") + (f" {cap_str}" if cap_str else "") - ) - if include_self_loops: - # One loop per group is sufficient visually; deduplicate. - if (u, edge_label_text) not in self_loops: - self_loops.append((u, edge_label_text)) - continue - - seen: set[tuple[str, str]] = set() - for u, v in pairs: - if u == v: - continue - _key_pair = (u, v) if (u, v) not in seen else None - # For directionality, keep as given. Also add the reverse marker to - # the set so we do not repeat the same undirected pair when inputs - # contain both directions. - if (u, v) in seen or (v, u) in seen: - continue - seen.add((u, v)) - seen.add((v, u)) - k = abstract.add_edge(u, v) - edge_labels[(u, v, k)] = edge_label_text - - return AbstractView( - graph=abstract, - node_labels=node_labels, - edge_labels=edge_labels, - self_loops=self_loops, - ) + capacities: dict[tuple[str, str], float] = defaultdict(float) + for source, target, capacity in links: + a, b = sorted((parents[source], parents[target])) + capacities[a, b] += capacity + edges = {} + loops = [] + for (a, b), capacity in sorted(capacities.items()): + text = f"{capacity:,.0f} Gbps" + if a == b: + labels[a] += f"\ninternal: {text}" + if include_self_loops: + loops.append((a, text)) + else: + key = graph.add_edge(a, b) + edges[a, b, key] = text + return AbstractView(graph, labels, edges, loops) def collect_concrete_site( - net: Any, selected_site_path: str + net: Network, selected_site_path: str ) -> tuple[list[str], dict[str, tuple[float, float]], list[tuple[str, str, float]]]: """Return node names, layout positions, and internal links for one site. - Positions cluster nodes by local name prefix around a unit circle, using - seeded jitter. Links are ``(source, target, capacity)`` tuples. + Positions use evenly spaced columns for actual blueprint parent groups. + Links are ``(source, target, capacity)`` tuples. """ def _site_head(name: str) -> str: @@ -220,61 +83,24 @@ def _site_head(name: str) -> str: return "/".join(parts[:2]) if len(parts) >= 2 else str(name) internal_nodes: list[str] = [] - for node in getattr(net, "nodes", {}).values(): # type: ignore[union-attr] - try: - nname = str(node.name) - except Exception: - nname = str(node) + for node in net.nodes.values(): + nname = node.name if _site_head(nname) == selected_site_path: internal_nodes.append(nname) - # Group heuristic based on local suffix before first digit - import math as _m - import re as _re - - import numpy as _np - - def _group_of(node_name: str) -> str: - tail = node_name.split("/", 2)[-1] - m = _re.match(r"([A-Za-z_]+)", tail) - return m.group(1) if m else tail - groups_concrete: dict[str, list[str]] = {} - for n in internal_nodes: - g = _group_of(n) - groups_concrete.setdefault(g, []).append(n) - - rng = _np.random.default_rng(7) - K = max(1, len(groups_concrete)) - R_group = 1.0 - r_node = 0.25 - group_angles = { - g: (2.0 * _m.pi * i) / float(K) for i, g in enumerate(sorted(groups_concrete)) - } + for name in internal_nodes: + parent = name.rsplit("/", 1)[0] + groups_concrete.setdefault(parent, []).append(name) node_pos: dict[str, tuple[float, float]] = {} - for g, angle in group_angles.items(): - gx = R_group * _m.cos(angle) - gy = R_group * _m.sin(angle) - members = groups_concrete[g] - n = len(members) - if n == 1: - node_pos[members[0]] = (gx, gy) - else: - for j, nn in enumerate(sorted(members)): - theta = (2.0 * _m.pi * j) / float(n) - rr = r_node * (0.85 + 0.3 * float(rng.random())) - node_pos[nn] = (gx + rr * _m.cos(theta), gy + rr * _m.sin(theta)) + for column, group in enumerate(sorted(groups_concrete)): + members = sorted(groups_concrete[group]) + for row, name in enumerate(members): + node_pos[name] = (float(column), row - (len(members) - 1) / 2) internal_links: list[tuple[str, str, float]] = [] - for link in getattr(net, "links", {}).values(): # type: ignore[union-attr] - try: - s_obj = link.source - t_obj = link.target - s = str(getattr(s_obj, "name", s_obj)) - t = str(getattr(t_obj, "name", t_obj)) - cap = float(getattr(link, "capacity", 0.0) or 0.0) - except Exception: - continue + for link in net.links.values(): + s, t, cap = link.source, link.target, link.capacity if _site_head(s) == selected_site_path and _site_head(t) == selected_site_path: internal_links.append((s, t, cap)) diff --git a/topogen/blueprints_lib.py b/topogen/blueprints_lib.py index d0bb2a0..2edd1b6 100644 --- a/topogen/blueprints_lib.py +++ b/topogen/blueprints_lib.py @@ -6,10 +6,9 @@ from __future__ import annotations from copy import deepcopy -from pathlib import Path from typing import Any -import yaml +from topogen.library_io import load_user_library _BUILTIN_BLUEPRINTS: dict[str, dict[str, Any]] = { "DCRegion": { @@ -410,27 +409,9 @@ } -def _load_user_library(file_name: str) -> dict[str, Any]: - """Read a name-to-definition mapping from ``cwd/lib/`` if present.""" - lib_path = Path.cwd() / "lib" / file_name - if not lib_path.exists(): - return {} - - try: - with lib_path.open("r", encoding="utf-8") as f: - data = yaml.safe_load(f) or {} - except Exception as exc: # noqa: BLE001 - raise ValueError(f"Failed to parse YAML: {lib_path}") from exc - - if not isinstance(data, dict): - raise ValueError(f"User library YAML must be a mapping: {lib_path}") - - return data - - def get_builtin_blueprints() -> dict[str, dict[str, Any]]: """Return built-in blueprints with user overrides applied.""" blueprints = deepcopy(_BUILTIN_BLUEPRINTS) - user_blueprints = _load_user_library("blueprints.yml") + user_blueprints = load_user_library("blueprints.yml") blueprints.update(user_blueprints) return blueprints diff --git a/topogen/cli.py b/topogen/cli.py index 47c906a..4d8ffe5 100644 --- a/topogen/cli.py +++ b/topogen/cli.py @@ -3,12 +3,15 @@ from __future__ import annotations import argparse +import os import sys import time -from contextlib import contextmanager +from contextlib import contextmanager, redirect_stdout from pathlib import Path +from tempfile import NamedTemporaryFile from topogen.config import TopologyConfig +from topogen.context import RunContext from topogen.log_config import get_logger logger = get_logger(__name__) @@ -19,14 +22,14 @@ def Timer(description: str): """Print and log elapsed time on completion or failure.""" print(f"🔄 {description}...") logger.info(f"Starting {description}") - start = time.time() + start = time.perf_counter() try: yield - elapsed = time.time() - start + elapsed = time.perf_counter() - start print(f"✅ {description} (completed in {elapsed:.1f}s)") logger.info(f"Completed {description} in {elapsed:.1f}s") except Exception as e: - elapsed = time.time() - start + elapsed = time.perf_counter() - start print(f"❌ {description} (failed after {elapsed:.1f}s)") logger.error(f"Failed {description} after {elapsed:.1f}s: {e}") raise @@ -55,48 +58,20 @@ def build_command(args: argparse.Namespace) -> None: try: config_path = Path(args.config) config_obj = _load_config(config_path) - # Compute output directory and scenario path. - # If -o is a directory, write '_scenario.yml' inside it. - # If -o is a file, use it directly and treat its parent as output dir. - prefix_path = getattr(config_obj, "_source_path", config_path) - stem = Path(prefix_path).stem if isinstance(prefix_path, Path) else "scenario" - if getattr(args, "output", None): - output_arg = Path(args.output) - if output_arg.suffix.lower() in {".yml", ".yaml"}: - output_dir = output_arg.parent - output_path = output_arg - else: - output_dir = output_arg - output_path = output_dir / f"{stem}_scenario.yml" + output_arg = Path(args.output) if args.output else Path.cwd() + if output_arg.suffix.lower() in {".yml", ".yaml"}: + output_dir, output_path = output_arg.parent, output_arg else: - output_dir = Path.cwd() - output_path = output_dir / f"{stem}_scenario.yml" - # Persist chosen output directory on config for downstream artefacts - try: - output_dir.mkdir(parents=True, exist_ok=True) - except Exception: - pass - try: - config_obj._output_dir = output_dir # type: ignore[attr-defined] - except Exception: - pass - - # Attach optional debug directory to config for downstream use - if getattr(args, "debug_dir", None): - try: - debug_dir = Path(args.debug_dir) - debug_dir.mkdir(parents=True, exist_ok=True) - config_obj._debug_dir = debug_dir - # Provide a stable stem to downstream exporters - config_obj._source_stem = Path(config_path).stem - except Exception: - # Non-fatal: continue without debug directory - pass + output_dir = output_arg + output_path = output_dir / f"{config_path.stem}_scenario.yml" + context = RunContext( + output_dir, + config_path.stem, + Path(args.debug_dir) if args.debug_dir else None, + ) with Timer("Topology generation pipeline"): - scenario_yaml = _run_pipeline( - config_obj, output_path, print_yaml=args.print - ) + scenario_yaml = _run_pipeline(config_obj, output_path, context=context) if args.print: print("\n" + "=" * 60) @@ -121,127 +96,75 @@ def build_command(args: argparse.Namespace) -> None: sys.exit(1) # Runtime error +def _write_scenario(path: Path, content: str) -> None: + """Replace a scenario atomically, only after it has passed validation.""" + path.parent.mkdir(parents=True, exist_ok=True) + temporary: Path | None = None + try: + with NamedTemporaryFile( + mode="w", + encoding="utf-8", + dir=path.parent, + prefix=f".{path.name}.", + delete=False, + ) as stream: + temporary = Path(stream.name) + stream.write(content) + temporary.replace(path) + finally: + if temporary is not None: + temporary.unlink(missing_ok=True) + + def _run_pipeline( - config: TopologyConfig, output_path: Path, print_yaml: bool = False + config: TopologyConfig, output_path: Path, *, context: RunContext ) -> str: - """Build and write scenario YAML from the saved corridor graph. - - With ``print_yaml``, also print the YAML and skip validation. Otherwise, - validate the scenario and raise ValueError on issues (CLI exit code 3). - A missing graph exits with code 1. - """ + """Build, validate, then publish scenario YAML from the saved corridor graph.""" from topogen import load_from_json - from topogen.scenario_builder import build_scenario - - # Check for integrated graph in configured output dir (fallback to CWD) - source_path = getattr(config, "_source_path", None) - prefix = Path(source_path).stem if isinstance(source_path, Path) else "scenario" - output_dir = getattr(config, "_output_dir", None) - base_dir = Path(output_dir) if isinstance(output_dir, (str, Path)) else Path.cwd() - graph_path = base_dir / f"{prefix}_integrated_graph.json" - if not graph_path.exists(): - print("❌ No integrated graph found!") - print(" Run generation first: python -m topogen generate") - sys.exit(1) - - print("Topology Generation Pipeline") - print("=" * 50) - - print("🔄 Loading integrated graph...") - graph, crs = load_from_json(graph_path) - - print(f"📊 Graph loaded: {len(graph.nodes):,} nodes, {len(graph.edges):,} edges") + from topogen.scenario import build_scenario + from topogen.validation import validate_scenario_yaml - metro_nodes = [ - n - for n, d in graph.nodes(data=True) - if d.get("node_type") in ["metro", "metro+highway"] - ] - highway_nodes = [n for n in graph.nodes() if n not in metro_nodes] - print(f" Metro nodes: {len(metro_nodes)}") - print(f" Highway nodes: {len(highway_nodes)}") - - output_path.parent.mkdir(parents=True, exist_ok=True) - - with Timer("Generate NetGraph scenario"): - scenario_yaml = build_scenario(graph, config) - - with Timer(f"Write scenario to {output_path}"): - with open(output_path, "w") as f: - f.write(scenario_yaml) - - print(f"\n📄 Scenario written to: {output_path}") - - if not print_yaml: - from topogen.validation import validate_scenario_yaml - - print("🔄 Validating generated scenario...") - comp_obj = getattr(config, "components", None) - hw_map = ( - getattr(comp_obj, "hw_component", None) if comp_obj is not None else None + graph_path = context.path("integrated_graph.json") + if not graph_path.exists(): + raise FileNotFoundError( + f"No integrated graph found: {graph_path}. Run topogen generate first." ) - optics_map = getattr(comp_obj, "optics", None) if comp_obj is not None else None - # Be strict about unexpected types to avoid silently disabling audits - if hw_map is not None and not isinstance(hw_map, dict): - raise ValueError( - "'components.hw_component' must be a mapping when provided" - ) - if optics_map is not None and not isinstance(optics_map, dict): - raise ValueError("'components.optics' must be a mapping when provided") - issues = validate_scenario_yaml( - scenario_yaml, - integrated_graph_path=graph_path, - run_ngraph=True, - hw_component_map=hw_map, - optics_map=optics_map, + graph, crs = load_from_json(graph_path) + if crs != config.projection.target_crs: + raise ValueError( + f"Saved graph CRS {crs} differs from configured {config.projection.target_crs}; regenerate the integrated graph" ) - if issues: - print("❌ Scenario validation found issues:") - for s in issues: - print(f" - {s}") - print(" Scenario file generated but has issues") - raise ValueError("Scenario validation failed") - else: - print("✅ Scenario validation passed") - + print(f"Graph loaded: {len(graph.nodes):,} metros, {len(graph.edges):,} corridors") + with Timer("Generate NetGraph scenario"): + scenario_yaml = build_scenario(graph, config, context=context) + issues = validate_scenario_yaml( + scenario_yaml, + integrated_graph_path=graph_path, + hw_component_map=config.components.hw_component, + optics_map=config.components.optics, + ) + if issues: + raise ValueError("Scenario validation failed:\n" + "\n".join(issues)) + print("✅ Scenario validation passed") + _write_scenario(output_path, scenario_yaml) + print(f"Scenario written to: {output_path}") return scenario_yaml -def _run_generation(config: TopologyConfig) -> None: - """Generate and save the corridor graph in the configured output directory.""" +def _run_generation(config: TopologyConfig, context: RunContext) -> None: + """Generate and save the corridor graph at the explicit artifact destination.""" from topogen import build_integrated_graph, save_to_json - print("Integrated Graph Generation Pipeline") - print("=" * 50) - - # Artefacts in configured output directory (fallback to CWD) - cfg_out = getattr(config, "_output_dir", None) - output_dir = Path(cfg_out) if isinstance(cfg_out, (str, Path)) else Path.cwd() - - source_path = getattr(config, "_source_path", None) - prefix = Path(source_path).stem if isinstance(source_path, Path) else "scenario" - graph_output = output_dir / f"{prefix}_integrated_graph.json" - - print(f" Urban areas: {config.clustering.metro_clusters}") - print(f" UAC data: {config.data_sources.uac_polygons}") - print(f" Highway data: {config.data_sources.tiger_roads}") - + config.validate() + context.output_dir.mkdir(parents=True, exist_ok=True) with Timer("Generate a metro-to-metro corridor graph"): - graph = build_integrated_graph(config) - - with Timer("Save integrated graph"): - save_to_json( - graph, graph_output, config.projection.target_crs, config.output.formatting - ) - - print("\n🎉 Generation complete!") - print(f"📁 Integrated graph: {graph_output}") - print(f"📊 Graph summary: {len(graph.nodes):,} nodes, {len(graph.edges):,} edges") - print("🔗 Ready for topology generation with:") - print( - f" python -m topogen build -c {getattr(config, '_source_path', 'config.yml')}" - f" -o {output_dir}" + graph = build_integrated_graph(config, context=context) + graph_output = context.path("integrated_graph.json") + save_to_json( + graph, graph_output, config.projection.target_crs, config.output.formatting ) + print(f"Integrated graph: {graph_output}") + print(f"Graph summary: {len(graph.nodes):,} nodes, {len(graph.edges):,} edges") def generate_command(args: argparse.Namespace) -> None: @@ -249,16 +172,11 @@ def generate_command(args: argparse.Namespace) -> None: try: config_path = Path(args.config) config_obj = _load_config(config_path) - # If output directory provided, persist on config - if getattr(args, "output", None): - try: - out_dir = Path(args.output) - out_dir.mkdir(parents=True, exist_ok=True) - config_obj._output_dir = out_dir # type: ignore[attr-defined] - except Exception: - pass - - _run_generation(config_obj) + context = RunContext( + Path(args.output) if args.output else Path.cwd(), + config_path.stem, + ) + _run_generation(config_obj, context) except Exception as e: print(f"❌ ERROR: {e}") @@ -280,12 +198,14 @@ def info_command(args: argparse.Namespace) -> None: print("=" * 20) print(f"UAC polygons: {config_obj.data_sources.uac_polygons}") print(f"TIGER roads: {config_obj.data_sources.tiger_roads}") + print(f"CONUS boundary: {config_obj.data_sources.conus_boundary}") print("\nData Availability") print("=" * 20) uac_path = Path(config_obj.data_sources.uac_polygons) tiger_path = Path(config_obj.data_sources.tiger_roads) + boundary_path = Path(config_obj.data_sources.conus_boundary) uac_status = "✅" if uac_path.exists() else "❌" tiger_status = "✅" if tiger_path.exists() else "❌" @@ -293,7 +213,11 @@ def info_command(args: argparse.Namespace) -> None: print(f"UAC data: {uac_status} {uac_path}") print(f"TIGER roads: {tiger_status} {tiger_path}") - if not uac_path.exists() or not tiger_path.exists(): + print( + f"CONUS boundary: {'✅' if boundary_path.exists() else '❌'} {boundary_path}" + ) + + if not all(path.exists() for path in (uac_path, tiger_path, boundary_path)): print("\n⚠️ Missing data files - download required before generation") except Exception as e: @@ -322,6 +246,7 @@ def main() -> None: help="Suppress console output (logs only)", ) + parser.set_defaults(func=None) subparsers = parser.add_subparsers(dest="command", help="Available commands") # Build command @@ -343,15 +268,12 @@ def main() -> None: build_parser.add_argument( "--print", action="store_true", - help="Also print YAML to stdout and skip scenario validation", + help="Also print the validated scenario YAML to stdout", ) build_parser.add_argument( "--debug-dir", default=None, - help=( - "Optional directory to write debug artifacts (e.g., traffic matrix " - "internals as JSON) when -v is enabled" - ), + help=("Optional directory to write the generated traffic matrices as JSON"), ) build_parser.set_defaults(func=build_command) @@ -403,17 +325,15 @@ def main() -> None: set_global_log_level(log_level) - # Suppress print output if --quiet is set - if args.quiet: - import builtins - - builtins.print = lambda *args, **kwargs: None - - if not hasattr(args, "func") or args.func is None: + if args.func is None: parser.print_help() sys.exit(1) - args.func(args) + if args.quiet: + with open(os.devnull, "w") as sink, redirect_stdout(sink): + args.func(args) + else: + args.func(args) if __name__ == "__main__": diff --git a/topogen/components_lib.py b/topogen/components_lib.py index bd1ce28..6f7d874 100644 --- a/topogen/components_lib.py +++ b/topogen/components_lib.py @@ -6,10 +6,9 @@ from __future__ import annotations from copy import deepcopy -from pathlib import Path from typing import Any -import yaml +from topogen.library_io import load_user_library _BUILTIN_COMPONENTS: dict[str, dict[str, Any]] = { # Router Chassis Components @@ -76,62 +75,9 @@ } -def _load_user_library(file_name: str) -> dict[str, Any]: - """Read a mapping from ``cwd/lib/``, or return {} if absent. - - Raises ValueError for invalid YAML or a non-mapping value. - """ - lib_path = Path.cwd() / "lib" / file_name - if not lib_path.exists(): - return {} - - try: - with lib_path.open("r", encoding="utf-8") as f: - data = yaml.safe_load(f) or {} - except Exception as exc: # noqa: BLE001 - provide clear context - raise ValueError(f"Failed to parse YAML: {lib_path}") from exc - - if not isinstance(data, dict): - raise ValueError(f"User library YAML must be a mapping: {lib_path}") - - return data - - def get_builtin_components() -> dict[str, dict[str, Any]]: """Return a copy of built-in components with user overrides applied.""" components = deepcopy(_BUILTIN_COMPONENTS) - user_components = _load_user_library("components.yml") + user_components = load_user_library("components.yml") components.update(user_components) return components - - -def get_builtin_component(name: str) -> dict[str, Any]: - """Return a component from the merged library; raise KeyError if absent.""" - if name not in _BUILTIN_COMPONENTS: - available = list(_BUILTIN_COMPONENTS.keys()) - raise KeyError(f"Component '{name}' not found. Available: {available}") - - return deepcopy(_BUILTIN_COMPONENTS[name]) - - -def list_builtin_component_names() -> list[str]: - """List component names from the merged library.""" - return sorted(_BUILTIN_COMPONENTS.keys()) - - -def get_components_by_type(component_type: str) -> dict[str, dict[str, Any]]: - """Filter the merged library by component type, such as chassis or optic.""" - return { - name: deepcopy(comp) - for name, comp in _BUILTIN_COMPONENTS.items() - if comp.get("component_type") == component_type - } - - -def get_components_by_role(role: str) -> dict[str, dict[str, Any]]: - """Filter the merged library by supported role.""" - return { - name: deepcopy(comp) - for name, comp in _BUILTIN_COMPONENTS.items() - if comp.get("attrs", {}).get("role") == role - } diff --git a/topogen/config.py b/topogen/config.py index 4ea6a04..9a456eb 100644 --- a/topogen/config.py +++ b/topogen/config.py @@ -3,12 +3,17 @@ from __future__ import annotations import json -from dataclasses import dataclass, field +import math +from dataclasses import asdict, dataclass, field +from decimal import Decimal, InvalidOperation +from functools import lru_cache from importlib import resources from pathlib import Path from typing import Any +import jsonschema import yaml +from pyproj import CRS from topogen.log_config import get_logger from topogen.naming import metro_slug @@ -34,33 +39,39 @@ def _validate_flow_policy_names(policies: dict[int, str]) -> None: ) -def _normalize_int(value: Any, field: str) -> int: - """Return an int from possibly underscored or float-like input. +def _check_finite_numbers(value: Any, path: str = "config") -> None: + """YAML accepts NaN/infinity, while JSON Schema bounds do not reject NaN.""" + if isinstance(value, float) and not math.isfinite(value): + raise ValueError(f"{path} must be finite") + if isinstance(value, dict): + for key, child in value.items(): + _check_finite_numbers(child, f"{path}.{key}") + elif isinstance(value, list): + for index, child in enumerate(value): + _check_finite_numbers(child, f"{path}[{index}]") - Accepts ints, floats, or strings like "12_800" or "3200.0". Raises a - ValueError with field context on failure. - """ + +def _normalize_int(value: Any, field: str) -> int: + """Parse exact, finite integer quantities; never truncate or accept booleans.""" try: if isinstance(value, bool): - return int(value) + raise ValueError("boolean is not an integer quantity") if isinstance(value, int): return value - if isinstance(value, float): - return int(value) if isinstance(value, str): - s = value.strip().replace("_", "") - if not s: - raise ValueError("empty string") - # Support simple decimal strings by truncation to int - if "." in s: - return int(float(s)) - return int(s) - return int(value) # type: ignore[arg-type] - except Exception as exc: # noqa: BLE001 + exact = Decimal(value.replace("_", "")) + if not exact.is_finite() or exact != exact.to_integral_value(): + raise ValueError("expected a finite integer") + return int(exact) + parsed = float(value) + if not math.isfinite(parsed) or not parsed.is_integer(): + raise ValueError("expected a finite integer") + return int(parsed) + except (TypeError, ValueError, OverflowError, InvalidOperation) as exc: raise ValueError(f"Invalid integer for {field}: {value!r}") from exc -@dataclass +@dataclass(slots=True) class DataSources: """Paths to Census urban areas, TIGER/Line roads, and state boundaries.""" @@ -75,33 +86,44 @@ def __post_init__(self) -> None: self.conus_boundary = Path(self.conus_boundary) -@dataclass +@lru_cache(maxsize=16) +def _validate_projection(target_crs: str) -> None: + crs = CRS.from_user_input(target_crs) + if ( + not crs.is_projected + or len(crs.axis_info) != 2 + or any(axis.unit_conversion_factor != 1.0 for axis in crs.axis_info) + ): + raise ValueError( + "projection.target_crs must be a projected CRS with metre units" + ) + + +@dataclass(slots=True) class ProjectionConfig: """Target coordinate reference system for spatial operations.""" target_crs: str = "EPSG:5070" + def __post_init__(self) -> None: + _validate_projection(self.target_crs) -@dataclass -class HighwayProcessingConfig: - """Highway class filtering, coordinate snapping, and graph cleanup settings. - The pipeline does not use ``min_cycle_nodes`` or ``validation_sample_size``. - """ +@dataclass(slots=True) +class HighwayProcessingConfig: + """Highway filtering, coordinate snapping, and graph cleanup settings.""" min_edge_length_km: float = 0.05 snap_precision_m: float = 10.0 highway_classes: list[str] = field( default_factory=lambda: ["S1100", "S1200"] ) # TIGER highway classes to keep - min_cycle_nodes: int = 3 filter_largest_component: bool = ( True # Keep only largest connected component if highway graph is disconnected ) - validation_sample_size: int = 5 -@dataclass +@dataclass(slots=True) class RiskGroupsConfig: """Corridor risk-group naming and metro-radius exclusion settings.""" @@ -112,14 +134,14 @@ class RiskGroupsConfig: ) -@dataclass +@dataclass(slots=True) class CorridorsConfig: """Metro adjacency, path discovery, and corridor risk-group settings.""" k_paths: int = 1 # Maximum number of shortest simple paths per adjacent metro pair k_nearest: int = 3 # Number of nearest neighbors per metro for adjacency # Euclidean threshold (km) for k-NN adjacency between metros. - # Computed from centroid separation in target CRS; does not limit path length. + # Computed between representative points in target CRS; does not limit path length. max_edge_km: float = 600.0 # Path-length threshold (km) along the discovered corridor over the highway graph. # Skip a metro pair when all candidate paths exceed this length. @@ -127,7 +149,7 @@ class CorridorsConfig: risk_groups: RiskGroupsConfig = field(default_factory=RiskGroupsConfig) -@dataclass +@dataclass(slots=True) class ValidationConfig: """Distance, degree, and connectivity thresholds for graph validation.""" @@ -140,9 +162,9 @@ class ValidationConfig: ) -@dataclass +@dataclass(slots=True) class LinkParams: - """Capacity budget, cost, and attributes for a site-to-site adjacency. + """Capacity budget, minimum routing cost, and attributes for an adjacency. ``match`` is applied to both endpoints during NetGraph expansion. """ @@ -152,9 +174,9 @@ class LinkParams: attrs: dict[str, Any] = field(default_factory=dict) match: dict[str, Any] = field(default_factory=dict) # Unordered role-pairs allowed for this link type. Examples: - # ["core|core", "core|leaf"] or [["core", "core"], ["core", "leaf"]] + # ["core|core", "core|leaf"] # The pipeline will auto-render a symmetric match from the union of roles. - role_pairs: list[Any] = field(default_factory=list) + role_pairs: list[str] = field(default_factory=list) # Optional striping configuration for controlled device-group partitioning # during adjacency creation. Example: {"width": 4} striping: dict[str, Any] = field(default_factory=dict) @@ -164,7 +186,7 @@ class LinkParams: mode: str = "mesh" -@dataclass +@dataclass(slots=True) class BuildDefaults: """Site counts, blueprints, and link settings used unless a metro overrides them.""" @@ -201,18 +223,18 @@ class BuildDefaults: ) -@dataclass +@dataclass(slots=True) class BuildConfig: """Scenario build defaults, per-metro overrides, and traffic sizing settings.""" build_defaults: BuildDefaults = field(default_factory=BuildDefaults) build_overrides: dict[str, dict[str, Any]] = field(default_factory=dict) - tm_sizing: "BuildTmSizingConfig" = field( # type: ignore[name-defined] + tm_sizing: "BuildTmSizingConfig" = field( default_factory=lambda: BuildTmSizingConfig() ) -@dataclass +@dataclass(slots=True) class BuildTmSizingConfig: """Capacity sizing from traffic routed on a collapsed metro graph. @@ -227,8 +249,6 @@ class BuildTmSizingConfig: beta_intra_pop: Intra-metro capacity multiplier relative to the smaller egress of the two PoPs. flow_placement: ``EQUAL_BALANCED`` or ``PROPORTIONAL`` splitting. - edge_select: Stored setting; sizing uses all minimum-cost edges - regardless of this value. respect_min_base_capacity: Keep capacities at least at their configured values. """ @@ -239,54 +259,30 @@ class BuildTmSizingConfig: alpha_dc_to_pop: float = 1.2 beta_intra_pop: float = 0.8 flow_placement: str = "EQUAL_BALANCED" - edge_select: str = "ALL_MIN_COST" respect_min_base_capacity: bool = True -@dataclass -class ComponentAssignment: - """Platform and optic names assigned to a network role.""" - - hw_component: str = "" - optics: str = "" - - -@dataclass -class ComponentAssignments: - """Component assignments for network roles.""" - - spine: ComponentAssignment = field(default_factory=ComponentAssignment) - leaf: ComponentAssignment = field(default_factory=ComponentAssignment) - core: ComponentAssignment = field(default_factory=ComponentAssignment) - dc: ComponentAssignment = field(default_factory=ComponentAssignment) - - -@dataclass +@dataclass(slots=True) class ComponentsConfig: """Role-to-platform and role-pair-to-optic assignments. Definitions come from the built-ins and ``cwd/lib/components.yml``. """ - assignments: ComponentAssignments = field(default_factory=ComponentAssignments) # hw_component: role -> platform component name - # optics: "source_role|target_role" -> optic component name + # optics: "local_role->remote_role" -> local optic component name hw_component: dict[str, str] = field(default_factory=dict) optics: dict[str, str] = field(default_factory=dict) -@dataclass +@dataclass(slots=True) class FailurePolicyAssignments: - """Failure-policy selection. - - Scenario assembly uses ``default``; ``scenario_overrides`` is stored but not applied. - """ + """Failure-policy selection.""" default: str = "single_random_link_failure" - scenario_overrides: dict[str, dict[str, str]] = field(default_factory=dict) -@dataclass +@dataclass(slots=True) class FailurePoliciesConfig: """Select policies from the built-ins and ``cwd/lib/failure_policies.yml``.""" @@ -295,25 +291,21 @@ class FailurePoliciesConfig: ) -@dataclass +@dataclass(slots=True) class WorkflowAssignments: - """Workflow selection. - - Scenario assembly uses ``default``; ``scenario_overrides`` is stored but not applied. - """ + """Workflow selection.""" default: str = "design_analysis_brief" - scenario_overrides: dict[str, dict[str, str]] = field(default_factory=dict) -@dataclass +@dataclass(slots=True) class WorkflowsConfig: """Select workflows from the built-ins and ``cwd/lib/workflows.yml``.""" assignments: WorkflowAssignments = field(default_factory=WorkflowAssignments) -@dataclass +@dataclass(slots=True) class TrafficGravityConfig: """Gravity model parameters for DC-to-DC traffic generation. @@ -343,7 +335,7 @@ class TrafficGravityConfig: mw_per_dc_region_overrides: dict[str, float] = field(default_factory=dict) -@dataclass +@dataclass(slots=True) class TrafficHoseConfig: """Hose model parameters for DC-to-DC traffic generation. @@ -353,7 +345,8 @@ class TrafficHoseConfig: shorter-distance pairs via a distance kernel before IPF. beta: Distance exponent in the tilt kernel (km-based). min_distance_km: Minimum effective distance to avoid singularities. - exclude_same_metro: If True, skip intra-metro DC pairs in the tilt kernel. + exclude_same_metro: With nonzero tilt, downweight same-metro pairs while + retaining positive support for marginal fitting; does not ban traffic. """ tilt_exponent: float = 0.0 @@ -364,7 +357,7 @@ class TrafficHoseConfig: carve_top_k: int | None = None -@dataclass +@dataclass(slots=True) class TrafficConfig: """Traffic generation configuration for scenario build. @@ -390,7 +383,7 @@ class TrafficConfig: flow_policy_config: dict[int, str] = field(default_factory=dict) matrix_name: str = "default" model: str = "uniform" - # Number of samples for models that support stochastic emission (e.g., hose) + # Number of hose samples; uniform and gravity emit one matrix. samples: int = 1 gravity: TrafficGravityConfig = field(default_factory=TrafficGravityConfig) hose: TrafficHoseConfig = field(default_factory=TrafficHoseConfig) @@ -399,18 +392,7 @@ def __post_init__(self) -> None: _validate_flow_policy_names(self.flow_policy_config) -@dataclass -class ScenarioMetadata: - """Title, description, and version for generated scenarios.""" - - title: str = "Continental US Backbone Topology" - description: str = ( - "Generated backbone topology from Census urban areas and highway corridors" - ) - version: str = "1.0" - - -@dataclass +@dataclass(slots=True) class ClusteringConfig: """Urban-area selection, radius limits, precision, and map export settings.""" @@ -427,7 +409,7 @@ class ClusteringConfig: area_precision: int = 2 # Decimal places for area/radius rounding -@dataclass +@dataclass(slots=True) class FormattingConfig: """JSON indentation and YAML anchor settings.""" @@ -435,17 +417,26 @@ class FormattingConfig: yaml_anchors: bool = True # Emit YAML anchors/aliases when dumping scenario YAML -@dataclass +@dataclass(slots=True) class OutputConfig: - """Scenario metadata, formatting, and random seed.""" + """Formatting and scenario random seed.""" - scenario_metadata: ScenarioMetadata = field(default_factory=ScenarioMetadata) formatting: FormattingConfig = field(default_factory=FormattingConfig) # Top-level scenario random seed to emit in generated scenario YAML scenario_seed: int = 42 -@dataclass +@dataclass(slots=True) +class VisualizationConfig: + """Rendering settings, independent of artifact destinations.""" + + use_real_corridor_geometry: bool = False + export_site_graph: bool = False + export_blueprint_diagrams: bool = False + dpi: int = 300 + + +@dataclass(slots=True) class TopologyConfig: """Settings for geographic graph generation and NetGraph scenario assembly.""" @@ -473,15 +464,7 @@ class TopologyConfig: ) workflows: WorkflowsConfig = field(default_factory=WorkflowsConfig) traffic: TrafficConfig = field(default_factory=TrafficConfig) - _use_real_corridor_geometry: bool = False - _export_site_graph: bool = False - _visualization_dpi: int = 300 - _source_path: Path | None = None - # Export per-blueprint abstract+concrete diagrams - _export_blueprint_diagrams: bool = False - # Optional instrumentation fields for debugging/export - _debug_dir: Path | None = None - _source_stem: str | None = None + visualization: VisualizationConfig = field(default_factory=VisualizationConfig) @classmethod def from_yaml(cls, config_path: Path) -> TopologyConfig: @@ -503,546 +486,119 @@ def from_yaml(cls, config_path: Path) -> TopologyConfig: logger.error(f"Invalid YAML in configuration: {e}") raise - # Unconditional JSON Schema validation for TopoGen config - try: - import jsonschema # type: ignore - except Exception as exc: # pragma: no cover - raise RuntimeError( - "jsonschema is required for TopoGen configuration validation. Install dev extras or add 'jsonschema' to dependencies." - ) from exc - - # Load packaged schema - try: - with ( - resources.files("topogen.schemas") - .joinpath("topogen_config.json") - .open("r", encoding="utf-8") - ) as f: - schema_data = json.load(f) - except Exception as exc: - raise RuntimeError( - "Failed to locate packaged TopoGen config schema 'topogen/schemas/topogen_config.json'." - ) from exc - - try: - jsonschema.validate(raw_config, schema_data) # type: ignore[arg-type] - except Exception as exc: - raise ValueError( - f"TopoGen config JSON Schema validation failed: {exc}" - ) from exc - - cfg = cls._from_dict(raw_config) - # Attach the config file source path for downstream naming of artefacts - cfg._source_path = Path(config_path) - return cfg + return cls._from_dict(raw_config) @classmethod def _from_dict(cls, config_dict: dict[str, Any]) -> TopologyConfig: - """Parse configuration sections and validate their types and values.""" - if "data_sources" not in config_dict: - raise ValueError("Missing required 'data_sources' configuration section") - if "projection" not in config_dict: - raise ValueError("Missing required 'projection' configuration section") - if "clustering" not in config_dict: - raise ValueError("Missing required 'clustering' configuration section") - if "highway_processing" not in config_dict: - raise ValueError( - "Missing required 'highway_processing' configuration section" - ) - if "corridors" not in config_dict: - raise ValueError("Missing required 'corridors' configuration section") - if "validation" not in config_dict: - raise ValueError("Missing required 'validation' configuration section") - if "output" not in config_dict: - raise ValueError("Missing required 'output' configuration section") - - data_sources_dict = config_dict["data_sources"] - projection_dict = config_dict["projection"] - clustering_dict = config_dict["clustering"] - - if "override_metro_clusters" not in clustering_dict: - clustering_dict["override_metro_clusters"] = [] - highway_dict = config_dict["highway_processing"] - corridors_dict = config_dict["corridors"] - validation_dict = config_dict["validation"] - - if "risk_groups" in corridors_dict: - risk_groups_dict = corridors_dict["risk_groups"] - risk_groups_config = RiskGroupsConfig(**risk_groups_dict) - corridors_dict = corridors_dict.copy() - corridors_dict["risk_groups"] = risk_groups_config - output_dict = config_dict["output"] - - data_sources = DataSources(**data_sources_dict) - projection = ProjectionConfig(**projection_dict) - clustering = ClusteringConfig(**clustering_dict) - highway_processing = HighwayProcessingConfig(**highway_dict) - corridors = CorridorsConfig(**corridors_dict) - validation = ValidationConfig(**validation_dict) - - if "scenario_metadata" not in output_dict: - raise ValueError( - "Missing required 'scenario_metadata' in output configuration" - ) - if "formatting" not in output_dict: - raise ValueError("Missing required 'formatting' in output configuration") - - scenario_metadata_dict = output_dict["scenario_metadata"] - formatting_dict = output_dict["formatting"] - - scenario_metadata = ScenarioMetadata(**scenario_metadata_dict) - formatting = FormattingConfig(**formatting_dict) - output = OutputConfig( - scenario_metadata=scenario_metadata, - formatting=formatting, - scenario_seed=int(output_dict.get("scenario_seed", 42)), + """Validate and parse settings without mutating the input mapping.""" + _check_finite_numbers(config_dict) + try: + _config_validator().validate(config_dict) + except jsonschema.ValidationError as exc: + path = ".".join(str(part) for part in exc.absolute_path) or "config" + raise ValueError(f"TopoGen config {path}: {exc.message}") from exc + + corridors_dict = dict(config_dict["corridors"]) + corridors_dict["risk_groups"] = RiskGroupsConfig( + **corridors_dict.get("risk_groups", {}) ) + output_dict = dict(config_dict["output"]) + output_dict["formatting"] = FormattingConfig(**output_dict["formatting"]) build_dict = config_dict.get("build", {}) - if not isinstance(build_dict, dict): - raise ValueError("'build' configuration section must be a dictionary") - - build_defaults_dict = build_dict.get("build_defaults", {}) - build_overrides_list = build_dict.get("build_overrides", []) - tm_sizing_dict = build_dict.get("tm_sizing", {}) - - if not isinstance(build_defaults_dict, dict): - raise ValueError("'build_defaults' must be a dictionary") - if not isinstance(build_overrides_list, list): - raise ValueError("'build_overrides' must be a list of override entries") - if tm_sizing_dict is None: - tm_sizing_dict = {} - if not isinstance(tm_sizing_dict, dict): - raise ValueError("'tm_sizing' must be a dictionary if provided") - - intra_metro_link_dict = build_defaults_dict.get("intra_metro_link", {}) - inter_metro_link_dict = build_defaults_dict.get("inter_metro_link", {}) - dc_to_pop_link_dict = build_defaults_dict.get("dc_to_pop_link", {}) - - if not isinstance(intra_metro_link_dict, dict): - raise ValueError("'build_defaults.intra_metro_link' must be a dictionary") - if not isinstance(inter_metro_link_dict, dict): - raise ValueError("'build_defaults.inter_metro_link' must be a dictionary") - if not isinstance(dc_to_pop_link_dict, dict): - raise ValueError("'build_defaults.dc_to_pop_link' must be a dictionary") - - intra_metro_link = LinkParams( - capacity=_normalize_int( - intra_metro_link_dict.get("capacity", 400), - "build_defaults.intra_metro_link.capacity", - ), - cost=_normalize_int( - intra_metro_link_dict.get("cost", 1), - "build_defaults.intra_metro_link.cost", - ), - attrs={ - **{"link_type": "intra_metro"}, - **intra_metro_link_dict.get("attrs", {}), - }, - match=intra_metro_link_dict.get("match", {}), - role_pairs=intra_metro_link_dict.get("role_pairs", []) or [], - striping=intra_metro_link_dict.get("striping", {}) or {}, - mode=str(intra_metro_link_dict.get("mode", "mesh")), - ) - inter_metro_link = LinkParams( - capacity=_normalize_int( - inter_metro_link_dict.get("capacity", 100), - "build_defaults.inter_metro_link.capacity", - ), - cost=_normalize_int( - inter_metro_link_dict.get("cost", 1), - "build_defaults.inter_metro_link.cost", - ), - attrs={ - **{"link_type": "inter_metro_corridor"}, - **inter_metro_link_dict.get("attrs", {}), - }, - match=inter_metro_link_dict.get("match", {}), - role_pairs=inter_metro_link_dict.get("role_pairs", []) or [], - striping=inter_metro_link_dict.get("striping", {}) or {}, - mode=str(inter_metro_link_dict.get("mode", "mesh")), - ) - dc_to_pop_link = LinkParams( - capacity=_normalize_int( - dc_to_pop_link_dict.get("capacity", 400), - "build_defaults.dc_to_pop_link.capacity", - ), - cost=_normalize_int( - dc_to_pop_link_dict.get("cost", 1), "build_defaults.dc_to_pop_link.cost" - ), - attrs={ - **{"link_type": "dc_to_pop"}, - **dc_to_pop_link_dict.get("attrs", {}), - }, - match=dc_to_pop_link_dict.get("match", {}), - role_pairs=dc_to_pop_link_dict.get("role_pairs", []) or [], - striping=dc_to_pop_link_dict.get("striping", {}) or {}, - mode=str(dc_to_pop_link_dict.get("mode", "mesh")), - ) - - build_defaults = BuildDefaults( - pop_per_metro=build_defaults_dict.get("pop_per_metro", 2), - site_blueprint=build_defaults_dict.get("site_blueprint", "SingleRouter"), - dc_regions_per_metro=build_defaults_dict.get("dc_regions_per_metro", 2), - dc_region_blueprint=build_defaults_dict.get( - "dc_region_blueprint", "DCRegion" - ), - intra_metro_link=intra_metro_link, - inter_metro_link=inter_metro_link, - dc_to_pop_link=dc_to_pop_link, - ) - _allowed_tm_keys = { - "enabled", - "matrix_name", - "quantum_gbps", - "headroom", - "alpha_dc_to_pop", - "beta_intra_pop", - "flow_placement", - "edge_select", - "respect_min_base_capacity", - } - _tm_extra = set(tm_sizing_dict.keys()) - _allowed_tm_keys - if _tm_extra: - raise ValueError( - f"Unknown keys in 'build.tm_sizing': {sorted(_tm_extra)}. Allowed keys: {sorted(_allowed_tm_keys)}" + defaults = BuildDefaults() + defaults_dict = dict(build_dict.get("build_defaults", {})) + for name in LINK_TYPES: + defaults_dict[name] = parse_link_params( + defaults_dict.get(name, {}), getattr(defaults, name), name ) - tm_sizing = BuildTmSizingConfig( - enabled=bool(tm_sizing_dict.get("enabled", False)), - matrix_name=( - str(tm_sizing_dict.get("matrix_name")) - if tm_sizing_dict.get("matrix_name") is not None - else None - ), - quantum_gbps=float(tm_sizing_dict.get("quantum_gbps", 3200.0)), - headroom=float(tm_sizing_dict.get("headroom", 1.3)), - alpha_dc_to_pop=float(tm_sizing_dict.get("alpha_dc_to_pop", 1.2)), - beta_intra_pop=float(tm_sizing_dict.get("beta_intra_pop", 0.8)), - flow_placement=str(tm_sizing_dict.get("flow_placement", "EQUAL_BALANCED")), - edge_select=str(tm_sizing_dict.get("edge_select", "ALL_MIN_COST")), - respect_min_base_capacity=bool( - tm_sizing_dict.get("respect_min_base_capacity", True) - ), - ) - - # Normalize list of overrides into slug->override mapping. - # Schema per entry: { metros: [str, ...], } - allowed_override_keys = { - "pop_per_metro", - "site_blueprint", - "dc_regions_per_metro", - "dc_region_blueprint", - "intra_metro_link", - "inter_metro_link", - "dc_to_pop_link", - } - normalized_overrides: dict[str, dict[str, Any]] = {} - for idx, entry in enumerate(build_overrides_list): - if not isinstance(entry, dict): - raise ValueError("Each item in 'build_overrides' must be a dictionary") - if "metros" not in entry: - raise ValueError("Each build override must include 'metros' list") - metros_field = entry.get("metros") - if isinstance(metros_field, str): - metro_names: list[str] = [metros_field] - elif isinstance(metros_field, list): - if not all(isinstance(m, str) for m in metros_field): - raise ValueError("'metros' must be a list of strings") - metro_names = list(metros_field) - else: - raise ValueError("'metros' must be a string or a list of strings") - - override_body = {k: v for k, v in entry.items() if k != "metros"} - - extra_keys = set(override_body.keys()) - allowed_override_keys - if extra_keys: - raise ValueError( - f"Unknown keys in build override entry {idx}: {sorted(extra_keys)}. " - f"Allowed keys: {sorted(allowed_override_keys)}" - ) - - for raw_name in metro_names: - slug = metro_slug(raw_name) - # Last one wins for duplicates - normalized_overrides[slug] = override_body - + for entry in build_dict.get("build_overrides", []): + names = entry["metros"] + if isinstance(names, str): + names = [names] + body = {key: value for key, value in entry.items() if key != "metros"} + for name in names: + slug = metro_slug(name) + if slug in normalized_overrides: + raise ValueError(f"Duplicate metro override: {slug}") + normalized_overrides[slug] = body build = BuildConfig( - build_defaults=build_defaults, + build_defaults=BuildDefaults(**defaults_dict), build_overrides=normalized_overrides, - tm_sizing=tm_sizing, + tm_sizing=BuildTmSizingConfig(**build_dict.get("tm_sizing", {})), ) - - components_dict = config_dict.get("components", {}) - if not isinstance(components_dict, dict): - raise ValueError("'components' configuration section must be a dictionary") - - assignments_dict = components_dict.get("assignments", {}) - if not isinstance(assignments_dict, dict): - raise ValueError("'components.assignments' must be a dictionary") - - spine_assignment = ComponentAssignment(**assignments_dict.get("spine", {})) - leaf_assignment = ComponentAssignment(**assignments_dict.get("leaf", {})) - core_assignment = ComponentAssignment(**assignments_dict.get("core", {})) - dc_assignment = ComponentAssignment(**assignments_dict.get("dc", {})) - - assignments = ComponentAssignments( - spine=spine_assignment, - leaf=leaf_assignment, - core=core_assignment, - dc=dc_assignment, - ) - - hw_component_map = components_dict.get("hw_component", {}) or {} - optics_map = components_dict.get("optics", {}) or {} - if hw_component_map is not None and not isinstance(hw_component_map, dict): - raise ValueError( - "'components.hw_component' must be a mapping when provided" + components = ComponentsConfig(**config_dict.get("components", {})) + failure_policies = FailurePoliciesConfig( + assignments=FailurePolicyAssignments( + **config_dict.get("failure_policies", {}).get("assignments", {}) ) - if optics_map is not None and not isinstance(optics_map, dict): - raise ValueError("'components.optics' must be a mapping when provided") - - components = ComponentsConfig( - assignments=assignments, - hw_component=hw_component_map if isinstance(hw_component_map, dict) else {}, - optics=optics_map if isinstance(optics_map, dict) else {}, ) - - failure_policies_dict = config_dict.get("failure_policies", {}) - if not isinstance(failure_policies_dict, dict): - raise ValueError( - "'failure_policies' configuration section must be a dictionary" + workflows = WorkflowsConfig( + assignments=WorkflowAssignments( + **config_dict.get("workflows", {}).get("assignments", {}) ) - - fp_assignments_dict = failure_policies_dict.get("assignments", {}) - if fp_assignments_dict is None: - fp_assignments_dict = {} - if not isinstance(fp_assignments_dict, dict): - raise ValueError("'failure_policies.assignments' must be a dictionary") - - fp_default = fp_assignments_dict.get("default", "single_random_link_failure") - fp_scenario_overrides = fp_assignments_dict.get("scenario_overrides", {}) - if fp_scenario_overrides is None: - fp_scenario_overrides = {} - - fp_assignments = FailurePolicyAssignments( - default=fp_default, - scenario_overrides=fp_scenario_overrides, ) - failure_policies = FailurePoliciesConfig(assignments=fp_assignments) - - workflows_dict = config_dict.get("workflows", {}) - if not isinstance(workflows_dict, dict): - raise ValueError("'workflows' configuration section must be a dictionary") - - wf_assignments_dict = workflows_dict.get("assignments", {}) - if wf_assignments_dict is None: - wf_assignments_dict = {} - if not isinstance(wf_assignments_dict, dict): - raise ValueError("'workflows.assignments' must be a dictionary") - - wf_default = wf_assignments_dict.get("default", WorkflowAssignments().default) - wf_scenario_overrides = wf_assignments_dict.get("scenario_overrides", {}) - if wf_scenario_overrides is None: - wf_scenario_overrides = {} - - wf_assignments = WorkflowAssignments( - default=wf_default, - scenario_overrides=wf_scenario_overrides, - ) - - workflows = WorkflowsConfig(assignments=wf_assignments) - - traffic_dict = config_dict.get("traffic", {}) - if traffic_dict is None: - traffic_dict = {} - if not isinstance(traffic_dict, dict): - raise ValueError("'traffic' configuration section must be a dictionary") - - # Normalize ratios to int keys if strings were provided by YAML loader - ratios_input = traffic_dict.get("priority_ratios") - if ratios_input is not None: - if not isinstance(ratios_input, dict): - raise ValueError("'traffic.priority_ratios' must be a dictionary") - normalized_ratios: dict[int, float] = {} - for k, v in ratios_input.items(): - try: - key_int = int(k) - except (TypeError, ValueError) as exc: - raise ValueError( - "'traffic.priority_ratios' keys must be integers" - ) from exc - normalized_ratios[key_int] = float(v) - traffic_dict = {**traffic_dict, "priority_ratios": normalized_ratios} - - gravity_dict = traffic_dict.get("gravity", None) - if gravity_dict is None: - gravity_cfg = TrafficGravityConfig() - else: - if not isinstance(gravity_dict, dict): - raise ValueError("'traffic.gravity' must be a dictionary") - gravity_cfg = TrafficGravityConfig(**gravity_dict) - - hose_dict = traffic_dict.get("hose", None) - if hose_dict is None: - hose_cfg = TrafficHoseConfig() - else: - if not isinstance(hose_dict, dict): - raise ValueError("'traffic.hose' must be a dictionary") - hose_cfg = TrafficHoseConfig(**hose_dict) - - # Normalize optional flow_policy_config mapping to int keys - fpc_input = traffic_dict.get("flow_policy_config") - if fpc_input is None: - flow_policy_cfg_map: dict[int, str] = {} - else: - if not isinstance(fpc_input, dict): - raise ValueError( - "'traffic.flow_policy_config' must be a dictionary mapping " - "priority class to policy name" - ) - flow_policy_cfg_map = {} - for k, v in fpc_input.items(): - try: - key_int = int(k) - except (TypeError, ValueError) as exc: + traffic_dict = dict(config_dict.get("traffic", {})) + for field_name in ("priority_ratios", "flow_policy_config"): + if field_name not in traffic_dict: + continue + normalized = {} + for key, value in traffic_dict[field_name].items(): + priority = _normalize_int(key, f"traffic.{field_name} key") + if priority in normalized: raise ValueError( - "'traffic.flow_policy_config' keys must be integers" - ) from exc - flow_policy_cfg_map[key_int] = v - - traffic = TrafficConfig( - enabled=bool(traffic_dict.get("enabled", True)), - gbps_per_mw=float(traffic_dict.get("gbps_per_mw", 1000.0)), - mw_per_dc_region=float(traffic_dict.get("mw_per_dc_region", 150.0)), - priority_ratios=traffic_dict.get( - "priority_ratios", {0: 0.3, 1: 0.3, 2: 0.4} - ), - flow_policy_config=flow_policy_cfg_map, - matrix_name=str(traffic_dict.get("matrix_name", "default")), - model=str(traffic_dict.get("model", "uniform")).strip(), - samples=int(traffic_dict.get("samples", 1)), - gravity=gravity_cfg, - hose=hose_cfg, - ) - - if traffic.enabled: - if traffic.gbps_per_mw < 0: - raise ValueError("traffic.gbps_per_mw must be non-negative") - if traffic.mw_per_dc_region < 0: - raise ValueError("traffic.mw_per_dc_region must be non-negative") - if not traffic.priority_ratios: - raise ValueError("traffic.priority_ratios must not be empty") - # Require contiguous classes from 0..N-1 - classes = sorted(traffic.priority_ratios.keys()) - expected = list(range(len(classes))) - if classes != expected: - raise ValueError( - "traffic.priority_ratios must have contiguous integer keys from 0..N-1" - ) - total_ratio = sum(traffic.priority_ratios.values()) - if abs(total_ratio - 1.0) > 1e-9: - raise ValueError("traffic.priority_ratios values must sum to 1.0") - - if traffic.flow_policy_config: - # Keys must be subset of defined priority classes - extra_keys = [ - k for k in traffic.flow_policy_config.keys() if k not in classes - ] - if extra_keys: - raise ValueError( - "traffic.flow_policy_config contains unknown priority classes; " - f"allowed: {classes}, got: {sorted(extra_keys)}" - ) - - if traffic.model not in {"uniform", "gravity", "hose"}: - raise ValueError( - "traffic.model must be 'uniform', 'gravity', or 'hose'" - ) - if traffic.model == "hose": - if not isinstance(traffic.samples, int) or traffic.samples <= 0: - raise ValueError( - "traffic.samples must be a positive integer for hose model" + f"Duplicate traffic.{field_name} priority: {priority}" ) - h = getattr(traffic, "hose", TrafficHoseConfig()) - if h.tilt_exponent < 0.0: - raise ValueError("traffic.hose.tilt_exponent must be non-negative") - if h.beta <= 0.0: - raise ValueError("traffic.hose.beta must be positive") - if h.min_distance_km <= 0.0: - raise ValueError("traffic.hose.min_distance_km must be positive") - if h.carve_top_k is not None and int(h.carve_top_k) <= 0: - raise ValueError( - "traffic.hose.carve_top_k must be positive when set" - ) - - g = traffic.gravity - if g.alpha <= 0.0: - raise ValueError("traffic.gravity.alpha must be positive") - if g.beta <= 0.0: - raise ValueError("traffic.gravity.beta must be positive") - if g.min_distance_km <= 0.0: - raise ValueError("traffic.gravity.min_distance_km must be positive") - if g.max_partners_per_dc is not None and g.max_partners_per_dc <= 0: - raise ValueError( - "traffic.gravity.max_partners_per_dc must be positive when set" - ) - if g.jitter_stddev < 0.0: - raise ValueError("traffic.gravity.jitter_stddev must be non-negative") - if g.rounding_gbps < 0.0: - raise ValueError("traffic.gravity.rounding_gbps must be non-negative") - - vis = config_dict.get("visualization", {}) or {} - if not isinstance(vis, dict): - raise ValueError( - "'visualization' configuration section must be a dictionary if provided" - ) - vis_corridors = vis.get("corridors", {}) or {} - if not isinstance(vis_corridors, dict): - raise ValueError( - "'visualization.corridors' must be a dictionary if provided" - ) - use_real_geometry = bool(vis_corridors.get("use_real_geometry", False)) - vis_site = vis.get("site_graph", {}) or {} - if not isinstance(vis_site, dict): + normalized[priority] = value + traffic_dict[field_name] = normalized + traffic_dict["gravity"] = TrafficGravityConfig( + **traffic_dict.get("gravity", {}) + ) + traffic_dict["hose"] = TrafficHoseConfig(**traffic_dict.get("hose", {})) + traffic = TrafficConfig(**traffic_dict) + classes = sorted(traffic.priority_ratios) + if classes != list(range(len(classes))) or not classes: raise ValueError( - "'visualization.site_graph' must be a dictionary if provided" + "traffic.priority_ratios must have contiguous integer keys from 0..N-1" ) - export_site_graph = bool(vis_site.get("export", False)) - vis_blueprints = vis.get("blueprints", {}) or {} - if not isinstance(vis_blueprints, dict): + if not math.isclose( + sum(traffic.priority_ratios.values()), 1.0, rel_tol=0, abs_tol=1e-9 + ): + raise ValueError("traffic.priority_ratios values must sum to 1.0") + if extra := traffic.flow_policy_config.keys() - traffic.priority_ratios.keys(): raise ValueError( - "'visualization.blueprints' must be a dictionary if provided" + f"traffic.flow_policy_config contains unknown priority classes: {sorted(extra)}" ) - export_blueprints = bool(vis_blueprints.get("export", False)) - dpi_val = vis.get("dpi", 300) - try: - visualization_dpi = int(dpi_val) - if visualization_dpi <= 0: - raise ValueError - except Exception as exc: - raise ValueError("'visualization.dpi' must be a positive integer") from exc - - cfg = cls( - data_sources=data_sources, - projection=projection, - clustering=clustering, - highway_processing=highway_processing, - corridors=corridors, - validation=validation, - output=output, + + vis = config_dict.get("visualization", {}) + return cls( + data_sources=DataSources(**config_dict["data_sources"]), + projection=ProjectionConfig(**config_dict["projection"]), + clustering=ClusteringConfig(**config_dict["clustering"]), + highway_processing=HighwayProcessingConfig( + **config_dict["highway_processing"] + ), + corridors=CorridorsConfig(**corridors_dict), + validation=ValidationConfig(**config_dict["validation"]), + output=OutputConfig(**output_dict), build=build, components=components, failure_policies=failure_policies, workflows=workflows, traffic=traffic, + visualization=VisualizationConfig( + use_real_corridor_geometry=vis.get("corridors", {}).get( + "use_real_geometry", False + ), + export_site_graph=vis.get("site_graph", {}).get("export", False), + export_blueprint_diagrams=vis.get("blueprints", {}).get( + "export", False + ), + dpi=vis.get("dpi", 300), + ), ) - cfg._use_real_corridor_geometry = use_real_geometry - cfg._export_site_graph = export_site_graph - cfg._visualization_dpi = visualization_dpi - cfg._export_blueprint_diagrams = export_blueprints - return cfg def validate(self) -> None: """Check the metro count and required data files, raising ValueError on failure.""" @@ -1068,39 +624,26 @@ def validate(self) -> None: logger.info("Configuration validation passed") - def summary(self) -> str: - """Return a human-readable configuration summary.""" - lines = [ - "TOPOLOGY GENERATOR CONFIGURATION", - "=" * 60, - "", - "CLUSTERING PARAMETERS", - "-" * 30, - f" Metro Clusters: ~{self.clustering.metro_clusters}", - f" Max UAC Radius: {self.clustering.max_uac_radius_km}km", - f" Export Clusters: {self.clustering.export_clusters}", - "", - "DATA SOURCES", - "-" * 30, - f" UAC Polygons: {self.data_sources.uac_polygons}", - f" TIGER Roads: {self.data_sources.tiger_roads}", - f" CONUS Boundary: {self.data_sources.conus_boundary}", - f" Target CRS: {self.projection.target_crs}", - "", - "HIGHWAY PROCESSING (Lightweight)", - "-" * 30, - f" Min Edge Length: {self.highway_processing.min_edge_length_km}km", - "", - "CORRIDOR DISCOVERY", - "-" * 30, - f" K-Paths per Metro Pair: {self.corridors.k_paths}", - "", - "VALIDATION", - "-" * 30, - f" Max Metro-Highway Distance: {self.validation.max_metro_highway_distance_km}km", - f" Require Connected: {self.validation.require_connected}", - "", - "=" * 60, - ] - - return "\n".join(lines) + +LINK_TYPES = ("intra_metro_link", "inter_metro_link", "dc_to_pop_link") + + +def parse_link_params( + values: dict[str, Any], defaults: LinkParams, name: str +) -> LinkParams: + """Merge attributes, replace selectors, and normalize numeric link settings.""" + merged = asdict(defaults) + merged.update(values) + merged["attrs"] = {**defaults.attrs, **values.get("attrs", {})} + for key in ("capacity", "cost"): + merged[key] = _normalize_int(merged[key], f"{name}.{key}") + return LinkParams(**merged) + + +@lru_cache(maxsize=1) +def _config_validator() -> jsonschema.Draft7Validator: + schema = json.loads( + resources.files("topogen.schemas").joinpath("topogen_config.json").read_text() + ) + jsonschema.Draft7Validator.check_schema(schema) + return jsonschema.Draft7Validator(schema) diff --git a/topogen/context.py b/topogen/context.py new file mode 100644 index 0000000..a1ae975 --- /dev/null +++ b/topogen/context.py @@ -0,0 +1,14 @@ +"""Explicit, per-invocation artifact destinations; never part of topology settings.""" + +from dataclasses import dataclass +from pathlib import Path + + +@dataclass(frozen=True) +class RunContext: + output_dir: Path + stem: str = "scenario" + debug_dir: Path | None = None + + def path(self, suffix: str) -> Path: + return self.output_dir / f"{self.stem}_{suffix}" diff --git a/topogen/corridors.py b/topogen/corridors.py index 945ff55..c4704db 100644 --- a/topogen/corridors.py +++ b/topogen/corridors.py @@ -3,11 +3,11 @@ from __future__ import annotations from dataclasses import dataclass -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING import networkx as nx import numpy as np -from scipy.spatial import KDTree # type: ignore[import-untyped] +from scipy.spatial import KDTree # pyright: ignore[reportMissingTypeStubs] from shapely.geometry import Point from topogen.log_config import get_logger @@ -27,18 +27,12 @@ class CorridorPath: """Concrete corridor path details connecting two metros. Attributes: - metros: Sorted pair of metro IDs (metro_a, metro_b). - path_index: K-shortest path index. - nodes: Ordered node coordinates along the path (including metros and anchors). edges: Ordered endpoint pairs along the path. segment_ids: Ordered contracted highway segment IDs (excludes anchor edges). length_km: Total length of path in kilometers. geometry: Ordered polyline coordinates (concatenated edge geometries). """ - metros: tuple[str, str] - path_index: int - nodes: list[tuple[float, float]] edges: list[tuple[tuple[float, float], tuple[float, float]]] segment_ids: list[str] length_km: float @@ -50,298 +44,97 @@ def add_corridors( metros: list[MetroCluster], corridors_config: CorridorsConfig, ) -> None: - """Add corridor tags to edges along metro-to-metro k-shortest paths. - - Computes k-shortest paths between adjacent metro pairs using the integrated - graph that already contains metro nodes connected to their highway anchors. - Paths take the form ``metro → anchor → ...highway... → anchor → metro``. - The path length is the sum of ``length_km`` over all edges in the path. - ``metro_anchor`` edges use Euclidean length in kilometers. - - Args: - graph: Integrated graph containing highway and metro nodes and edges. - metros: List of metro clusters. - corridors_config: Corridor discovery configuration. - - Raises: - ValueError: If no adjacent metro pairs or no corridors are found. - - Notes: - - Adjacency is built via k-nearest neighbors on metro centroids and - filtered by ``max_edge_km`` (Euclidean km). - - ``max_corridor_distance_km`` is enforced on the actual path length over - the graph (sum of per-edge ``length_km``), not on Euclidean separation. - - Each path edge is tagged with a ``corridor`` entry and a membership set - ``corridor_path_ids``. Use ``extract_corridor_graph`` to build a - metro-to-metro corridor graph for downstream use. + """Register shortest simple paths for the union of metro k-nearest pairs. + + Distances use projected meters; equal-distance neighbors are ordered by + coordinates and ID. Path geometry must match its road edge endpoints. """ - logger.info(f"Starting corridor discovery for {len(metros)} metros") - logger.info( - f"Corridor configuration: k_paths={corridors_config.k_paths}, " - f"k_nearest={corridors_config.k_nearest}, " - f"max_edge_km={corridors_config.max_edge_km}km, " - f"max_corridor_distance_km={corridors_config.max_corridor_distance_km}km" - ) + from itertools import islice - # Build adjacency using k-nearest neighbors if len(metros) < 2: raise ValueError("At least two metros are required for corridor discovery") - - metro_coords = np.array([[m.centroid_x, m.centroid_y] for m in metros]) - tree = KDTree(metro_coords) - - adjacent_pairs: list[tuple[str, str, float]] = [] - for metro in metros: - k_query = min(corridors_config.k_nearest + 1, len(metros)) - distances, indices = tree.query([metro.centroid_x, metro.centroid_y], k=k_query) - - indices_list = ( - indices.tolist() if isinstance(indices, np.ndarray) else [indices] + coords = np.array([m.node_key for m in metros]) + tree = KDTree(coords) + pairs = set() + by_id = {m.metro_id: m for m in metros} + if len(by_id) != len(metros) or len({m.node_key for m in metros}) != len(metros): + raise ValueError("Metro IDs and coordinates must be unique") + k = min(corridors_config.k_nearest, len(metros) - 1) + for index, metro in enumerate(metros): + distances, _ = tree.query(coords[index], k=k + 1) + candidates = tree.query_ball_point( + coords[index], np.nextafter(distances[-1], np.inf) ) - distances_list = ( - distances.tolist() if isinstance(distances, np.ndarray) else [distances] + candidates = sorted( + (j for j in candidates if j != index), + key=lambda j: ( + float(np.linalg.norm(coords[index] - coords[j])), + metros[j].node_key, + metros[j].metro_id, + ), ) - - for j in range(1, min(len(indices_list), corridors_config.k_nearest + 1)): - neighbor_idx = indices_list[j] - distance_km = distances_list[j] / 1000.0 - - if distance_km <= corridors_config.max_edge_km: - metro_a_id = metro.metro_id - metro_b_id = metros[neighbor_idx].metro_id - if metro_a_id < metro_b_id: - adjacent_pairs.append((metro_a_id, metro_b_id, distance_km)) - - logger.info( - f"Found {len(adjacent_pairs)} adjacent metro pairs for corridor discovery" - ) - - if not adjacent_pairs: + for neighbor in candidates[:k]: + if ( + np.linalg.norm(coords[index] - coords[neighbor]) / 1000 + <= corridors_config.max_edge_km + ): + pairs.add(tuple(sorted((metro.metro_id, metros[neighbor].metro_id)))) + if not pairs: raise ValueError("No adjacent metro pairs found for corridor discovery") - if "corridor_paths" not in graph.graph: - graph.graph["corridor_paths"] = {} - - corridor_count = 0 - processed_pairs = 0 - skipped_pairs = 0 - successful_pairs = 0 - - logger.info(f"Processing {len(adjacent_pairs)} metro pairs for corridor discovery") - - metro_id_to_node: dict[str, tuple[float, float]] = { - m.metro_id: m.node_key for m in metros - } - - for metro_a_id, metro_b_id, pair_distance in adjacent_pairs: - processed_pairs += 1 - - if processed_pairs % 10 == 0 or processed_pairs == len(adjacent_pairs): - logger.info( - f"Progress: {processed_pairs:,}/{len(adjacent_pairs):,} pairs processed" - ) - - # Do not apply max_corridor_distance_km to Euclidean separation. - # The threshold is enforced below using the actual path length along the graph. - - if metro_a_id not in metro_id_to_node or metro_b_id not in metro_id_to_node: - logger.warning( - f"Skipping {metro_a_id}-{metro_b_id}: metro node(s) missing in mapping" - ) - continue - node_a = metro_id_to_node[metro_a_id] - node_b = metro_id_to_node[metro_b_id] - - if not graph.has_node(node_a) or not graph.has_node(node_b): - logger.warning( - f"Skipping {metro_a_id}-{metro_b_id}: metro node(s) not present in graph" - ) - continue - - logger.debug( - f"Finding paths between {metro_a_id} and {metro_b_id} ({pair_distance:.1f}km)" - ) - + registry: dict[PathId, CorridorPath] = {} + graph.graph["corridor_paths"] = registry + for _, _, data in graph.edges(data=True): + data.pop("corridor", None) + data.pop("risk_groups", None) + for a, b in sorted(pairs): try: - import itertools - import time - - start_time = time.time() - paths_generator = nx.shortest_simple_paths( - graph, node_a, node_b, weight="length_km" - ) - paths = list(itertools.islice(paths_generator, corridors_config.k_paths)) - elapsed = time.time() - start_time - - if not paths: - logger.warning(f"No paths found between {metro_a_id} and {metro_b_id}") - continue - - logger.debug( - f"Found {len(paths)} candidate paths between {metro_a_id}-{metro_b_id} in {elapsed:.2f}s" - ) - - except nx.NetworkXNoPath: - logger.warning( - f"No path found between metros {metro_a_id} and {metro_b_id}" - ) - continue - except Exception as e: # pragma: no cover - defensive - logger.error( - f"Error finding paths between {metro_a_id} and {metro_b_id}: {e}" + paths = nx.shortest_simple_paths( + graph, by_id[a].node_key, by_id[b].node_key, weight="length_km" ) - continue - - # Compute and tag only paths that satisfy the path-length threshold - added_any_path_for_pair = False - too_long_paths = 0 - for path_idx, path in enumerate(paths): - path_length_km = 0.0 - for i in range(len(path) - 1): - u, v = path[i], path[i + 1] - if not graph.has_edge(u, v): - continue - path_length_km += float(graph[u][v].get("length_km", 0.0)) - - if path_length_km > corridors_config.max_corridor_distance_km: - too_long_paths += 1 - logger.debug( - f"Skipping path {path_idx} for {metro_a_id}-{metro_b_id}: " - f"{path_length_km:.1f}km > {corridors_config.max_corridor_distance_km}km" - ) - continue - - ordered_edges: list[tuple[tuple[float, float], tuple[float, float]]] = [] - for i in range(len(path) - 1): - ordered_edges.append((path[i], path[i + 1])) - - # Collect segment IDs from contracted highway edges only - segment_ids: list[str] = [] - for u, v in ordered_edges: - if not graph.has_edge(u, v): - continue - seg = graph[u][v].get("segment_id") - if seg: - segment_ids.append(str(seg)) - - # Build merged geometry by concatenating per-edge geometry - merged_geometry: list[tuple[float, float]] = [] - - for u, v in ordered_edges: - if not graph.has_edge(u, v): - continue - ed = graph[u][v] - geom = ed.get("geometry") - if isinstance(geom, list) and geom: - # Normalize direction to match u->v when possible - g0 = tuple(geom[0]) - g1 = tuple(geom[-1]) - if g0 == u and g1 == v: - coords: list[tuple[float, float]] = [ - (float(p[0]), float(p[1])) for p in geom - ] - elif g0 == v and g1 == u: - coords = [(float(p[0]), float(p[1])) for p in reversed(geom)] - else: - # Fallback to endpoints if geometry endpoints don't align - coords = [ - (float(u[0]), float(u[1])), - (float(v[0]), float(v[1])), - ] - else: - coords = [ - (float(u[0]), float(u[1])), - (float(v[0]), float(v[1])), - ] - - if not merged_geometry: - merged_geometry.extend(coords) - else: - # Drop duplicate shared vertex when appending - if merged_geometry[-1] == coords[0]: - merged_geometry.extend(coords[1:]) - else: - merged_geometry.extend(coords) - - m_sorted = ( - (metro_a_id, metro_b_id) - if metro_a_id < metro_b_id - else ( - metro_b_id, - metro_a_id, - ) - ) - path_id: PathId = (m_sorted[0], m_sorted[1], path_idx) - - nodes_seq: list[tuple[float, float]] = [ - (float(n[0]), float(n[1])) for n in path - ] - edges_seq: list[tuple[tuple[float, float], tuple[float, float]]] = [ - ((float(u[0]), float(u[1])), (float(v[0]), float(v[1]))) - for (u, v) in ordered_edges - ] - - cp = CorridorPath( - metros=m_sorted, - path_index=path_idx, - nodes=nodes_seq, - edges=edges_seq, - segment_ids=segment_ids, - length_km=path_length_km, - geometry=merged_geometry, - ) - graph.graph["corridor_paths"][path_id] = cp - - # Record corridor metadata and path IDs on each edge. - for u, v in ordered_edges: - if not graph.has_edge(u, v): - continue - edge_data = graph[u][v] - if "corridor" not in edge_data: - edge_data["corridor"] = [] - edge_data["corridor"].append( - { - "metro_a": metro_a_id, - "metro_b": metro_b_id, - "path_index": path_idx, - "distance_km": path_length_km, - } - ) - # Track path membership for risk aggregation. - if "corridor_path_ids" not in edge_data: - edge_data["corridor_path_ids"] = set() - try: - edge_data["corridor_path_ids"].add(path_id) - except Exception: - s = edge_data.get("corridor_path_ids") - edge_data["corridor_path_ids"] = ( - set(s) if isinstance(s, list) else set() + for path_index, nodes in enumerate(islice(paths, corridors_config.k_paths)): + edges = list(zip(nodes, nodes[1:], strict=False)) + length = sum(float(graph[u][v]["length_km"]) for u, v in edges) + if length > corridors_config.max_corridor_distance_km: + break # Subsequent paths are at least as long. + geometry = [] + segments = [] + pid = (a, b, path_index) + for u, v in edges: + data = graph[u][v] + points = [tuple(point) for point in data["geometry"]] + if points[0] == v and points[-1] == u: + points.reverse() + if points[0] != u or points[-1] != v: + raise ValueError( + f"Road geometry does not match endpoints {u}, {v}" + ) + geometry.extend(points if not geometry else points[1:]) + if "segment_id" in data: + segments.append(data["segment_id"]) + data.setdefault("corridor", []).append( + { + "metro_a": a, + "metro_b": b, + "path_index": path_index, + "distance_km": length, + } ) - edge_data["corridor_path_ids"].add(path_id) - corridor_count += 1 - added_any_path_for_pair = True - - if added_any_path_for_pair: - successful_pairs += 1 - else: - skipped_pairs += 1 - if too_long_paths > 0: - logger.debug( - f"Skipping {metro_a_id}-{metro_b_id}: all {too_long_paths} candidate paths exceed max_corridor_distance_km" - ) - - failed_pairs = processed_pairs - skipped_pairs - successful_pairs + registry[pid] = CorridorPath(edges, segments, length, geometry) + except nx.NetworkXNoPath: + logger.info("No road path between metros %s and %s", a, b) + if not registry: + raise ValueError("No corridors found - corridor discovery failed") logger.info( - "Corridor discovery complete: " - f"Processed {processed_pairs:,}/{len(adjacent_pairs):,} pairs, " - f"Successful {successful_pairs:,}, " - f"Failed {failed_pairs:,}, " - f"Skipped {skipped_pairs:,} (too far), " - f"Tagged {corridor_count:,} highway edges with corridor labels" + "Discovered %d paths for %d adjacent metro pairs", len(registry), len(pairs) ) - if corridor_count == 0: - raise ValueError("No corridors found - corridor discovery failed") + +def corridor_risk_name(prefix: str, source: str, target: str, path_index: int) -> str: + """Name one corridor path consistently in geography and scenario output.""" + first, second = sorted((source, target)) + name = f"{prefix}_{first}_{second}" + return name if path_index == 0 else f"{name}_path{path_index}" def assign_risk_groups( @@ -404,13 +197,12 @@ def assign_risk_groups( metro_a_name = metro_id_to_name[metro_a_id] metro_b_name = metro_id_to_name[metro_b_id] - if metro_a_name < metro_b_name: - risk_group_name = f"{corridors_config.risk_groups.group_prefix}_{metro_a_name}_{metro_b_name}" - else: - risk_group_name = f"{corridors_config.risk_groups.group_prefix}_{metro_b_name}_{metro_a_name}" - - if path_index > 0: - risk_group_name += f"_path{path_index}" + risk_group_name = corridor_risk_name( + corridors_config.risk_groups.group_prefix, + metro_a_name, + metro_b_name, + path_index, + ) if "risk_groups" not in edge_data: edge_data["risk_groups"] = [] @@ -426,38 +218,14 @@ def assign_risk_groups( def extract_corridor_graph( full_graph: nx.Graph, metros: list[MetroCluster] -) -> nx.Graph: - """Build a metro graph from ``full_graph.graph["corridor_paths"]``. - - Keep the shortest registered path for each metro pair and its risk groups. - """ - logger.info("Extracting corridor-level graph from corridor path registry") - - registry = full_graph.graph.get("corridor_paths") +) -> nx.MultiGraph: + """Preserve every registered path and aggregate risks along its own edges.""" + registry: dict[PathId, CorridorPath] = full_graph.graph["corridor_paths"] if not registry: - raise ValueError( - "Corridor path registry missing: graph.graph['corridor_paths'] not found" - ) - - items = registry.items() if isinstance(registry, dict) else [] - - # Choose shortest path per metro pair - best: dict[tuple[str, str], tuple[PathId, CorridorPath]] = {} - for pid, cp in items: - metros_sorted = tuple(sorted(cp.metros)) - key = (metros_sorted[0], metros_sorted[1]) - prev = best.get(key) - if ( - prev is None - or (cp.length_km < prev[1].length_km) - or (cp.length_km == prev[1].length_km and pid[2] < prev[0][2]) - ): - best[key] = (pid, cp) # type: ignore[assignment] - - corridor_graph = nx.Graph() - + raise ValueError("Corridor path registry is empty") + graph = nx.MultiGraph() for metro in metros: - corridor_graph.add_node( + graph.add_node( metro.node_key, node_type="metro", metro_id=metro.metro_id, @@ -469,61 +237,31 @@ def extract_corridor_graph( uac_code=metro.uac_code, land_area_km2=metro.land_area_km2, ) - - metro_id_to_coords = {metro.metro_id: metro.node_key for metro in metros} - - # Aggregate risk groups only from edges that include the chosen path id - risks_by_pair: dict[tuple[str, str], set[str]] = {} - for (metro_a_id, metro_b_id), (pid, _cp) in best.items(): - key = (metro_a_id, metro_b_id) - risks: set[str] = set() - for _u, _v, ed in full_graph.edges(data=True): - pids = ed.get("corridor_path_ids") - if not pids: - continue - if isinstance(pids, set): - has = pid in pids - else: - try: - has = pid in set(pids) - except Exception: - has = False - if has: - for rg in ed.get("risk_groups", []) or []: - risks.add(str(rg)) - risks_by_pair[key] = risks - - edges_added = 0 - for (metro_a_id, metro_b_id), (_pid, cp) in best.items(): - if metro_a_id not in metro_id_to_coords or metro_b_id not in metro_id_to_coords: - continue - node_a = metro_id_to_coords[metro_a_id] - node_b = metro_id_to_coords[metro_b_id] - - euclidean_km = Point(node_a).distance(Point(node_b)) / 1000.0 - detour_ratio = (cp.length_km / euclidean_km) if euclidean_km > 0 else None - - corridor_graph.add_edge( - node_a, - node_b, + by_id = {metro.metro_id: metro.node_key for metro in metros} + for (a, b, index), path in sorted(registry.items()): + u, v = by_id[a], by_id[b] + euclidean = Point(u).distance(Point(v)) / 1000 + risks = { + risk + for x, y in path.edges + for risk in full_graph[x][y].get("risk_groups", []) + } + graph.add_edge( + u, + v, + key=index, + path_index=index, edge_type="corridor", - length_km=cp.length_km, - metro_a=metro_a_id, - metro_b=metro_b_id, - euclidean_km=euclidean_km, - detour_ratio=detour_ratio, - geometry=cp.geometry, - contracted_segments=cp.segment_ids, - risk_groups=sorted(risks_by_pair.get((metro_a_id, metro_b_id), set())), + length_km=path.length_km, + metro_a=a, + metro_b=b, + euclidean_km=euclidean, + detour_ratio=path.length_km / euclidean, + geometry=path.geometry, + contracted_segments=path.segment_ids, + risk_groups=sorted(risks), ) - edges_added += 1 - - logger.info( - f"Extracted corridor graph: {len(corridor_graph.nodes):,} metro nodes, {edges_added} corridor edges" - ) - if edges_added == 0: - raise ValueError("Corridor graph extraction failed: no corridor edges found") - return corridor_graph + return graph def validate_corridor_graph( @@ -589,34 +327,3 @@ def validate_corridor_graph( logger.info( f"Corridor graph validation successful: {len(corridor_graph.nodes):,} metros, {corridor_edges:,} corridors, avg distance {avg_distance:.1f}km" ) - - -def extract_corridor_edges_for_metros_graph(graph: nx.Graph) -> list[dict[str, Any]]: - """Extract simplified metro-to-metro corridor edges for downstream pipelines. - - Returns list entries with: - - source: metro node key - - target: metro node key - - length_km: corridor path length in km - - edge_type: 'corridor' - - risk_groups: list[str] - - capacity: optional capacity if already present - """ - edges: list[dict[str, Any]] = [] - for u, v, data in graph.edges(data=True): - src = graph.nodes[u] - tgt = graph.nodes[v] - if src.get("node_type") in {"metro", "metro+highway"} and tgt.get( - "node_type" - ) in {"metro", "metro+highway"}: - entry: dict[str, Any] = { - "source": u, - "target": v, - "length_km": data.get("length_km", 0.0), - "edge_type": data.get("edge_type", "corridor"), - "risk_groups": data.get("risk_groups", []), - } - if "capacity" in data: - entry["capacity"] = data["capacity"] - edges.append(entry) - return edges diff --git a/topogen/failure_policies_lib.py b/topogen/failure_policies_lib.py index 5d584d7..23210d7 100644 --- a/topogen/failure_policies_lib.py +++ b/topogen/failure_policies_lib.py @@ -6,10 +6,9 @@ from __future__ import annotations from copy import deepcopy -from pathlib import Path from typing import Any -import yaml +from topogen.library_io import load_user_library _BUILTIN_FAILURE_POLICIES: dict[str, dict[str, Any]] = { "empty": {"modes": [{"weight": 1.0, "rules": []}]}, @@ -129,24 +128,6 @@ def get_builtin_failure_policies() -> dict[str, dict[str, Any]]: """Return built-in failure policies with user overrides applied.""" policies = deepcopy(_BUILTIN_FAILURE_POLICIES) - user_policies = _load_user_library("failure_policies.yml") + user_policies = load_user_library("failure_policies.yml") policies.update(user_policies) return policies - - -def _load_user_library(file_name: str) -> dict[str, Any]: - """Load user failure policies from ``lib/`` if present.""" - lib_path = Path.cwd() / "lib" / file_name - if not lib_path.exists(): - return {} - - try: - with lib_path.open("r", encoding="utf-8") as f: - data = yaml.safe_load(f) or {} - except Exception as exc: # noqa: BLE001 - raise ValueError(f"Failed to parse YAML: {lib_path}") from exc - - if not isinstance(data, dict): - raise ValueError(f"User library YAML must be a mapping: {lib_path}") - - return data diff --git a/topogen/geo_utils.py b/topogen/geo_utils.py index 8d42909..1892883 100644 --- a/topogen/geo_utils.py +++ b/topogen/geo_utils.py @@ -1,51 +1,15 @@ -"""Geographic utilities for coordinate projections and spatial operations.""" +"""Build the projected contiguous-US mask shared by filtering and map rendering.""" from __future__ import annotations -import math -from functools import lru_cache from pathlib import Path import geopandas as gpd -import pyproj -from shapely.geometry import Point from topogen.log_config import get_logger logger = get_logger(__name__) -WGS84 = "EPSG:4326" # Geographic coordinate system (lat/lon) -CONUS_ALBERS = "EPSG:5070" # NAD83 / Conus Albers Equal Area - - -@lru_cache(maxsize=32) -def get_transformer(src_crs: str, dst_crs: str) -> pyproj.Transformer: - """Return a CRS transformer with x/y (longitude/latitude) axis order.""" - return pyproj.Transformer.from_crs(src_crs, dst_crs, always_xy=True) - - -def transform_point(point: Point, src_crs: str, dst_crs: str) -> Point: - """Transform a point from ``src_crs`` to ``dst_crs``.""" - transformer = get_transformer(src_crs, dst_crs) - x, y = transformer.transform(point.x, point.y) - return Point(x, y) - - -def bearing_to_offset(bearing_deg: float, radius_m: float) -> tuple[float, float]: - """Return an (x, y) offset in meters for a bearing clockwise from north.""" - bearing_rad = math.radians(bearing_deg) - x_offset = radius_m * math.sin(bearing_rad) - y_offset = radius_m * math.cos(bearing_rad) - return x_offset, y_offset - - -def point_at_bearing( - center_point: Point, bearing_deg: float, distance_m: float -) -> Point: - """Offset a projected point by meters at a bearing clockwise from north.""" - x_offset, y_offset = bearing_to_offset(bearing_deg, distance_m) - return Point(center_point.x + x_offset, center_point.y + y_offset) - def create_conus_mask(conus_boundary_path: Path, target_crs: str) -> gpd.GeoDataFrame: """Dissolve Census state boundaries into a contiguous-US mask in ``target_crs``. diff --git a/topogen/highway_graph.py b/topogen/highway_graph.py index 05323f1..971156b 100644 --- a/topogen/highway_graph.py +++ b/topogen/highway_graph.py @@ -104,19 +104,17 @@ def _fix_geometries(gdf: gpd.GeoDataFrame) -> gpd.GeoDataFrame: if gdf.empty: raise ValueError("No valid geometries remain after geometry filter") + excluded_count = initial_count - len(gdf) exploded = gdf.explode(index_parts=False, ignore_index=True) - gdf = exploded if isinstance(exploded, gpd.GeoDataFrame) else gdf + assert isinstance(exploded, gpd.GeoDataFrame) + gdf = exploded - final_count = len(gdf) - excluded_count = initial_count - final_count - if excluded_count > 0: - logger.info( - f"Geometry cleanup: {initial_count:,} -> {final_count:,} segments (excluded {excluded_count:,} invalid)" - ) - else: - logger.info( - f"Geometry cleanup: {final_count:,} segments (no invalid geometries)" - ) + logger.info( + "Geometry cleanup: %d input rows, %d empty/invalid rows removed, %d segments after splitting", + initial_count, + excluded_count, + len(gdf), + ) return gdf @@ -144,7 +142,7 @@ def _iter_snapped_edges(lines: list, snap_m: float): def _build_intersection_graph(lines: list, snap_precision_m: float) -> nx.Graph: - """Build a graph keyed by snapped coordinate pairs, keeping the shortest edge. + """Build a graph keyed by snapped coordinate pairs, merging duplicate segments. Raise ValueError if no edges can be created. """ @@ -155,16 +153,8 @@ def _build_intersection_graph(lines: list, snap_precision_m: float) -> nx.Graph: G = nx.Graph() for p, q, length_km in _iter_snapped_edges(lines, snap_precision_m): - if length_km <= 0: - continue - - # Add edge, automatically merging parallel edges to keep shorter one - if G.has_edge(p, q): - existing_length = G.edges[p, q]["length_km"] - if length_km < existing_length: - G.edges[p, q]["length_km"] = length_km - else: - G.add_edge(p, q, length_km=length_km) + # Snapped endpoints determine length, so repeated segments have equal lengths. + G.add_edge(p, q, length_km=length_km) if len(G.edges) == 0: raise ValueError("No valid edges created from geometries") @@ -193,7 +183,7 @@ def _validate_final_graph(G: nx.Graph, validation_config: ValidationConfig) -> N logger.debug( f"Highway graph has {len(components)} disconnected components. " f"Largest: {component_sizes[0]:,} nodes. " - f"Components will be filtered during contraction in integrated pipeline." + "The integrated pipeline applies the configured component filter after contraction." ) for u, v, data in G.edges(data=True): @@ -201,11 +191,15 @@ def _validate_final_graph(G: nx.Graph, validation_config: ValidationConfig) -> N raise ValueError(f"Edge {u}-{v} missing length_km attribute") length = data["length_km"] - if not isinstance(length, (int, float)) or length <= 0 or np.isnan(length): + if ( + not isinstance(length, (int, float)) + or length <= 0 + or not np.isfinite(length) + ): raise ValueError(f"Edge {u}-{v} has invalid length_km: {length}") # Check for excessive node degrees (indicates snapping bugs) - degrees = [len(list(G.neighbors(node))) for node in G.nodes()] + degrees = [G.degree[node] for node in G] max_degree = max(degrees) if degrees else 0 if max_degree > validation_config.max_degree_threshold: raise ValueError( @@ -213,9 +207,7 @@ def _validate_final_graph(G: nx.Graph, validation_config: ValidationConfig) -> N ) high_degree_nodes = [ - node - for node in G.nodes() - if len(list(G.neighbors(node))) > validation_config.high_degree_warning + node for node in G if G.degree[node] > validation_config.high_degree_warning ] if high_degree_nodes: logger.warning( diff --git a/topogen/integrated_graph.py b/topogen/integrated_graph.py index 99e51a8..24c987f 100644 --- a/topogen/integrated_graph.py +++ b/topogen/integrated_graph.py @@ -9,15 +9,15 @@ import json from pathlib import Path -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING import networkx as nx import numpy as np -from scipy.spatial import KDTree # type: ignore[import-untyped] +from scipy.spatial import KDTree # pyright: ignore[reportMissingTypeStubs] from shapely.geometry import Point +from topogen.context import RunContext from topogen.corridors import ( - CorridorPath, add_corridors, assign_risk_groups, extract_corridor_graph, @@ -40,173 +40,65 @@ def _contract_degree2_chains( G: nx.Graph, protected_nodes: set[tuple[float, float]] | None = None ) -> nx.Graph: - """Collapse degree-2 chains, preserving junctions, dead ends, and protected nodes. + """Contract chains without erasing alternative routes or distorting cycles. - Sum lengths into ``length_km`` and retain traversed coordinates in ``geometry``. - Isolated cycles become one edge between two of their nodes. + Keep one interior vertex for parallel chains so routing remains a simple + graph. Preserve rings and loops at junctions as their original segments. """ - logger.info("Contracting degree-2 chains") + from collections import defaultdict + from hashlib import blake2b - if protected_nodes is None: - protected_nodes = set() - - if protected_nodes: - logger.info( - f"Protecting {len(protected_nodes)} nodes from contraction (metro anchors)" - ) - - components_before = list(nx.connected_components(G)) - logger.debug( - f"Input graph: {len(G.nodes)} nodes, {len(G.edges)} edges, {len(components_before)} components" - ) - if len(components_before) > 1: - component_sizes = sorted([len(c) for c in components_before], reverse=True) - logger.debug(f"Component sizes before contraction: {component_sizes[:10]}") - - degree_counts = {} - for node in G.nodes(): - deg = len(list(G.neighbors(node))) - degree_counts[deg] = degree_counts.get(deg, 0) + 1 - logger.debug(f"Degree distribution: {dict(sorted(degree_counts.items()))}") - - contracted = nx.Graph() + protected = set() if protected_nodes is None else protected_nodes + terminals = {node for node in G if G.degree[node] != 2 or node in protected} + chains = defaultdict(list) visited = set() - processed_nodes = set() - - def _segment_id(u: tuple[float, float], v: tuple[float, float]) -> str: - """Return deterministic segment id based on sorted integerized endpoints. - - Endpoints are snapped to a grid in upstream processing, so integerization - via round() is stable. - """ - (ux, uy) = (int(round(u[0])), int(round(u[1]))) - (vx, vy) = (int(round(v[0])), int(round(v[1]))) - a = (ux, uy) - b = (vx, vy) - a, b = (a, b) if a <= b else (b, a) - return f"{a[0]}_{a[1]}__{b[0]}_{b[1]}" - - def is_contractible(node): - """Check if a node can be contracted (degree-2 and not protected).""" - return len(list(G.neighbors(node))) == 2 and node not in protected_nodes - - chains_contracted = 0 - for node in G.nodes(): - if not is_contractible(node): # a junction, dead-end, or protected node - for nbr in G.neighbors(node): - key = tuple(sorted((node, nbr))) - if key in visited: - continue - - path = [node] - length = G.edges[node, nbr]["length_km"] - visited.add(key) - - prev, curr = node, nbr - path.append(curr) - - while is_contractible(curr): - nxt = next(n for n in G.neighbors(curr) if n != prev) - length += G.edges[curr, nxt]["length_km"] - visited.add(tuple(sorted((curr, nxt)))) - path.append(nxt) - prev, curr = curr, nxt - - contracted.add_edge( - path[0], - path[-1], - length_km=length, - geometry=path, - segment_id=_segment_id(path[0], path[-1]), - ) - processed_nodes.update(path) - chains_contracted += 1 - - if len(path) > 100: - logger.debug( - f"Long chain contracted: {len(path)} nodes, {length:.1f}km from {path[0]} to {path[-1]}" - ) - - logger.debug( - f"Phase 1: Contracted {chains_contracted} chains connected to junctions" - ) - - # Second pass: handle isolated degree-2 cycles (rings) - remaining_nodes = set(G.nodes()) - processed_nodes - - processed_cycles = set() - for node in remaining_nodes: - if node in processed_cycles or len(list(G.neighbors(node))) != 2: - continue - - cycle_nodes = [] - cycle_length = 0.0 - current = node - prev = None - - while True: - cycle_nodes.append(current) - processed_cycles.add(current) - - neighbors = list(G.neighbors(current)) - if len(neighbors) != 2: - # Node no longer has exactly 2 neighbors, break to avoid infinite loop - break - next_node = neighbors[0] if neighbors[0] != prev else neighbors[1] - - cycle_length += G.edges[current, next_node]["length_km"] - - prev = current - current = next_node - - if current == node: - break - - if len(cycle_nodes) >= 3: # Only contract cycles with 3+ nodes - # Use first and "middle" node as endpoints to avoid self-loops - start_node = cycle_nodes[0] - mid_node = cycle_nodes[len(cycle_nodes) // 2] - - if start_node != mid_node: - contracted.add_edge( - start_node, - mid_node, - length_km=cycle_length, - geometry=cycle_nodes + [cycle_nodes[0]], - segment_id=_segment_id(start_node, mid_node), - ) - - if contracted.number_of_edges() == 0: - raise ValueError("Graph contraction produced empty result") - - components_after = list(nx.connected_components(contracted)) - logger.debug( - f"Output graph: {len(contracted.nodes)} nodes, {len(contracted.edges)} edges, {len(components_after)} components" - ) - - if len(components_after) > 1: - component_sizes_after = sorted([len(c) for c in components_after], reverse=True) - logger.debug(f"Component sizes after contraction: {component_sizes_after[:10]}") - - if len(components_after) != len(components_before): - logger.warning( - f"Component count changed during contraction: {len(components_before)} → {len(components_after)}" - ) - - nodes_removed = G.number_of_nodes() - contracted.number_of_nodes() - original_nodes_count = G.number_of_nodes() - if original_nodes_count > 0: - percentage = nodes_removed / original_nodes_count * 100 - logger.debug( - f"Contraction summary: removed {nodes_removed} degree-2 nodes ({percentage:.1f}%)" + for start in sorted(terminals): + for neighbor in sorted(G[start]): + if frozenset((start, neighbor)) in visited: + continue + path = [start, neighbor] + visited.add(frozenset((start, neighbor))) + while path[-1] not in terminals: + previous, current = path[-2:] + following = next(node for node in G[current] if node != previous) + visited.add(frozenset((current, following))) + path.append(following) + chains[tuple(sorted((path[0], path[-1])))].append(path) + + result = nx.Graph() + result.add_nodes_from((node, dict(G.nodes[node])) for node in sorted(terminals)) + + def add_path(path): + canonical = min(path, list(reversed(path))) + segment_id = blake2b(repr(canonical).encode(), digest_size=12).hexdigest() + result.add_edge( + path[0], + path[-1], + length_km=sum( + G[u][v]["length_km"] for u, v in zip(path, path[1:], strict=False) + ), + geometry=path, + segment_id=segment_id, ) - else: - logger.debug("Contraction summary: no nodes to process") - logger.info( - f"Contracted graph: {contracted.number_of_nodes():,} nodes, {contracted.number_of_edges():,} edges" - ) - return contracted + for (start, end), paths in sorted(chains.items()): + for path in paths: + if start == end: + for u, v in zip(path, path[1:], strict=False): + add_path([u, v]) + elif len(paths) > 1 and len(path) > 2: + add_path(path[:2]) + add_path(path[1:]) + else: + add_path(path) + # Unvisited edges belong to isolated rings; preserving them preserves both arcs. + for u, v in sorted(G.edges()): + if frozenset((u, v)) not in visited: + add_path([u, v]) + if not result.edges: + raise ValueError("Graph contraction produced empty result") + logger.info("Contracted graph: %d nodes, %d edges", len(result), len(result.edges)) + return result def _remove_slivers( @@ -217,6 +109,8 @@ def _remove_slivers( Reject an empty result or a disconnected result whose largest component falls below the configured fraction of the original node count. """ + if min_length_km == 0: + return G logger.info( f"Removing edges shorter than {min_length_km}km (sliver removal threshold)" ) @@ -237,9 +131,7 @@ def _remove_slivers( G_clean = G.copy() G_clean.remove_edges_from(edges_to_remove) - isolated_nodes = [ - node for node in G_clean.nodes() if len(list(G_clean.neighbors(node))) == 0 - ] + isolated_nodes = list(nx.isolates(G_clean)) G_clean.remove_nodes_from(isolated_nodes) nodes_remaining = len(G_clean.nodes) @@ -303,13 +195,10 @@ def _remove_slivers( return G_clean -def _keep_largest_component( - G: nx.Graph, validation_config: ValidationConfig -) -> nx.Graph: +def _keep_largest_component(G: nx.Graph) -> nx.Graph: """Return G if connected, otherwise a copy of its largest component. - Log discarded nodes. ``validation_config`` is unused; component size is - not a rejection criterion here. + Log discarded nodes; component size is not a rejection criterion here. """ if nx.is_connected(G): logger.info("Graph is already connected") @@ -433,7 +322,9 @@ def anchor_metros( return anchors -def build_integrated_graph(config: TopologyConfig) -> nx.Graph: +def build_integrated_graph( + config: TopologyConfig, *, context: RunContext | None = None +) -> nx.MultiGraph: """Build and return the metro-to-metro corridor graph for scenario assembly. The highway graph, metro anchors, and tagged corridor paths are intermediate @@ -446,8 +337,8 @@ def build_integrated_graph(config: TopologyConfig) -> nx.Graph: k=config.clustering.metro_clusters, target_crs=config.projection.target_crs, clustering_config=config.clustering, - formatting_config=config.output.formatting, conus_boundary_path=config.data_sources.conus_boundary, + context=context, ) logger.info(f"Loaded {len(metros)} metro clusters") @@ -470,19 +361,22 @@ def build_integrated_graph(config: TopologyConfig) -> nx.Graph: highway_graph, protected_nodes=anchor_nodes ) + del highway_graph highway_clean = _remove_slivers( highway_contracted, config.highway_processing.min_edge_length_km, config.validation, ) + del highway_contracted if config.highway_processing.filter_largest_component: - highway_final = _keep_largest_component(highway_clean, config.validation) + highway_final = _keep_largest_component(highway_clean) else: highway_final = highway_clean logger.info("Component filtering disabled - keeping all highway components") - logger.info("Verifying metro anchors after contraction") + del highway_clean + logger.info("Verifying metro anchors after highway filtering") missing_anchors = [] for metro in metros: @@ -494,52 +388,42 @@ def build_integrated_graph(config: TopologyConfig) -> nx.Graph: anchor_list = ", ".join( [f"{name} ({anchor})" for name, anchor in missing_anchors] ) - raise RuntimeError( - f"BUG: Protected anchor nodes were removed during contraction: {anchor_list}. " - f"This should never happen with anchor protection enabled." + raise ValueError( + f"Highway filtering removed metro anchors: {anchor_list}. " + "Check min_edge_length_km, filter_largest_component, and selected metros." ) - logger.info(f"All {len(metros)} metro anchors preserved during contraction") + logger.info(f"All {len(metros)} metro anchors survived highway filtering") logger.info("Building integrated graph") - G = highway_final.copy() + G = highway_final for metro in metros: key = metro.node_key # (x, y) tuple anchor = anchors[metro.metro_id] - if G.has_node(key): - # Merge full metro attributes into existing highway node - G.nodes[key]["node_type"] = "metro+highway" - G.nodes[key]["name"] = metro.name - G.nodes[key]["name_orig"] = metro.name_orig - G.nodes[key]["radius_km"] = metro.radius_km - G.nodes[key]["metro_id"] = metro.metro_id - G.nodes[key]["x"] = metro.centroid_x - G.nodes[key]["y"] = metro.centroid_y - G.nodes[key]["uac_code"] = metro.uac_code - G.nodes[key]["land_area_km2"] = metro.land_area_km2 - else: - G.add_node( - key, - node_type="metro", - x=metro.centroid_x, - y=metro.centroid_y, - name=metro.name, - name_orig=metro.name_orig, - radius_km=metro.radius_km, - metro_id=metro.metro_id, - uac_code=metro.uac_code, - land_area_km2=metro.land_area_km2, - ) + G.add_node( + key, + node_type="metro+highway" if G.has_node(key) else "metro", + x=metro.centroid_x, + y=metro.centroid_y, + name=metro.name, + name_orig=metro.name_orig, + radius_km=metro.radius_km, + metro_id=metro.metro_id, + uac_code=metro.uac_code, + land_area_km2=metro.land_area_km2, + ) dist_km = Point(metro.coordinates).distance(Point(anchor)) / 1000.0 + if key == anchor: + continue G.add_edge( key, anchor, edge_type="metro_anchor", length_km=dist_km, - geometry=[key, anchor], # straight line from metro centroid to anchor + geometry=[key, anchor], # Straight line from representative point to road. ) logger.info(f"Added {len(metros)} metro nodes with anchor connections") @@ -555,37 +439,19 @@ def build_integrated_graph(config: TopologyConfig) -> nx.Graph: validate_corridor_graph(corridor_graph, metros, config.validation) - if config.clustering.export_integrated_graph: - cfg_out = getattr(config, "_output_dir", None) - if cfg_out is not None and isinstance(cfg_out, (str, Path)): - logger.info("Exporting corridor graph visualization") - try: - from topogen.visualization import export_integrated_graph_map - - output_dir = Path(cfg_out) - prefix = getattr(config, "_source_path", None) - stem = Path(prefix).stem if isinstance(prefix, Path) else "scenario" - visualization_path = output_dir / f"{stem}_integrated_graph.jpg" - - export_integrated_graph_map( - metros=metros, - graph=corridor_graph, - output_path=visualization_path, - conus_boundary_path=config.data_sources.conus_boundary, - target_crs=config.projection.target_crs, - use_real_geometry=bool( - getattr(config, "_use_real_corridor_geometry", False) - ), - dpi=int(getattr(config, "_visualization_dpi", 300)), - ) - except Exception as e: - raise RuntimeError( - f"Failed to export corridor graph visualization: {e}" - ) from e - else: - logger.debug( - "Skipping corridor graph visualization (no output directory configured)" - ) + if context is not None and config.clustering.export_integrated_graph: + from topogen.visualization import export_integrated_graph_map + + context.output_dir.mkdir(parents=True, exist_ok=True) + export_integrated_graph_map( + metros=metros, + graph=corridor_graph, + output_path=context.path("integrated_graph.jpg"), + conus_boundary_path=config.data_sources.conus_boundary, + target_crs=config.projection.target_crs, + use_real_geometry=config.visualization.use_real_corridor_geometry, + dpi=config.visualization.dpi, + ) logger.info( f"Successfully built integrated graph with corridor extraction: " @@ -628,19 +494,20 @@ def validate_integrated_graph( anchor_edges = [ (u, v) - for u, v, d in graph.edges(data=True) - if (u == key or v == key) and d.get("edge_type") == "metro_anchor" + for u, v, d in graph.edges(key, data=True) + if d.get("edge_type") == "metro_anchor" ] - if len(anchor_edges) != 1: + expected = 0 if graph.nodes[key]["node_type"] == "metro+highway" else 1 + if len(anchor_edges) != expected: raise ValueError( - f"Metro {metro.name} ({metro.metro_id}) has {len(anchor_edges)} anchor edges (expected 1)" + f"Metro {metro.name} ({metro.metro_id}) has {len(anchor_edges)} anchor edges (expected {expected})" ) metro_anchor_count += 1 node_count = graph.number_of_nodes() if node_count > 0: - node_degrees = list(graph.degree()) # type: ignore[arg-type] + node_degrees = [(node, graph.degree[node]) for node in graph] degrees = [d for _, d in node_degrees] max_degree = max(degrees) if degrees else 0 if max_degree > validation_config.max_degree_threshold: @@ -672,157 +539,50 @@ def validate_integrated_graph( def save_to_json( - graph: nx.Graph, - path: Path, - crs: str, - formatting_config: FormattingConfig, + graph: nx.MultiGraph, path: Path, crs: str, formatting_config: FormattingConfig ) -> None: - """Save integrated graph to JSON format with tuple encoding. - - Args: - graph: NetworkX graph to save. - path: Output path for JSON file. - crs: Coordinate reference system string. - formatting_config: Formatting configuration for JSON output. - """ - logger.info(f"Saving integrated graph to JSON: {path}") - - def _to_python(obj: Any): - """Convert common non-JSON types to JSON-friendly Python types.""" - if isinstance(obj, (np.floating, np.integer)): - return obj.item() - if isinstance(obj, set): - # Sort for determinism when possible - try: - return sorted(list(obj)) - except Exception: - return list(obj) - if isinstance(obj, tuple): - return list(obj) - return obj - - out: dict[str, Any] = {"target_crs": crs, "nodes": [], "edges": []} - - for node, data in graph.nodes(data=True): - x, y = node - node_data = { - "id": [float(x), float(y)], - **{k: _to_python(v) for k, v in data.items()}, - } - out["nodes"].append(node_data) - - for u, v, data in graph.edges(data=True): - edge_data = { - "source": [float(u[0]), float(u[1])], - "target": [float(v[0]), float(v[1])], - **{k: _to_python(v) for k, v in data.items()}, - } - out["edges"].append(edge_data) - - registry = graph.graph.get("corridor_paths") - if registry: - serialized_paths: list[dict[str, Any]] = [] - # registry may be a dict keyed by PathId or already a list; normalize to dict iteration - if isinstance(registry, dict): - items = registry.items() - else: # pragma: no cover - defensive - try: - items = list(enumerate(registry)) # type: ignore[assignment] - except Exception: - items = [] - for _key, cp in items: # type: ignore[misc] - # cp may be a CorridorPath dataclass or a plain dict - if hasattr(cp, "metros") and hasattr(cp, "geometry"): - metros_tuple = cp.metros - path_index = int(cp.path_index) - length_km = float(cp.length_km) - segment_ids = list(cp.segment_ids) - geometry = [(float(p[0]), float(p[1])) for p in cp.geometry] - serialized_paths.append( - { - "metro_a": metros_tuple[0], - "metro_b": metros_tuple[1], - "path_index": path_index, - "length_km": length_km, - "segment_ids": segment_ids, - "geometry": [list(p) for p in geometry], - } - ) - elif isinstance(cp, dict): - # Assume already in serializable form - serialized_paths.append(cp) - out["corridor_paths"] = serialized_paths - + """Save the corridor multigraph with explicit edge keys and projected CRS.""" + if not graph.is_multigraph() or graph.is_directed(): + raise TypeError("Expected a corridor MultiGraph") + out = { + "graph_type": "corridors", + "target_crs": crs, + "nodes": [ + {"id": list(node), **attrs} for node, attrs in graph.nodes(data=True) + ], + "edges": [ + {"source": list(u), "target": list(v), "key": key, **attrs} + for u, v, key, attrs in graph.edges(keys=True, data=True) + ], + } path.parent.mkdir(parents=True, exist_ok=True) - with path.open("w") as f: - json.dump(out, f, indent=formatting_config.json_indent) - - file_size_mb = path.stat().st_size / (1024 * 1024) - logger.info(f"Saved integrated graph: {file_size_mb:.1f} MB") - - -def load_from_json(path: Path) -> tuple[nx.Graph, str]: - """Load integrated graph from JSON format with tuple decoding. - - Args: - path: Input path for JSON file. - - Returns: - Tuple of (NetworkX graph, CRS string). - """ - logger.info(f"Loading integrated graph from JSON: {path}") - - with path.open("r") as f: - data = json.load(f) - - graph = nx.Graph() - - for node_data in data["nodes"]: - key = tuple(node_data.pop("id")) - graph.add_node(key, **node_data) - - for edge_data in data["edges"]: - u = tuple(edge_data.pop("source")) - v = tuple(edge_data.pop("target")) - graph.add_edge(u, v, **edge_data) - - crs = data["target_crs"] - - try: - raw_paths = data.get("corridor_paths") - except Exception: # pragma: no cover - defensive - raw_paths = None - if raw_paths: - registry: dict[tuple[str, str, int], CorridorPath] = {} - for entry in raw_paths: - try: - metro_a = str(entry["metro_a"]) - metro_b = str(entry["metro_b"]) - path_index = int(entry["path_index"]) - length_km = float(entry.get("length_km", 0.0)) - segment_ids = list(entry.get("segment_ids", [])) - geometry = [ - (float(p[0]), float(p[1])) - for p in entry.get("geometry", []) - if isinstance(p, (list, tuple)) and len(p) >= 2 - ] - metros = tuple(sorted((metro_a, metro_b))) # type: ignore[assignment] - cp = CorridorPath( - metros=metros, # type: ignore[arg-type] - path_index=path_index, - nodes=[], - edges=[], - segment_ids=segment_ids, - length_km=length_km, - geometry=geometry, - ) - registry[(metros[0], metros[1], path_index)] = cp - except Exception: # pragma: no cover - robust to partial data - continue - if registry: - graph.graph["corridor_paths"] = registry - logger.info( - f"Loaded integrated graph: {len(graph.nodes):,} nodes, {len(graph.edges):,} edges" + path.write_text( + json.dumps(out, indent=formatting_config.json_indent, allow_nan=False) + "\n" ) - return graph, crs + +def load_from_json(path: Path) -> tuple[nx.MultiGraph, str]: + """Read the corridor contract; reject obsolete and duplicate records.""" + data = json.loads(path.read_text()) + if data.get("graph_type") != "corridors": + raise ValueError( + "Expected graph_type='corridors'; regenerate the integrated graph" + ) + graph = nx.MultiGraph() + for record in data["nodes"]: + attrs = dict(record) + node = tuple(attrs.pop("id")) + if len(node) != 2 or graph.has_node(node): + raise ValueError(f"Invalid or duplicate metro coordinates: {node}") + graph.add_node(node, **attrs) + for record in data["edges"]: + attrs = dict(record) + u, v = tuple(attrs.pop("source")), tuple(attrs.pop("target")) + key = attrs.pop("key") + if u not in graph or v not in graph or graph.has_edge(u, v, key): + raise ValueError( + f"Invalid or duplicate corridor endpoints/key: {u}, {v}, {key}" + ) + attrs["geometry"] = [tuple(point) for point in attrs["geometry"]] + graph.add_edge(u, v, key=key, **attrs) + return graph, data["target_crs"] diff --git a/topogen/library_io.py b/topogen/library_io.py new file mode 100644 index 0000000..0b0abde --- /dev/null +++ b/topogen/library_io.py @@ -0,0 +1,19 @@ +"""Read optional user libraries with one strict mapping contract.""" + +from pathlib import Path +from typing import Any + +import yaml + + +def load_user_library(file_name: str) -> dict[str, Any]: + path = Path.cwd() / "lib" / file_name + if not path.exists(): + return {} + try: + data = yaml.safe_load(path.read_text(encoding="utf-8")) + except yaml.YAMLError as exc: + raise ValueError(f"Failed to parse YAML: {path}") from exc + if not isinstance(data, dict): + raise ValueError(f"User library YAML must be a mapping: {path}") + return data diff --git a/topogen/metro_clusters.py b/topogen/metro_clusters.py index 8c64dc4..5dc97c1 100644 --- a/topogen/metro_clusters.py +++ b/topogen/metro_clusters.py @@ -9,17 +9,20 @@ import math from dataclasses import dataclass from pathlib import Path -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, cast -import geopandas as gpd # type: ignore[import-untyped] +import geopandas as gpd import numpy as np import pandas as pd +from shapely import Geometry from shapely.geometry import Point +from topogen.context import RunContext from topogen.log_config import get_logger +from topogen.naming import metro_slug if TYPE_CHECKING: - from topogen.config import ClusteringConfig, FormattingConfig + from topogen.config import ClusteringConfig logger = get_logger(__name__) @@ -37,21 +40,6 @@ class MetroCluster: centroid_y: float radius_km: float - @staticmethod - def _sanitize_metro_name(name: str) -> str: - """Remove state suffixes and normalize a metro name to a 30-character slug.""" - import re - - base = name.split(",")[0] - - clean = re.sub(r"[\s\-]+", "-", base.lower()) - - clean = re.sub(r"[^a-z0-9-]", "", clean) - - clean = re.sub(r"-+", "-", clean).strip("-") - - return clean[:30] - @property def coordinates(self) -> tuple[float, float]: """Return coordinates as (x, y) tuple.""" @@ -62,31 +50,14 @@ def node_key(self) -> tuple[float, float]: """Return node key as (x, y) tuple for graph integration.""" return (self.centroid_x, self.centroid_y) - @property - def coordinates_array(self) -> np.ndarray: - """Return coordinates as numpy array for spatial operations.""" - return np.array([self.centroid_x, self.centroid_y]) - - def distance_to(self, other: MetroCluster) -> float: - """Return Euclidean distance in projected units (meters for EPSG:5070).""" - dx = self.centroid_x - other.centroid_x - dy = self.centroid_y - other.centroid_y - return math.sqrt(dx * dx + dy * dy) - - def overlaps_with(self, other: MetroCluster) -> bool: - """Return whether the two metro circles overlap.""" - distance_m = self.distance_to(other) - combined_radius_m = (self.radius_km + other.radius_km) * 1000 # km to meters - return distance_m < combined_radius_m - def load_metro_clusters( uac_path: Path, k: int, target_crs: str, clustering_config: ClusteringConfig, - formatting_config: FormattingConfig, conus_boundary_path: Path | None = None, + context: RunContext | None = None, ) -> list[MetroCluster]: """Select metropolitan clusters from Census urban-area polygons. @@ -95,8 +66,8 @@ def load_metro_clusters( k: Number of top urban areas to select by land area. target_crs: Target coordinate reference system. clustering_config: Configuration for clustering parameters. - formatting_config: Output formatting settings; unused by this loader. conus_boundary_path: Path to CONUS boundary file for territory filtering. + context: Optional destinations for GeoJSON and map exports. Returns: List of MetroCluster objects with standardized point-radius representation. @@ -120,17 +91,14 @@ def load_metro_clusters( if gdf_raw.crs is None: raise ValueError("UAC data has no CRS information") - # Exclude Alaska and Hawaii before projecting to the CONUS CRS. + # Exclude non-contiguous states and territories before projection. if conus_boundary_path is not None: from topogen.geo_utils import create_conus_mask logger.info("Filtering urban areas to continental US (before reprojection)") - conus_mask_wgs84 = create_conus_mask( - conus_boundary_path, "EPSG:4269" - ) # Same as source CRS - # Dissolve mask in case it has multiple geometries - dissolved_mask = conus_mask_wgs84.geometry.union_all() - gdf_filtered = gdf_raw[gdf_raw.geometry.intersects(dissolved_mask)] + conus_mask = create_conus_mask(conus_boundary_path, str(gdf_raw.crs)) + mask_geometry = cast(Geometry, conus_mask.geometry.iloc[0]) + gdf_filtered = gdf_raw[gdf_raw.geometry.intersects(mask_geometry)] excluded_count = len(gdf_raw) - len(gdf_filtered) if excluded_count > 0: @@ -153,139 +121,36 @@ def load_metro_clusters( f"Only {len(gdf)} urban areas available{conus_note}, but {k} requested" ) - gdf["ALAND20"] = pd.to_numeric(gdf["ALAND20"], errors="coerce") - - override_areas = pd.DataFrame() - remaining_k = k - - if clustering_config.override_metro_clusters: - logger.info( - f"Processing {len(clustering_config.override_metro_clusters)} override metro clusters" - ) - - override_indices = [] - for override_pattern in clustering_config.override_metro_clusters: - # Find metros matching the pattern (case-insensitive partial match) - matches = gdf[ - gdf["NAME20"].str.contains(override_pattern, case=False, na=False) - ] - - if len(matches) == 0: - logger.warning( - f"Override metro pattern '{override_pattern}' matched no metros" - ) - continue - elif len(matches) > 1: - # Multiple matches - take the largest - largest_match = matches.nlargest(1, "ALAND20").iloc[0] - logger.info( - f"Override pattern '{override_pattern}' matched {len(matches)} metros, selecting largest: {largest_match['NAME20']}" - ) - override_indices.append(largest_match.name) - else: - match = matches.iloc[0] - logger.info( - f"Override pattern '{override_pattern}' matched: {match['NAME20']}" - ) - override_indices.append(match.name) - - if override_indices: - override_areas = gdf.loc[override_indices].copy().reset_index(drop=True) - gdf_remaining = gdf.drop(override_indices).copy() - remaining_k = k - len(override_areas) - - logger.info( - f"Successfully selected {len(override_areas):,} override metros:" - ) - for _idx, row in override_areas.iterrows(): - logger.info( - f" Override: {row['NAME20']} (UAC: {row['UACE20']}, Area: {row['ALAND20']:,.0f} sq m)" - ) - - logger.info(f"Remaining {remaining_k} metros to select by size ranking") - else: - gdf_remaining = gdf.copy() - logger.warning( - "No override metros found, proceeding with size-based selection only" - ) - else: - gdf_remaining = gdf.copy() - - if remaining_k > 0: - if len(gdf_remaining) < remaining_k: - raise ValueError( - f"Not enough metros available for selection: need {remaining_k} more, " - f"but only {len(gdf_remaining)} available after override selection" - ) - - size_selected = ( - gdf_remaining.sort_values("ALAND20", ascending=False) - .head(remaining_k) - .copy() - .reset_index(drop=True) - ) - - if len(size_selected) > 0: - logger.info( - f"Successfully selected {len(size_selected)} metros by size ranking:" - ) - for _idx, row in size_selected.iterrows(): - logger.info( - f" ✓ Dynamic: {row['NAME20']} (UAC: {row['UACE20']}, Area: {row['ALAND20']:,.0f} sq m)" - ) - - if len(override_areas) > 0: - top_areas = pd.concat([override_areas, size_selected], ignore_index=True) - else: - top_areas = size_selected - else: - top_areas = override_areas - - override_count = len(override_areas) if len(override_areas) > 0 else 0 - size_selected_count = remaining_k if remaining_k > 0 else 0 - - logger.info(f"Metro selection complete: {len(top_areas)} total metros") - if override_count > 0: - logger.info(f" • {override_count} override metros (forced inclusion)") - logger.info(f" • {size_selected_count} metros selected by size ranking") - else: - logger.info( - f" • {len(top_areas)} metros selected by size ranking (no overrides)" - ) - - if len(top_areas) != k: - raise ValueError(f"Expected {k} areas after selection, got {len(top_areas)}") - if not top_areas.index.equals(pd.RangeIndex(k)): + required_cols = {"UACE20", "NAME20", "ALAND20"} + if missing := required_cols - set(gdf.columns): + raise ValueError(f"Missing required columns in UAC data: {sorted(missing)}") + gdf = gdf.reset_index(drop=True) + gdf["ALAND20"] = pd.to_numeric(gdf["ALAND20"], errors="raise") + if not np.isfinite(gdf["ALAND20"]).all() or (gdf["ALAND20"] < 0).any(): + raise ValueError("UAC land areas must be finite and non-negative") + selected = [] + for pattern in clustering_config.override_metro_clusters: + matches = gdf[gdf["NAME20"].str.contains(pattern, case=False, na=False)] + if matches.empty: + raise ValueError(f"Override metro pattern '{pattern}' matched no metros") + index = matches["ALAND20"].idxmax() + if index in selected: + raise ValueError(f"Duplicate metro override selection for '{pattern}'") + selected.append(index) + if len(selected) > k: raise ValueError( - f"DataFrame index is not sequential 0-{k - 1} after reset_index(). " - f"Got index: {top_areas.index.tolist()}" + f"Metro overrides select {len(selected)} areas, but only {k} requested" ) - - required_cols = ["UACE20", "NAME20", "ALAND20"] - missing_cols = [col for col in required_cols if col not in top_areas.columns] - if missing_cols: - raise ValueError(f"Missing required columns in UAC data: {missing_cols}") + remainder = gdf.drop(selected).nlargest(k - len(selected), "ALAND20") + top_areas = gdf.loc[selected + remainder.index.tolist()].reset_index(drop=True) + logger.info("Selected %d metros (%d overrides)", len(top_areas), len(selected)) # Calculate representative points (safer than centroids for concave polygons) - centroids = np.column_stack( - [ - top_areas.geometry.representative_point().x, - top_areas.geometry.representative_point().y, - ] - ) - logger.info(f"Calculated centroids for {len(centroids)} urban areas") - - if len(centroids) > 0: - sample_idx = 0 - sample_name = ( - top_areas["NAME20"].iloc[sample_idx] - if len(top_areas) > sample_idx - else "N/A" - ) - logger.info( - f"Sample metro coordinates: {sample_name} at " - f"({centroids[sample_idx, 0]:.0f}, {centroids[sample_idx, 1]:.0f}) in {target_crs}" - ) + points = top_areas.geometry.representative_point() + centroids = np.column_stack([points.x, points.y]) + if not np.isfinite(centroids).all(): + raise ValueError("Metro coordinates must be finite") + logger.info(f"Calculated representative points for {len(centroids)} urban areas") # Calculate equivalent circular radius from land area land_areas_km2 = top_areas["ALAND20"] / 1_000_000 # Convert m² to km² @@ -301,19 +166,6 @@ def load_metro_clusters( ) logger.debug("Using equivalent circular radius: r = sqrt(area/π)") - if len(centroids) != len(top_areas): - raise ValueError( - f"Data length mismatch: {len(centroids)} centroids vs {len(top_areas)} areas" - ) - if len(land_areas_km2) != len(top_areas): - raise ValueError( - f"Data length mismatch: {len(land_areas_km2)} land areas vs {len(top_areas)} areas" - ) - if len(radii_km) != len(top_areas): - raise ValueError( - f"Data length mismatch: {len(radii_km)} radii vs {len(top_areas)} areas" - ) - # Create MetroCluster objects with UACE20 codes as stable IDs metro_clusters = [] for i in range(len(centroids)): @@ -326,17 +178,17 @@ def load_metro_clusters( original_name = top_areas["NAME20"].iloc[i] cluster = MetroCluster( metro_id=str(uace_code).strip(), - name=MetroCluster._sanitize_metro_name(original_name), + name=metro_slug(original_name), name_orig=original_name, uac_code=str(uace_code).strip(), land_area_km2=round( float(land_areas_km2.iloc[i]), clustering_config.area_precision ), centroid_x=round( - centroids[i, 0], clustering_config.coordinate_precision + float(centroids[i, 0]), clustering_config.coordinate_precision ), centroid_y=round( - centroids[i, 1], clustering_config.coordinate_precision + float(centroids[i, 1]), clustering_config.coordinate_precision ), radius_km=round(float(radii_km[i]), clustering_config.area_precision), ) @@ -360,20 +212,22 @@ def load_metro_clusters( duplicate_info.append(f"{cluster.name} ({cluster.metro_id})") raise ValueError(f"Duplicate metro IDs found: {', '.join(duplicate_info)}") - if clustering_config.export_clusters: - _export_cluster_files(metro_clusters, target_crs) + if clustering_config.export_clusters and context is not None: + if conus_boundary_path is None: + raise ValueError("Cluster maps require conus_boundary_path") + _export_cluster_files(metro_clusters, target_crs, context, conus_boundary_path) return metro_clusters -def _export_cluster_files(metro_clusters: list[MetroCluster], target_crs: str) -> None: - """Export cluster visualization files. - - Args: - metro_clusters: List of MetroCluster objects to export. - target_crs: Target coordinate reference system for export. - """ - output_dir = Path.cwd() +def _export_cluster_files( + metro_clusters: list[MetroCluster], + target_crs: str, + context: RunContext, + conus_boundary_path: Path, +) -> None: + """Export representative points as GeoJSON and a map at context destinations.""" + output_dir = context.output_dir output_dir.mkdir(parents=True, exist_ok=True) metro_data = [] @@ -392,23 +246,18 @@ def _export_cluster_files(metro_clusters: list[MetroCluster], target_crs: str) - ) if metro_data: - import pandas as pd - df = pd.DataFrame(metro_data) metro_gdf = gpd.GeoDataFrame(df) metro_gdf = metro_gdf.set_crs(target_crs) - # Prefix not available here; use fixed filename for GeoJSON - metro_path = output_dir / "metro_clusters.geojson" - if metro_gdf is not None: - metro_gdf.to_file(metro_path, driver="GeoJSON") + metro_path = context.path("metro_clusters.geojson") + metro_gdf.to_file(metro_path, driver="GeoJSON") logger.info( - f"Exported metro centroids: {metro_path} ({len(metro_data)} clusters)" + f"Exported metro representative points: {metro_path} ({len(metro_data)} clusters)" ) from topogen.visualization import export_cluster_map centroids = np.array([c.coordinates for c in metro_clusters]) - jpg_path = output_dir / "metro_clusters.jpg" - conus_boundary_path = Path("data/cb_2024_us_state_500k.zip") + jpg_path = context.path("metro_clusters.jpg") export_cluster_map(centroids, jpg_path, conus_boundary_path, target_crs) logger.info(f"Exported cluster map: {jpg_path}") diff --git a/topogen/naming.py b/topogen/naming.py index 81b676e..93c77ff 100644 --- a/topogen/naming.py +++ b/topogen/naming.py @@ -1,4 +1,4 @@ -"""Normalize metro names for configuration keys and path identifiers.""" +"""Metro configuration names and stable site-adjacency identifiers.""" from __future__ import annotations @@ -25,3 +25,8 @@ def metro_slug(name: str) -> str: collapsed = re.sub(r"-+", "-", cleaned).strip("-") return collapsed[:30] + + +def site_edge_id(source: str, target: str, key: str) -> str: + """Identify one site adjacency in the graph's edge iteration order.""" + return f"{source}|{target}|{key}" diff --git a/topogen/roles.py b/topogen/roles.py new file mode 100644 index 0000000..5aa63bc --- /dev/null +++ b/topogen/roles.py @@ -0,0 +1,26 @@ +"""Explicit role selectors and directional optic endpoint assignments.""" + +from __future__ import annotations + + +def role_pair(value: str) -> tuple[str, str]: + parts = value.split("|") + if len(parts) != 2 or any(not part.strip() for part in parts): + raise ValueError(f"Expected role pair 'role|role', got {value!r}") + return parts[0].strip(), parts[1].strip() + + +def role_optics(assignments: dict[str, str]) -> dict[tuple[str, str], str]: + """Map the local and remote endpoint roles to the local optic type.""" + resolved: dict[tuple[str, str], str] = {} + for pair, component in assignments.items(): + parts = pair.split("->") + if len(parts) != 2 or any(not part.strip() for part in parts): + raise ValueError( + f"Expected optic endpoint mapping 'role->role', got {pair!r}" + ) + key = parts[0].strip(), parts[1].strip() + if key in resolved: + raise ValueError(f"Duplicate optic endpoint mapping: {key}") + resolved[key] = component + return resolved diff --git a/topogen/scenario/__init__.py b/topogen/scenario/__init__.py index 1c6bef0..c1f97b3 100644 --- a/topogen/scenario/__init__.py +++ b/topogen/scenario/__init__.py @@ -1,3 +1,5 @@ -"""Scenario-building subpackage.""" +"""Build NetGraph scenarios from corridor multigraphs.""" -from __future__ import annotations +from .assembly import build_scenario + +__all__ = ["build_scenario"] diff --git a/topogen/scenario/artifacts.py b/topogen/scenario/artifacts.py new file mode 100644 index 0000000..95f85aa --- /dev/null +++ b/topogen/scenario/artifacts.py @@ -0,0 +1,74 @@ +"""Optional artifacts from the already resolved site and device networks.""" + +from __future__ import annotations + +import json +from collections import defaultdict +from typing import Any + +import networkx as nx +from ngraph import Network + +from topogen.config import TopologyConfig +from topogen.context import RunContext + +from .network import save_site_graph_json + + +def export_artifacts( + graph: nx.MultiGraph, + network: Network, + config: TopologyConfig, + context: RunContext, + traffic_matrices: dict[str, Any], +) -> None: + """Export requested files; errors propagate to the caller.""" + output_dir = context.output_dir + output_dir.mkdir(parents=True, exist_ok=True) + stem = context.stem + if context.debug_dir is not None: + context.debug_dir.mkdir(parents=True, exist_ok=True) + debug_path = context.debug_dir / f"{stem}_traffic_debug.json" + debug_path.write_text( + json.dumps( + {"model": config.traffic.model, "matrices": traffic_matrices}, indent=2 + ) + + "\n" + ) + save_site_graph_json( + graph, + output_dir / f"{stem}_network_graph.json", + json_indent=config.output.formatting.json_indent, + ) + if config.visualization.export_site_graph: + from topogen.visualization import export_site_graph_map + + export_site_graph_map( + graph, + output_dir / f"{stem}_site_graph.jpg", + dpi=config.visualization.dpi, + target_crs=config.projection.target_crs, + ) + if not config.visualization.export_blueprint_diagrams: + return + + from topogen.visualization import export_blueprint_diagram + + attached: dict[str, float] = defaultdict(float) + for link in network.links.values(): + for endpoint in (link.source, link.target): + attached["/".join(endpoint.split("/")[:2])] += link.capacity + representatives: dict[str, str] = {} + for site, attrs in graph.nodes(data=True): + blueprint = attrs["site_blueprint"] + previous = representatives.get(blueprint) + if previous is None or attached[site] > attached[previous]: + representatives[blueprint] = site + for blueprint, site in sorted(representatives.items()): + export_blueprint_diagram( + blueprint, + network, + site, + output_dir / f"{stem}_blueprint_{blueprint}.jpg", + dpi=config.visualization.dpi, + ) diff --git a/topogen/scenario/assembly.py b/topogen/scenario/assembly.py index 47d9542..356323d 100644 --- a/topogen/scenario/assembly.py +++ b/topogen/scenario/assembly.py @@ -2,26 +2,24 @@ from __future__ import annotations -from pathlib import Path from typing import TYPE_CHECKING, Any import yaml -from topogen.blueprints_lib import get_builtin_blueprints +from topogen.context import RunContext from topogen.log_config import get_logger +from topogen.traffic_matrix import generate_traffic_matrix +from .artifacts import export_artifacts from .config import _determine_metro_settings -from .graph_pipeline import ( - assign_per_link_capacity, - build_site_graph, - tm_based_size_capacities, - to_network_sections, -) +from .expansion import resolve_network +from .graph_pipeline import build_site_graph from .libraries import _build_blueprints_section, _build_components_section -from .network import _extract_metros_from_graph +from .network import _extract_metros_from_graph, to_network_sections from .policies import _build_failure_policy_set_section, _build_workflow_section from .risk import _build_risk_groups_section -from .traffic import _build_traffic_matrix_section +from .sizing import tm_based_size_capacities +from .traffic import to_demand_sets if TYPE_CHECKING: # pragma: no cover - import-time types only import networkx as nx @@ -32,16 +30,15 @@ def _emit_yaml(scenario: dict[str, Any], *, yaml_anchors: bool = True) -> str: - """Serialize a scenario with adjacency comments and optional YAML anchors.""" - emit_anchors = bool(yaml_anchors) - if emit_anchors: + """Serialize a scenario with optional YAML anchors.""" + if yaml_anchors: yaml_output = yaml.safe_dump( scenario, sort_keys=False, default_flow_style=False ) else: class NoAliasDumper(yaml.SafeDumper): - def ignore_aliases(self, data): # type: ignore[override] + def ignore_aliases(self, data): return True yaml_output = yaml.dump( @@ -50,450 +47,67 @@ def ignore_aliases(self, data): # type: ignore[override] sort_keys=False, default_flow_style=False, ) - yaml_output = _add_adjacency_comments(yaml_output) logger.info("Generated NetGraph scenario YAML") return yaml_output -def build_scenario(graph: "nx.Graph", config: "TopologyConfig") -> str: +def build_scenario( + graph: "nx.MultiGraph", + config: "TopologyConfig", + *, + context: RunContext | None = None, +) -> str: """Build NetGraph scenario YAML from a metro-to-metro corridor graph. Expand metros into PoP and DC sites, size links, and resolve hardware and - scenario libraries. Write site-graph JSON when an output directory is set, + scenario libraries. Write site-graph JSON when a run context is supplied, and export maps when configured. The caller validates the returned YAML with ``validate_scenario_yaml``; this function does not run workflows. """ - logger.info("Building NetGraph scenario from integrated graph") - metros = _extract_metros_from_graph(graph) - logger.info(f"Found {len(metros)} metro nodes") - metro_settings = _determine_metro_settings(metros, config) - max_sites = max((s["pop_per_metro"] for s in metro_settings.values()), default=1) - max_dc_regions = max( - (s["dc_regions_per_metro"] for s in metro_settings.values()), default=0 - ) - logger.info(f"Maximum sites per metro: {max_sites}") - logger.info(f"Maximum DC regions per metro: {max_dc_regions}") - - scenario: dict[str, Any] = {} - - try: - scenario_seed = int(getattr(config.output, "scenario_seed", 42)) - except Exception: - scenario_seed = 42 - scenario["seed"] = scenario_seed - - logger.info("Building site-level MultiGraph") - G = build_site_graph(metros, metro_settings, graph, config) - # Optional TM-based sizing before per-link capacity split - tm_based_size_capacities(G, metros, metro_settings, config) - try: - tm_enabled = bool( - getattr(getattr(config.build, "tm_sizing", object()), "enabled", False) - ) - except Exception: - tm_enabled = False - if tm_enabled: - logger.info("Assigning per-link capacities after TM-based sizing") - else: - logger.info("Assigning per-link capacities from configured base capacities") - assign_per_link_capacity(G, config) - - try: - cfg_out = getattr(config, "_output_dir", None) - if cfg_out is not None and isinstance(cfg_out, (str, Path)): - from .graph_pipeline import save_site_graph_json - - output_dir = Path(cfg_out) - src_path = getattr(config, "_source_path", None) - stem = ( - Path(src_path).stem if isinstance(src_path, (str, Path)) else "scenario" - ) - network_graph_path = output_dir / f"{stem}_network_graph.json" - logger.info( - "Saving site-level network graph to JSON: %s", - str(network_graph_path), - ) - fmt = getattr(getattr(config, "output", None), "formatting", None) - json_indent = int(getattr(fmt, "json_indent", 2)) if fmt is not None else 2 - save_site_graph_json(G, network_graph_path, json_indent=json_indent) - else: - logger.debug( - "Skipping site-level network graph save (no output directory configured)" - ) - except Exception as e: # pragma: no cover - best-effort artefact save - logger.warning("Failed to save site-level network graph: %s", e) - - try: - cfg_out = getattr(config, "_output_dir", None) - if ( - bool(getattr(config, "_export_site_graph", False)) - and cfg_out is not None - and isinstance(cfg_out, (str, Path)) - ): - from topogen.visualization import export_site_graph_map - - output_dir = Path(cfg_out) - try: - output_dir.mkdir(parents=True, exist_ok=True) - except Exception: - pass - src_path = getattr(config, "_source_path", None) - stem = ( - Path(src_path).stem if isinstance(src_path, (str, Path)) else "scenario" - ) - site_vis_path = output_dir / f"{stem}_site_graph.jpg" - logger.info( - "Exporting site-level graph visualization to: %s", str(site_vis_path) - ) - dpi = int(getattr(config, "_visualization_dpi", 300)) - target_crs = str( - getattr(getattr(config, "projection", object()), "target_crs", "") - ) - export_site_graph_map(G, site_vis_path, dpi=dpi, target_crs=target_crs) - except Exception as e: # pragma: no cover - best-effort - logger.warning("Failed to export site-level graph visualization: %s", e) - used_blueprints = { - str(data.get("site_blueprint", "")) for _n, data in G.nodes(data=True) + settings[key] + for settings in metro_settings.values() + for count, key in ( + ("pop_per_metro", "site_blueprint"), + ("dc_regions_per_metro", "dc_region_blueprint"), + ) + if settings[count] > 0 } - used_blueprints = {bp for bp in used_blueprints if bp} - builtin_blueprints = get_builtin_blueprints() - for bp_name in used_blueprints: - if bp_name not in builtin_blueprints: - available = ", ".join(sorted(builtin_blueprints.keys())) - raise ValueError(f"Unknown blueprint '{bp_name}'. Available: {available}") - - # Emit libraries first to preserve expected YAML ordering - scenario["blueprints"] = _build_blueprints_section(used_blueprints, config) - scenario["components"] = _build_components_section(config, used_blueprints) - - logger.info("Serializing network sections from MultiGraph") - groups, adjacency = to_network_sections(G, metros, metro_settings, config) - scenario["network"] = {"nodes": groups, "links": adjacency} - try: - overrides = G.graph.get("__emitted_node_overrides__", []) - if isinstance(overrides, list) and overrides: - scenario["network"]["node_rules"] = overrides - except Exception: - pass + blueprints = _build_blueprints_section(used_blueprints, config) + site_graph = build_site_graph(metros, metro_settings, graph, blueprints) + traffic_matrices = generate_traffic_matrix(metros, metro_settings, config) + tm_based_size_capacities( + site_graph, metros, metro_settings, config, traffic_matrices + ) + groups, adjacency = to_network_sections(site_graph, metros, metro_settings, config) + scenario: dict[str, Any] = { + "seed": config.output.scenario_seed, + "blueprints": blueprints, + "components": _build_components_section(config, blueprints), + "network": {"nodes": groups, "links": adjacency}, + } + node_rules = site_graph.graph.get("node_overrides", []) + if node_rules: + scenario["network"]["node_rules"] = node_rules risk_groups = _build_risk_groups_section(graph, config) if risk_groups: scenario["risk_groups"] = risk_groups - scenario["failures"] = _build_failure_policy_set_section(config) - traffic_section = _build_traffic_matrix_section(metros, metro_settings, config) - if traffic_section: - scenario["demands"] = traffic_section - scenario["workflow"] = _build_workflow_section(config) - - # Resolve link hardware from expanded endpoint roles. - from collections.abc import Mapping as _Mapping - - comp_obj = getattr(config, "components", None) - raw_optics = getattr(comp_obj, "optics", {}) if comp_obj is not None else {} - optics_enabled = isinstance(raw_optics, _Mapping) and len(raw_optics) > 0 - - try: - from ngraph.dsl.blueprints.expand import ( # type: ignore[import-not-found] - expand_network_dsl as _ng_expand, + workflow = _build_workflow_section(config) + scenario["failures"] = _build_failure_policy_set_section(config, workflow) + if traffic_matrices: + scenario["demands"] = to_demand_sets(traffic_matrices) + scenario["workflow"] = workflow + + network = resolve_network(site_graph, scenario, config.components.optics) + if context is not None: + export_artifacts( + site_graph, + network, + config, + context, + traffic_matrices, ) - - from topogen.components_lib import ( - get_builtin_components as _get_components_lib, - ) - except Exception as exc: - if optics_enabled: - raise RuntimeError( - "Late hardware resolution requires DSL expansion when optics mapping is configured" - ) from exc - _fmt = getattr(getattr(config, "output", None), "formatting", None) - _anchors = ( - bool(getattr(_fmt, "yaml_anchors", True)) if _fmt is not None else True - ) - return _emit_yaml(scenario, yaml_anchors=_anchors) - - def _count_for_optic(name: str, capacity: float) -> float: - comps = _get_components_lib() or {} - spec = comps.get(name) - if spec is None: - raise ValueError(f"Unknown component '{name}' in optics mapping") - total = float(spec.get("capacity", 0.0)) - if total <= 0.0: - raise ValueError(f"Optic '{name}' must have positive capacity") - import math as _m - - modules = float(_m.ceil(capacity / total)) - return modules - - # Build probe network from current scenario to inspect concrete roles per adjacency - try: - probe = { - "blueprints": scenario["blueprints"], - "network": scenario["network"], - } - try: - overrides = G.graph.get("__emitted_node_overrides__", []) - if isinstance(overrides, list) and overrides: - probe["network"]["node_rules"] = overrides - except Exception: - pass - net = _ng_expand(probe) - except Exception as exc: - if optics_enabled: - raise RuntimeError( - "Late hardware resolution failed to expand network DSL with optics configured" - ) from exc - _fmt = getattr(getattr(config, "output", None), "formatting", None) - _anchors = ( - bool(getattr(_fmt, "yaml_anchors", True)) if _fmt is not None else True - ) - return _emit_yaml(scenario, yaml_anchors=_anchors) - - node_role: dict[str, str] = {} - for node in net.nodes.values(): - r = str(node.attrs.get("role", "")).strip() - if r: - node_role[str(node.name)] = r - - from collections import defaultdict - - per_adj: dict[str, list[tuple[str, str, float]]] = defaultdict(list) - for link in net.links.values(): - src = str(link.source) - dst = str(link.target) - rs = node_role.get(src, "") - rd = node_role.get(dst, "") - if not (rs and rd): - continue - aid = str(link.attrs.get("adjacency_id", "")) - if not aid: - # Fallback to link_type when adjacency id is absent - aid = str(link.attrs.get("link_type", "")) - per_adj[aid].append((rs, rd, float(link.capacity))) - - optics_lookup: dict[tuple[str, str], str] = {} - if isinstance(raw_optics, _Mapping) and len(raw_optics) > 0: - for k, v in raw_optics.items(): - key = str(k) - if "|" in key: - a, b = [p.strip() for p in key.split("|", 1)] - if a and b: - optics_lookup[(a, b)] = str(v) - optics_lookup[(b, a)] = str(v) - elif "-" in key: - a, b = [p.strip() for p in key.split("-", 1)] - optics_lookup[(a, b)] = str(v) - try: - logger.debug("Optics lookup built: %d entries", len(optics_lookup)) - except Exception: - pass - if optics_lookup: - augmented_total = 0 - augmented_by_pair: dict[tuple[str, str], int] = {} - for adj in scenario["network"].get("links", []): - attrs = adj.get("attrs", {}) - aid = str(attrs.get("adjacency_id") or attrs.get("link_type") or "") - if not aid or aid not in per_adj: - continue - entries = per_adj[aid] - roles = {r for s, t, _ in entries for r in (s, t)} - if len(roles) > 2: - raise ValueError( - f"Adjacency '{aid}' expands to more than two roles: {sorted(roles)}" - ) - pairs = {(s, t) for s, t, _ in entries} - if len(pairs) != 1: - # Mixed role-pairs in one class: leave HW unset - try: - logger.debug( - "HW: skip adj='%s' due to multiple role pairs: %s", - aid, - sorted(list(pairs)), - ) - except Exception: - pass - continue - (sr, tr) = next(iter(pairs)) - optic = optics_lookup.get((sr, tr)) - if not optic: - try: - logger.debug( - "HW: no optic configured for adj='%s' pair=(%s,%s); skipping", - aid, - sr, - tr, - ) - except Exception: - pass - continue - cap = float(entries[0][2]) - count = _count_for_optic(optic, cap) - attrs.setdefault("hardware", {}) - attrs["hardware"]["source"] = {"component": optic, "count": float(count)} - attrs["hardware"]["target"] = {"component": optic, "count": float(count)} - adj["attrs"] = attrs - augmented_total += 1 - augmented_by_pair[(sr, tr)] = augmented_by_pair.get((sr, tr), 0) + 1 - try: - logger.debug( - "HW: adj='%s' pair=(%s,%s) optic=%s capacity=%s count=%.3f", - aid, - sr, - tr, - optic, - f"{cap:.0f}", - float(count), - ) - except Exception: - pass - - try: - if augmented_total > 0: - top_pairs = sorted( - augmented_by_pair.items(), key=lambda kv: kv[1], reverse=True - )[:3] - if top_pairs: - examples = ", ".join( - f"({a},{b})={cnt}" for ((a, b), cnt) in top_pairs - ) - logger.info( - "HW: applied optics to %d adjacencies (top role-pairs: %s)", - augmented_total, - examples, - ) - else: - logger.info("HW: applied optics to %d adjacencies", augmented_total) - except Exception: - pass - - _fmt = getattr(getattr(config, "output", None), "formatting", None) - _anchors = bool(getattr(_fmt, "yaml_anchors", True)) if _fmt is not None else True - try: - if bool(getattr(config, "_export_blueprint_diagrams", False)): - from topogen.visualization import export_blueprint_diagram - - used_blueprints = set(scenario.get("blueprints", {}).keys()) - attached: dict[str, float] = {} - for link in net.links.values(): - cap = float(getattr(link, "capacity", 0.0) or 0.0) - s = str(getattr(link, "source", "")) - t = str(getattr(link, "target", "")) - if s: - attached[s] = attached.get(s, 0.0) + cap - if t: - attached[t] = attached.get(t, 0.0) + cap - - site_to_bp: dict[str, str] = {} - for gpath, gdef in scenario.get("network", {}).get("nodes", {}).items(): - bp = str(gdef.get("blueprint", "")) - if not bp: - continue - # Expand ranges like pop[1-4] into concrete prefixes - import re as _re - - m = _re.match(r"^(?P.+?)\[(?P\d+)-(?:\d+)\]$", gpath) - if m: - prefix = m.group("prefix") - for node in net.nodes.values(): - nname = str(node.name) - if nname.startswith(prefix.rstrip("/")): - head = nname.split("/", 2)[0:2] - site_path = "/".join(head) - site_to_bp[site_path] = bp - else: - # Normalize to the first two components (site scope) - head = gpath.split("/", 2)[0:2] - site_path = "/".join(head) - site_to_bp[site_path] = bp - - bp_to_best_site: dict[str, str] = {} - for site_path, bp in site_to_bp.items(): - prefix = f"{site_path}/" - tot = 0.0 - for node_name, val in attached.items(): - if str(node_name).startswith(prefix): - tot += float(val) - prev_site = bp_to_best_site.get(bp) - if prev_site is None: - bp_to_best_site[bp] = site_path - else: - prev_tot = 0.0 - for node_name, val in attached.items(): - if str(node_name).startswith(f"{prev_site}/"): - prev_tot += float(val) - if tot > prev_tot: - bp_to_best_site[bp] = site_path - - cfg_out = getattr(config, "_output_dir", None) - output_dir = ( - Path(cfg_out) if isinstance(cfg_out, (str, Path)) else Path.cwd() - ) - try: - output_dir.mkdir(parents=True, exist_ok=True) - except Exception: - pass - src_path = getattr(config, "_source_path", None) - stem = ( - Path(src_path).stem if isinstance(src_path, (str, Path)) else "scenario" - ) - dpi = int(getattr(config, "_visualization_dpi", 300)) - blueprints_defs = scenario.get("blueprints", {}) - for bp in sorted(used_blueprints): - if bp not in blueprints_defs: - continue - site = bp_to_best_site.get(bp) - if not site: - continue - out_path = output_dir / f"{stem}_blueprint_{bp}.jpg" - export_blueprint_diagram( - bp, - blueprints_defs[bp], - net, - site, - out_path, - dpi=dpi, - ) - except Exception as e: # pragma: no cover - non-fatal visualization optional - logger.warning("Failed to export blueprint diagrams: %s", e) - - return _emit_yaml(scenario, yaml_anchors=_anchors) - - -def _add_adjacency_comments(yaml_content: str) -> str: - """Label intra-metro and inter-metro link sections in emitted YAML.""" - lines = yaml_content.split("\n") - result_lines: list[str] = [] - in_adjacency = False - intra_metro_added = False - inter_metro_added = False - for i, line in enumerate(lines): - if line.strip() in ("adjacency:", "links:") and not in_adjacency: - in_adjacency = True - result_lines.append(line) - continue - if ( - in_adjacency - and line - and not line.startswith(" ") - and not line.startswith("-") - ): - in_adjacency = False - if in_adjacency and line.strip().startswith("- source:"): - link_type = None - for j in range(i, min(i + 15, len(lines))): - if "link_type: intra_metro" in lines[j]: - link_type = "intra_metro" - break - elif "link_type: inter_metro_corridor" in lines[j]: - link_type = "inter_metro" - break - if link_type == "intra_metro" and not intra_metro_added: - result_lines.append(" # Intra-metro links") - intra_metro_added = True - elif link_type == "inter_metro" and not inter_metro_added: - result_lines.append(" # Inter-metro corridors") - inter_metro_added = True - result_lines.append(line) - return "\n".join(result_lines) + return _emit_yaml(scenario, yaml_anchors=config.output.formatting.yaml_anchors) diff --git a/topogen/scenario/config.py b/topogen/scenario/config.py index 43a9d35..3ffcbc8 100644 --- a/topogen/scenario/config.py +++ b/topogen/scenario/config.py @@ -1,224 +1,53 @@ -"""Resolve build defaults and overrides for each metro.""" +"""Resolve typed build defaults and per-metro overrides.""" from __future__ import annotations -from typing import TYPE_CHECKING, Any +from dataclasses import asdict +from typing import Any -from topogen.config import _normalize_int +from topogen.config import LINK_TYPES, LinkParams, TopologyConfig, parse_link_params from topogen.naming import metro_slug -if TYPE_CHECKING: # pragma: no cover - import-time types only - from topogen.config import TopologyConfig - def _determine_metro_settings( - metros: list[dict[str, Any]], config: "TopologyConfig" + metros: list[dict[str, Any]], config: TopologyConfig ) -> dict[str, dict[str, Any]]: - """Return build settings keyed by metro name, with overrides applied. - - Raise ValueError for unknown metro names or invalid resolved settings. - """ - build_config = config.build - defaults = build_config.build_defaults - overrides = build_config.build_overrides - - if metros: - available_slugs = set() - for m in metros: - available_slugs.add(metro_slug(m.get("name", ""))) - available_slugs.add(metro_slug(m.get("name_orig", m.get("name", "")))) - for override_key in overrides.keys(): - if override_key not in available_slugs: - available_list = ", ".join(sorted(available_slugs)) - raise ValueError( - "Build override references unknown metro " - f"'{override_key}'. Available metro slugs: {available_list}" - ) - - settings: dict[str, dict[str, Any]] = {} + """Resolve independent settings for each metro, rejecting invalid budgets.""" + defaults = config.build.build_defaults + overrides = config.build.build_overrides + available = { + metro_slug(name) + for metro in metros + for name in (metro["name"], metro.get("name_orig", metro["name"])) + } + if metros and (unknown := set(overrides) - available): + raise ValueError( + f"Build override references unknown metro {sorted(unknown)}. " + f"Available metro slugs: {sorted(available)}" + ) + settings = {} for metro in metros: - metro_name = metro["name"] - - # Seed with defaults (single 'match' applied symmetrically) - def _rp_list(obj: Any) -> list[str | list[str]]: - try: - rp = getattr(obj, "role_pairs", None) - except Exception: # pragma: no cover - defensive for mocks - return [] - return list(rp) if isinstance(rp, list) else [] - - metro_settings = { - "pop_per_metro": defaults.pop_per_metro, - "site_blueprint": defaults.site_blueprint, - "dc_regions_per_metro": defaults.dc_regions_per_metro, - "dc_region_blueprint": defaults.dc_region_blueprint, - "intra_metro_link": { - "capacity": _normalize_int( - defaults.intra_metro_link.capacity, - "defaults.intra_metro_link.capacity", - ), - "cost": _normalize_int( - defaults.intra_metro_link.cost, "defaults.intra_metro_link.cost" - ), - "attrs": defaults.intra_metro_link.attrs.copy(), - "match": defaults.intra_metro_link.match.copy(), - "role_pairs": _rp_list(defaults.intra_metro_link), - "striping": getattr(defaults.intra_metro_link, "striping", {}) - if isinstance(getattr(defaults.intra_metro_link, "striping", {}), dict) - else {}, - "mode": getattr(defaults.intra_metro_link, "mode", "mesh"), - }, - "inter_metro_link": { - "capacity": _normalize_int( - defaults.inter_metro_link.capacity, - "defaults.inter_metro_link.capacity", - ), - "cost": _normalize_int( - defaults.inter_metro_link.cost, "defaults.inter_metro_link.cost" - ), - "attrs": defaults.inter_metro_link.attrs.copy(), - "match": defaults.inter_metro_link.match.copy(), - "role_pairs": _rp_list(defaults.inter_metro_link), - "striping": getattr(defaults.inter_metro_link, "striping", {}) - if isinstance(getattr(defaults.inter_metro_link, "striping", {}), dict) - else {}, - "mode": getattr(defaults.inter_metro_link, "mode", "mesh"), - }, - "dc_to_pop_link": { - "capacity": _normalize_int( - defaults.dc_to_pop_link.capacity, "defaults.dc_to_pop_link.capacity" - ), - "cost": _normalize_int( - defaults.dc_to_pop_link.cost, "defaults.dc_to_pop_link.cost" - ), - "attrs": defaults.dc_to_pop_link.attrs.copy(), - "match": defaults.dc_to_pop_link.match.copy(), - "role_pairs": _rp_list(defaults.dc_to_pop_link), - "striping": getattr(defaults.dc_to_pop_link, "striping", {}) - if isinstance(getattr(defaults.dc_to_pop_link, "striping", {}), dict) - else {}, - "mode": getattr(defaults.dc_to_pop_link, "mode", "mesh"), - }, - } - - metro_name_orig = metro.get("name_orig", metro_name) - override = None - slug_sanitized = metro_slug(metro_name) - slug_original = metro_slug(metro_name_orig) - if slug_sanitized in overrides: - override = overrides[slug_sanitized] - elif slug_original in overrides: - override = overrides[slug_original] - - if override: - if "pop_per_metro" in override: - metro_settings["pop_per_metro"] = override["pop_per_metro"] - if "site_blueprint" in override: - metro_settings["site_blueprint"] = override["site_blueprint"] - if "dc_regions_per_metro" in override: - metro_settings["dc_regions_per_metro"] = override[ - "dc_regions_per_metro" - ] - if "dc_region_blueprint" in override: - metro_settings["dc_region_blueprint"] = override["dc_region_blueprint"] - if "intra_metro_link" in override: - override_intra = override["intra_metro_link"] - if "attrs" in override_intra: - metro_settings["intra_metro_link"]["attrs"].update( - override_intra["attrs"] - ) - if "match" in override_intra and isinstance( - override_intra["match"], dict - ): - metro_settings["intra_metro_link"]["match"] = override_intra[ - "match" - ] - for key, value in override_intra.items(): - if key != "attrs": - if key in {"capacity", "cost"}: - metro_settings["intra_metro_link"][key] = _normalize_int( - value, f"override.intra_metro_link.{key}" - ) - else: - metro_settings["intra_metro_link"][key] = value - if "inter_metro_link" in override: - override_inter = override["inter_metro_link"] - if "attrs" in override_inter: - metro_settings["inter_metro_link"]["attrs"].update( - override_inter["attrs"] - ) - if "match" in override_inter and isinstance( - override_inter["match"], dict - ): - metro_settings["inter_metro_link"]["match"] = override_inter[ - "match" - ] - for key, value in override_inter.items(): - if key != "attrs": - if key in {"capacity", "cost"}: - metro_settings["inter_metro_link"][key] = _normalize_int( - value, f"override.inter_metro_link.{key}" - ) - else: - metro_settings["inter_metro_link"][key] = value - if "dc_to_pop_link" in override: - override_dc_pop = override["dc_to_pop_link"] - if "attrs" in override_dc_pop: - metro_settings["dc_to_pop_link"]["attrs"].update( - override_dc_pop["attrs"] - ) - if "match" in override_dc_pop and isinstance( - override_dc_pop["match"], dict - ): - metro_settings["dc_to_pop_link"]["match"] = override_dc_pop["match"] - for key, value in override_dc_pop.items(): - if key != "attrs": - if key in {"capacity", "cost"}: - metro_settings["dc_to_pop_link"][key] = _normalize_int( - value, f"override.dc_to_pop_link.{key}" - ) - else: - metro_settings["dc_to_pop_link"][key] = value - - if metro_settings["pop_per_metro"] < 1: - raise ValueError( - f"Metro '{metro_name}' has invalid pop_per_metro: {metro_settings['pop_per_metro']}" - ) - if metro_settings["dc_regions_per_metro"] < 0: - raise ValueError( - f"Metro '{metro_name}' has invalid dc_regions_per_metro: {metro_settings['dc_regions_per_metro']}" - ) - - intra_link = metro_settings["intra_metro_link"] - if intra_link["capacity"] <= 0: - raise ValueError( - f"Metro '{metro_name}' has invalid intra_metro_link capacity: {intra_link['capacity']}" - ) - if intra_link["cost"] <= 0: - raise ValueError( - f"Metro '{metro_name}' has invalid intra_metro_link cost: {intra_link['cost']}" - ) - - inter_link = metro_settings["inter_metro_link"] - if inter_link["capacity"] <= 0: - raise ValueError( - f"Metro '{metro_name}' has invalid inter_metro_link capacity: {inter_link['capacity']}" - ) - if inter_link["cost"] < 0: - raise ValueError( - f"Metro '{metro_name}' has invalid inter_metro_link cost: {inter_link['cost']}" - ) - - dc_pop_link = metro_settings["dc_to_pop_link"] - if dc_pop_link["capacity"] <= 0: - raise ValueError( - f"Metro '{metro_name}' has invalid dc_to_pop_link capacity: {dc_pop_link['capacity']}" - ) - if dc_pop_link["cost"] < 0: - raise ValueError( - f"Metro '{metro_name}' has invalid dc_to_pop_link cost: {dc_pop_link['cost']}" - ) - - settings[metro_name] = metro_settings - + name = metro["name"] + override = overrides.get( + metro_slug(name), + overrides.get(metro_slug(metro.get("name_orig", name)), {}), + ) + resolved = asdict(defaults) + resolved.update( + {key: value for key, value in override.items() if key not in LINK_TYPES} + ) + for key in LINK_TYPES: + link = parse_link_params( + override.get(key, {}), LinkParams(**resolved[key]), key + ) + if ( + link.capacity <= 0 + or link.cost < 0 + or (key == "intra_metro_link" and link.cost == 0) + ): + raise ValueError(f"Metro '{name}' has invalid {key} capacity or cost") + resolved[key] = asdict(link) + if resolved["pop_per_metro"] < 1 or resolved["dc_regions_per_metro"] < 0: + raise ValueError(f"Metro '{name}' has invalid PoP or DC count") + settings[name] = resolved return settings diff --git a/topogen/scenario/expansion.py b/topogen/scenario/expansion.py new file mode 100644 index 0000000..6aad7a0 --- /dev/null +++ b/topogen/scenario/expansion.py @@ -0,0 +1,90 @@ +"""Resolve site budgets and hardware against one complete device network.""" + +from __future__ import annotations + +import math +from collections import defaultdict +from copy import deepcopy +from typing import Any + +import networkx as nx +from ngraph import Link, Network +from ngraph.dsl.blueprints.expand import expand_network_dsl + +from topogen.naming import site_edge_id +from topogen.roles import role_optics + + +def resolve_network( + graph: nx.MultiGraph, + scenario: dict[str, Any], + optics: dict[str, str], +) -> Network: + """Expand once, then resolve capacities and optics for each site edge. + + The returned device network, emitted DSL, and site graph receive the same + capacities and hardware. Internal blueprint links keep their own budgets. + Node rules and actual scenario blueprints participate in this expansion. + """ + network = expand_network_dsl(scenario) + links_by_edge: dict[str, list[Link]] = defaultdict(list) + for link in network.links.values(): + edge_id = link.attrs.get("site_edge") + if edge_id is not None: + links_by_edge[edge_id].append(link) + rules = {rule["attrs"]["site_edge"]: rule for rule in scenario["network"]["links"]} + optics_by_pair = role_optics(optics) + + for source, target, key, data in graph.edges(keys=True, data=True): + edge_id = site_edge_id(source, target, key) + links = links_by_edge[edge_id] + if not links: + raise ValueError( + f"Adjacency expansion produced zero links for {source}↔{target}" + ) + base = float(data["base_capacity"]) + if not math.isfinite(base) or base <= 0: + raise ValueError(f"Edge {source}-{target} has invalid base_capacity={base}") + capacity = base / len(links) + data["capacity"] = rules[edge_id]["capacity"] = capacity + for link in links: + link.capacity = capacity + + if not optics: + continue + hardware = {} + for endpoint, local, remote in ( + ("source", "source", "target"), + ("target", "target", "source"), + ): + pairs = { + ( + network.nodes[getattr(link, local)].attrs["role"], + network.nodes[getattr(link, remote)].attrs["role"], + ) + for link in links + } + missing = pairs - optics_by_pair.keys() + if missing: + raise ValueError( + f"Adjacency '{edge_id}' has no {endpoint}-end optic mapping for {sorted(missing)}" + ) + components = {optics_by_pair[pair] for pair in pairs} + if len(components) != 1: + raise ValueError( + f"Adjacency '{edge_id}' requires different optics on its {endpoint} ends; select one role pair per adjacency" + ) + component = next(iter(components)) + spec = scenario["components"][component] + optic_capacity = float(spec["capacity"]) + if not math.isfinite(optic_capacity) or optic_capacity <= 0: + raise ValueError(f"Optic '{component}' must have positive capacity") + hardware[endpoint] = { + "component": component, + "count": float(math.ceil(capacity / optic_capacity)), + } + data["hardware"] = deepcopy(hardware) + rules[edge_id]["attrs"]["hardware"] = deepcopy(hardware) + for link in links: + link.attrs["hardware"] = deepcopy(hardware) + return network diff --git a/topogen/scenario/graph_pipeline.py b/topogen/scenario/graph_pipeline.py index e0fb7d8..afae41e 100644 --- a/topogen/scenario/graph_pipeline.py +++ b/topogen/scenario/graph_pipeline.py @@ -1,44 +1,17 @@ -"""Build a site-level MultiGraph, size its links, and emit NetGraph network sections.""" +"""Build a site-level MultiGraph with explicit adjacency and stripe contracts.""" from __future__ import annotations import itertools -import json import math -from collections.abc import Iterator -from pathlib import Path -from typing import TYPE_CHECKING, Any +from typing import Any -import netgraph_core import networkx as nx -from ngraph.dsl.blueprints.expand import expand_network_dsl as _ng_expand -from ngraph.lib.nx import from_networkx as _from_networkx -from ngraph.types.base import FlowPlacement -from topogen.blueprints_lib import get_builtin_blueprints as _get_builtins -from topogen.corridors import ( - extract_corridor_edges_for_metros_graph as _extract_corridor_edges, -) from topogen.log_config import get_logger +from topogen.roles import role_pair -from .striping import ( - build_node_overrides_for_site as _stripe_build_overrides, -) -from .striping import ( - eligible_device_names_from_blueprint as _stripe_names, -) -from .striping import ( - group_by_attr as _stripe_group_by_attr, -) -from .striping import ( - group_by_width as _stripe_group_by_width, -) -from .striping import ( - make_stripe_attr_name as _stripe_attr_name, -) - -if TYPE_CHECKING: # pragma: no cover - import-time types only - from topogen.config import TopologyConfig +from .striping import StripePlanner logger = get_logger(__name__) @@ -46,208 +19,128 @@ def _metro_index_maps( metros: list[dict[str, Any]], ) -> tuple[dict[str, int], dict[Any, dict[str, Any]]]: - """Build helper maps for metro indices and lookup by node key.""" - metro_idx_map = {metro["name"]: idx for idx, metro in enumerate(metros, 1)} - metro_by_node = {metro["node_key"]: metro for metro in metros} - return metro_idx_map, metro_by_node + return ( + {metro["name"]: idx for idx, metro in enumerate(metros, 1)}, + {metro["node_key"]: metro for metro in metros}, + ) def _site_node_id(metro_idx: int, kind: str, ordinal: int) -> str: - """Return a canonical site node id like 'metro1/pop2' or 'metro3/dc1'.""" return f"metro{metro_idx}/{kind}{ordinal}" def _assign_site_positions( G: nx.MultiGraph, metros: list[dict[str, Any]], metro_idx_map: dict[str, int] ) -> None: - """Assign deterministic 2D positions for site nodes within each metro radius. - - Positions are stored on nodes as ``pos_x`` and ``pos_y`` in the target CRS - coordinate units (meters for EPSG:5070). Sites are placed on rings centered - at the metro centroid, strictly within the metro radius to avoid overlap - with the outline. - - Args: - G: Site-level MultiGraph with nodes carrying ``metro_idx``, - ``site_kind`` and ``site_ordinal`` attributes. - metros: List of metro dicts as produced by ``_extract_metros_from_graph``. - metro_idx_map: Mapping from metro name to metro index used in node ids. - """ - idx_to_metro: dict[int, dict[str, Any]] = {} - for name, idx in metro_idx_map.items(): - for m in metros: - if m.get("name") == name: - idx_to_metro[idx] = m - break + """Position PoPs/DCs on separate rings in one pass over the site inventory.""" + sites: dict[tuple[int, str], list[str]] = {} + for node, attrs in G.nodes(data=True): + sites.setdefault((attrs["metro_idx"], attrs["site_kind"]), []).append(node) + for metro in metros: + idx = metro_idx_map[metro["name"]] + radius = 1000.0 * metro["radius_km"] + for kind, fraction, phase in (("pop", 0.7, 0.0), ("dc", 0.4, math.pi / 4)): + nodes = sorted( + sites.get((idx, kind), []), + key=lambda name: G.nodes[name]["site_ordinal"], + ) + for ordinal, name in enumerate(nodes): + theta = phase + math.tau * ordinal / len(nodes) + G.nodes[name].update( + pos_x=metro["x"] + fraction * radius * math.cos(theta), + pos_y=metro["y"] + fraction * radius * math.sin(theta), + center_x=metro["x"], + center_y=metro["y"], + radius_m=radius, + ) - by_idx_pop: dict[int, list[tuple[str, int]]] = {} - by_idx_dc: dict[int, list[tuple[str, int]]] = {} - for node_id, data in G.nodes(data=True): - try: - idx = int(data.get("metro_idx", 0)) - kind = str(data.get("site_kind", "")) - ordn = int(data.get("site_ordinal", 0)) - except Exception: - continue - if kind == "pop": - by_idx_pop.setdefault(idx, []).append((str(node_id), ordn)) - elif kind == "dc": - by_idx_dc.setdefault(idx, []).append((str(node_id), ordn)) - import math as _m +def _role_match(link: dict[str, Any]) -> dict[str, Any]: + roles = sorted( + {role for pair in link.get("role_pairs", []) for role in role_pair(pair)} + ) + return ( + { + "conditions": [ + {"attr": "role", "op": "==", "value": role} for role in roles + ], + "logic": "or", + } + if roles + else link.get("match", {}) + ) - for idx, metro in idx_to_metro.items(): - cx = float(metro.get("x", 0.0)) - cy = float(metro.get("y", 0.0)) - r_km = float(metro.get("radius_km", 0.0)) - r_m = max(0.0, 1000.0 * r_km) - # POP ring radius and DC ring radius as fractions of metro radius - r_pop = 0.70 * r_m if r_m > 0.0 else 0.0 - r_dc = 0.40 * r_m if r_m > 0.0 else 0.0 +def _ring_cost(count: int, source: int, target: int, radius: float) -> int: + delta = abs(source - target) % count + return math.ceil(min(delta, count - delta) * math.tau * radius / count) - pops = sorted(by_idx_pop.get(idx, []), key=lambda t: t[1]) - dcs = sorted(by_idx_dc.get(idx, []), key=lambda t: t[1]) - def place_ring( - nodes: list[tuple[str, int]], - radius: float, - phase: float, - *, - center_x: float = cx, - center_y: float = cy, - ) -> None: - n = len(nodes) - if n == 0: - return - if n == 1: - theta = phase - x = center_x + radius * _m.cos(theta) - y = center_y + radius * _m.sin(theta) - G.nodes[nodes[0][0]]["pos_x"] = float(x) - G.nodes[nodes[0][0]]["pos_y"] = float(y) - return - for i, (node_name, _ord) in enumerate(nodes): - theta = phase + (2.0 * _m.pi * i) / float(n) - x = center_x + radius * _m.cos(theta) - y = center_y + radius * _m.sin(theta) - G.nodes[node_name]["pos_x"] = float(x) - G.nodes[node_name]["pos_y"] = float(y) +def _add_site_edge( + G: nx.MultiGraph, + source: str, + target: str, + key: str, + link: dict[str, Any], + kind: str, + cost: int, + adjacency_id: str, + source_metro: str, + target_metro: str, + match: dict[str, Any], + **metadata: Any, +) -> None: + capacity = link["capacity"] + G.add_edge( + source, + target, + key=key, + attrs=link.get("attrs", {}), + link_type=kind, + base_capacity=capacity, + target_capacity=capacity, + cost=max(cost, link["cost"]), + adjacency_id=adjacency_id, + distance_km=cost, + source_metro=source_metro, + target_metro=target_metro, + match=match, + role_pairs=link.get("role_pairs", []), + **metadata, + ) - place_ring(pops, r_pop, phase=0.0) - # Phase DC ring by 45 degrees to reduce overlap with POPs when counts match - place_ring(dcs, r_dc, phase=_m.pi / 4.0) - # Attach metro center and radius metadata to nodes (useful for visualization) - for node_name, _ in pops + dcs: - G.nodes[node_name]["center_x"] = float(cx) - G.nodes[node_name]["center_y"] = float(cy) - G.nodes[node_name]["radius_m"] = float(r_m) +def _stripe_match(attribute: str, label: str) -> dict[str, Any]: + return {"conditions": [{"attr": attribute, "op": "==", "value": label}]} def _add_intra_metro_edges( G: nx.MultiGraph, metros: list[dict[str, Any]], metro_settings: dict[str, dict[str, Any]], - config: TopologyConfig, metro_idx_map: dict[str, int], ) -> None: - """Add PoP-to-PoP edges within each metro with ring-arc cost model.""" - logger.info("Forming intra-metro PoP mesh edges for %d metros", len(metros)) for metro in metros: - metro_name = metro["name"] - idx = metro_idx_map[metro_name] - link_cfg = metro_settings[metro_name]["intra_metro_link"] - base_capacity = ( - int(link_cfg["capacity"]) - if isinstance(link_cfg, dict) - else int(getattr(link_cfg, "capacity", 0)) - ) - base_cost = ( - int(link_cfg["cost"]) - if isinstance(link_cfg, dict) - else int(getattr(link_cfg, "cost", 0)) - ) - s = int(metro_settings[metro_name]["pop_per_metro"]) - if s <= 1: - continue - radius_km = float(metro.get("radius_km", 0.0)) - ring_radius_km = max(0.0, radius_km) - - def arc_ceil( - n: int, - i: int, - j: int, - *, - _radius_km: float = ring_radius_km, - _base_cost: int = base_cost, - ) -> int: - if n <= 1 or _radius_km <= 0.0: - return _base_cost - delta = abs(i - j) % n - steps = min(delta, n - delta) - arc = steps * ((2.0 * math.pi * _radius_km) / float(n)) - return int(math.ceil(arc)) - - pairs = list(itertools.combinations(range(1, s + 1), 2)) - adj_id = f"intra_metro:{metro_name}" - for i, j in pairs: - u = _site_node_id(idx, "pop", i) - v = _site_node_id(idx, "pop", j) - cost = arc_ceil(s, i, j) - rp = ( - link_cfg.get("role_pairs", []) - if isinstance(link_cfg, dict) - else getattr(link_cfg, "role_pairs", []) - ) - if rp: - roles: set[str] = set() - for item in rp: - if isinstance(item, str): - parts = [p.strip() for p in item.split("|") if p.strip()] - elif isinstance(item, (list, tuple)) and len(item) == 2: - parts = [str(item[0]).strip(), str(item[1]).strip()] - else: - parts = [] - for r in parts: - if r: - roles.add(r) - try: - logger.debug( - "intra_metro: metro=%s role_pairs=%s -> match.roles=%s", - metro_name, - rp, - sorted(list(roles)), - ) - except Exception: - pass - match_obj = { - "conditions": [ - {"attr": "role", "op": "==", "value": r} for r in sorted(roles) - ] - } - - else: - match_obj = ( - link_cfg.get("match", {}) - if isinstance(link_cfg, dict) - else getattr(link_cfg, "match", {}) - ) or {} - - G.add_edge( - u, - v, - key=f"{adj_id}:{i}-{j}", - link_type="intra_metro", - base_capacity=base_capacity, - target_capacity=base_capacity, - cost=cost, - adjacency_id=adj_id, - distance_km=cost, - source_metro=metro_name, - target_metro=metro_name, - match=match_obj, - role_pairs=rp, + name = metro["name"] + idx = metro_idx_map[name] + settings = metro_settings[name] + link = settings["intra_metro_link"] + count = settings["pop_per_metro"] + match = _role_match(link) + adjacency = f"intra_metro:{name}" + for a, b in itertools.combinations(range(1, count + 1), 2): + _add_site_edge( + G, + _site_node_id(idx, "pop", a), + _site_node_id(idx, "pop", b), + f"{adjacency}:{a}-{b}", + link, + "intra_metro", + _ring_cost(count, a, b, metro["radius_km"]), + adjacency, + name, + name, + match, ) @@ -255,486 +148,132 @@ def _add_dc_to_pop_edges( G: nx.MultiGraph, metros: list[dict[str, Any]], metro_settings: dict[str, dict[str, Any]], - config: TopologyConfig, metro_idx_map: dict[str, int], + stripes: StripePlanner, ) -> None: - """Add DC-to-PoP edges within each metro with optional striping.""" - logger.info("Forming DC-to-PoP edges where DC regions exist") for metro in metros: - metro_name = metro["name"] - idx = metro_idx_map[metro_name] - link_cfg = metro_settings[metro_name]["dc_to_pop_link"] - base_capacity = ( - int(link_cfg["capacity"]) - if isinstance(link_cfg, dict) - else int(getattr(link_cfg, "capacity", 0)) - ) - base_cost = ( - int(link_cfg["cost"]) - if isinstance(link_cfg, dict) - else int(getattr(link_cfg, "cost", 0)) - ) - s = int(metro_settings[metro_name]["pop_per_metro"]) - d = int(metro_settings[metro_name]["dc_regions_per_metro"]) # may be 0 - if s <= 0 or d <= 0: + name = metro["name"] + idx = metro_idx_map[name] + settings = metro_settings[name] + pops, dcs = settings["pop_per_metro"], settings["dc_regions_per_metro"] + if not dcs: continue - adj_id = f"dc_to_pop:{metro_name}" - radius_km = float(metro.get("radius_km", 0.0)) - ring_radius_km = max(0.0, radius_km) - - def arc_ceil( - n: int, - i: int, - j: int, - *, - _radius_km: float = ring_radius_km, - _base_cost: int = base_cost, - ) -> int: - if n <= 1 or _radius_km <= 0.0: - return _base_cost - delta = abs(i - j) % n - steps = min(delta, n - delta) - arc = steps * ((2.0 * math.pi * _radius_km) / float(n)) - return int(math.ceil(arc)) - - rp = ( - link_cfg.get("role_pairs", []) - if isinstance(link_cfg, dict) - else getattr(link_cfg, "role_pairs", []) - ) - if rp: - roles: set[str] = set() - for item in rp: - if isinstance(item, str): - parts = [p.strip() for p in item.split("|") if p.strip()] - elif isinstance(item, (list, tuple)) and len(item) == 2: - parts = [str(item[0]).strip(), str(item[1]).strip()] - else: - parts = [] - for r in parts: - if r: - roles.add(r) - try: - logger.debug( - "dc_to_pop: metro=%s role_pairs=%s -> match.roles=%s", - metro_name, - rp, - sorted(list(roles)), + link = settings["dc_to_pop_link"] + match = _role_match(link) + adjacency = f"dc_to_pop:{name}" + if striping := link.get("striping"): + pop_groups = stripes.groups(settings["site_blueprint"], striping, match) + dc_groups = stripes.groups(settings["dc_region_blueprint"], striping, match) + asymmetric = ( + striping.get("mode", "width") == "width" and striping["width"] == 1 + ) + if not asymmetric and pop_groups.keys() != dc_groups.keys(): + raise ValueError( + f"DC-to-PoP striping mismatch in {name}: pop={list(pop_groups)}, dc={list(dc_groups)}" ) - except Exception: - pass - match_obj = { - "conditions": [ - {"attr": "role", "op": "==", "value": r} for r in sorted(roles) - ], - "logic": "or", + attribute = f"stripe_dc_{idx}" + for kind, count, groups in ( + ("pop", pops, pop_groups), + ("dc", dcs, dc_groups), + ): + for ordinal in range(1, count + 1): + stripes.attach(_site_node_id(idx, kind, ordinal), attribute, groups) + match = _stripe_match(attribute, next(iter(pop_groups))) + for dc in range(1, dcs + 1): + fraction = (dc - 1) * pops / dcs + candidates = { + math.floor(fraction) % pops + 1, + math.ceil(fraction) % pops + 1, } - - else: - match_obj = ( - link_cfg.get("match", {}) - if isinstance(link_cfg, dict) - else getattr(link_cfg, "match", {}) - ) or {} - - striping = ( - link_cfg.get("striping", {}) - if isinstance(link_cfg, dict) - else getattr(link_cfg, "striping", {}) - ) or {} - - stripe_attr: str | None = None - stripe_labels: list[str] = [] - if striping: - role_set: set[str] | None = set() - for item in rp: - if isinstance(item, str): - parts = [p.strip() for p in item.split("|") if p.strip()] - elif isinstance(item, (list, tuple)) and len(item) == 2: - parts = [str(item[0]).strip(), str(item[1]).strip()] - else: - parts = [] - for r in parts: - if r: - role_set.add(r) - if not role_set: - role_set = None - - bps = _get_builtins() - pop_bp = bps.get(metro_settings[metro_name]["site_blueprint"]) - dc_bp = bps.get(metro_settings[metro_name]["dc_region_blueprint"]) - if not pop_bp or not dc_bp: - raise ValueError(f"Unknown blueprint for metro {metro_name}") - mode = str(striping.get("mode", "width")).strip() - if mode == "width": - width_val = int(striping.get("width", 0)) - pop_names = _stripe_names(pop_bp, role_set) - dc_names = _stripe_names(dc_bp, role_set) - pop_groups = _stripe_group_by_width(pop_names, width_val) - dc_groups = _stripe_group_by_width(dc_names, width_val) - # For width==1 allow asymmetric group counts (one-vs-many is valid). - # For width>1 require equal number of groups to maintain pairing semantics. - if width_val > 1 and (len(pop_groups) != len(dc_groups)): - raise ValueError( - f"DC-to-PoP striping mismatch in {metro_name}: groups pop={len(pop_groups)} dc={len(dc_groups)}" - ) - stripe_attr = _stripe_attr_name(f"dc_{idx}") - label_to_names_pop = { - f"g{i + 1}": grp for i, grp in enumerate(pop_groups) - } - label_to_names_dc = { - f"g{i + 1}": grp for i, grp in enumerate(dc_groups) - } - elif mode == "by_attr": - attr = str(striping.get("attribute", "")).strip() - if not attr: - raise ValueError("striping.by_attr requires 'attribute'") - pop_map = _stripe_group_by_attr(pop_bp, attr=attr, roles=role_set) - dc_map = _stripe_group_by_attr(dc_bp, attr=attr, roles=role_set) - if set(pop_map.keys()) != set(dc_map.keys()): - raise ValueError( - f"DC-to-PoP striping by_attr mismatch in {metro_name}: labels pop={sorted(pop_map)} dc={sorted(dc_map)}" - ) - stripe_attr = _stripe_attr_name(f"dc_{idx}") - label_to_names_pop = {lab: pop_map[lab] for lab in sorted(pop_map)} - label_to_names_dc = {lab: dc_map[lab] for lab in sorted(dc_map)} - else: - raise ValueError(f"Unknown striping.mode: {mode}") - - G.graph.setdefault("node_overrides", []) - for p_ord in range(1, s + 1): - site = _site_node_id(idx, "pop", p_ord) - G.graph["node_overrides"].extend( - _stripe_build_overrides(site, stripe_attr, label_to_names_pop) - ) - for d_ord in range(1, d + 1): - site = _site_node_id(idx, "dc", d_ord) - G.graph["node_overrides"].extend( - _stripe_build_overrides(site, stripe_attr, label_to_names_dc) - ) - stripe_labels = list(label_to_names_pop.keys()) - - for dc in range(1, d + 1): - for p in range(1, s + 1): - u = _site_node_id(idx, "dc", dc) - v = _site_node_id(idx, "pop", p) - # Place DCs evenly around the ring and compute nearest POP-based arc distance - n = s - frac_index = (dc - 1) * (n / max(d, 1)) - cand_idxs = { - int(math.floor(frac_index)) % n + 1, - int(math.ceil(frac_index)) % n + 1, - } - raw_arc = ( - min(arc_ceil(n, p, c) for c in cand_idxs) - if ring_radius_km > 0.0 - else base_cost + for pop in range(1, pops + 1): + cost = max( + 1, + min( + _ring_cost(pops, pop, candidate, metro["radius_km"]) + for candidate in candidates + ), ) - # Enforce strictly positive cost to avoid zero-length links - cost_arc = max(1, int(raw_arc)) - - if not striping: - match_payload = match_obj - else: - # pick first stripe by default - label = stripe_labels[0] if stripe_labels else "g1" - match_payload = { - "conditions": [ - {"attr": stripe_attr, "op": "==", "value": label} - ] - } - - G.add_edge( - u, - v, - key=f"{adj_id}:{dc}-{p}", - link_type="dc_to_pop", - base_capacity=base_capacity, - target_capacity=base_capacity, - cost=cost_arc, - adjacency_id=adj_id, - distance_km=cost_arc, - source_metro=metro_name, - target_metro=metro_name, - match=match_payload, - role_pairs=rp, + _add_site_edge( + G, + _site_node_id(idx, "dc", dc), + _site_node_id(idx, "pop", pop), + f"{adjacency}:{dc}-{pop}", + link, + "dc_to_pop", + cost, + adjacency, + name, + name, + match, ) def _add_inter_metro_edges( G: nx.MultiGraph, - metros: list[dict[str, Any]], metro_settings: dict[str, dict[str, Any]], - graph: nx.Graph, - config: TopologyConfig, + graph: nx.MultiGraph, metro_idx_map: dict[str, int], metro_by_node: dict[Any, dict[str, Any]], + stripes: StripePlanner, ) -> None: - """Add PoP-to-PoP edges across metro corridors with optional striping.""" - logger.info("Forming inter-metro corridor edges from integrated graph") - corridor_edges = _extract_corridor_edges(graph) - for edge in corridor_edges: - src = metro_by_node.get(edge["source"]) # type: ignore[index] - tgt = metro_by_node.get(edge["target"]) # type: ignore[index] - if not src or not tgt: - logger.warning("Skipping corridor with unknown metros: %s", edge) - continue - s_name = src["name"] - t_name = tgt["name"] - s_idx = metro_idx_map[s_name] - t_idx = metro_idx_map[t_name] - s_sites = int(metro_settings[s_name]["pop_per_metro"]) # type: ignore[index] - t_sites = int(metro_settings[t_name]["pop_per_metro"]) # type: ignore[index] - # Inter-metro precedence: use source metro's link config - src_cfg = metro_settings[s_name]["inter_metro_link"] - base_capacity = ( - int(src_cfg["capacity"]) - if isinstance(src_cfg, dict) - else int(getattr(src_cfg, "capacity", 0)) - ) - # Corridor length km preferred; otherwise use source metro's configured cost - cost = int( - math.ceil( - float( - edge.get( - "length_km", - ( - src_cfg.get("cost", 1) - if isinstance(src_cfg, dict) - else getattr(src_cfg, "cost", 1) - ), - ) - ) - ) - ) - risk_groups = list(edge.get("risk_groups", [])) - euclid_km = edge.get("euclidean_km") - detour_ratio = edge.get("detour_ratio") - adj_id = f"inter_metro:{min(s_idx, t_idx)}-{max(s_idx, t_idx)}" - striping = ( - src_cfg.get("striping", {}) - if isinstance(src_cfg, dict) - else getattr(src_cfg, "striping", {}) - ) or {} - - match_base = ( - { - "conditions": [ - { - "attr": "role", - "op": "==", - "value": r, - } - for r in sorted( - { - part.strip() - for item in ( - src_cfg.get("role_pairs", []) - if isinstance(src_cfg, dict) - else getattr(src_cfg, "role_pairs", []) - ) - for part in ( - item.split("|") - if isinstance(item, str) - else [ - str(item[0]) if len(item) > 0 else "", - str(item[1]) if len(item) > 1 else "", - ] - if isinstance(item, (list, tuple)) - else [] - ) - if part.strip() - } - ) - ], - "logic": "or", - } - if ( - src_cfg.get("role_pairs", []) - if isinstance(src_cfg, dict) - else getattr(src_cfg, "role_pairs", []) - ) - else ( - src_cfg.get("match", {}) - if isinstance(src_cfg, dict) - else getattr(src_cfg, "match", {}) - ) - ) - - if not striping: - mode_val = ( - src_cfg.get("mode", "mesh") - if isinstance(src_cfg, dict) - else getattr(src_cfg, "mode", "mesh") - ) - mode = str(mode_val).strip().lower() - if mode == "one_to_one": - limit = min(int(s_sites), int(t_sites)) - for p in range(1, limit + 1): - u = _site_node_id(s_idx, "pop", p) - v = _site_node_id(t_idx, "pop", p) - G.add_edge( - u, - v, - key=f"{adj_id}:{p}-{p}", - link_type="inter_metro_corridor", - base_capacity=base_capacity, - target_capacity=base_capacity, - cost=cost, - adjacency_id=adj_id, - distance_km=cost, - source_metro=s_name, - target_metro=t_name, - risk_groups=risk_groups, - euclidean_km=euclid_km, - detour_ratio=detour_ratio, - match=match_base, - role_pairs=( - src_cfg.get("role_pairs", []) - if isinstance(src_cfg, dict) - else getattr(src_cfg, "role_pairs", []) - ), - ) - else: - for p in range(1, s_sites + 1): - for q in range(1, t_sites + 1): - u = _site_node_id(s_idx, "pop", p) - v = _site_node_id(t_idx, "pop", q) - G.add_edge( - u, - v, - key=f"{adj_id}:{p}-{q}", - link_type="inter_metro_corridor", - base_capacity=base_capacity, - target_capacity=base_capacity, - cost=cost, - adjacency_id=adj_id, - distance_km=cost, - source_metro=s_name, - target_metro=t_name, - risk_groups=risk_groups, - euclidean_km=euclid_km, - detour_ratio=detour_ratio, - match=match_base, - role_pairs=( - src_cfg.get("role_pairs", []) - if isinstance(src_cfg, dict) - else getattr(src_cfg, "role_pairs", []) - ), - ) - continue - - bps = _get_builtins() - roles: set[str] | None = set() - for item in ( - src_cfg.get("role_pairs", []) - if isinstance(src_cfg, dict) - else getattr(src_cfg, "role_pairs", []) - ): - if isinstance(item, str): - parts = [p.strip() for p in item.split("|") if p.strip()] - elif isinstance(item, (list, tuple)) and len(item) == 2: - parts = [str(item[0]).strip(), str(item[1]).strip()] - else: - parts = [] - for r in parts: - if r: - roles.add(r) - if not roles: - roles = None - - bp_A = bps.get(metro_settings[s_name]["site_blueprint"]) # type: ignore[index] - bp_B = bps.get(metro_settings[t_name]["site_blueprint"]) # type: ignore[index] - if not bp_A or not bp_B: - raise ValueError(f"Unknown blueprint for metros {s_name} or {t_name}") - mode = str(striping.get("mode", "width")).strip() - - if mode == "width": - width_val = int(striping.get("width", 0)) - names_A = _stripe_names(bp_A, roles) - names_B = _stripe_names(bp_B, roles) - groups_A = _stripe_group_by_width(names_A, width_val) - groups_B = _stripe_group_by_width(names_B, width_val) - if len(groups_A) != len(groups_B): - raise ValueError( - f"Inter-metro striping mismatch: groups {s_name}={len(groups_A)} {t_name}={len(groups_B)}" - ) - label_to_names_A = {f"g{i + 1}": grp for i, grp in enumerate(groups_A)} - label_to_names_B = {f"g{i + 1}": grp for i, grp in enumerate(groups_B)} - elif mode == "by_attr": - attr = str(striping.get("attribute", "")).strip() - if not attr: - raise ValueError("striping.by_attr requires 'attribute'") - map_A = _stripe_group_by_attr(bp_A, attr=attr, roles=roles) - map_B = _stripe_group_by_attr(bp_B, attr=attr, roles=roles) - if set(map_A.keys()) != set(map_B.keys()): + for u, v, path_index, edge in graph.edges(keys=True, data=True): + source, target = metro_by_node[u], metro_by_node[v] + a, b = source["name"], target["name"] + ai, bi = metro_idx_map[a], metro_idx_map[b] + ac, bc = metro_settings[a], metro_settings[b] + # The first metro in the graph's node order supplies the corridor policy. + link = ac["inter_metro_link"] + match = _role_match(link) + ap, bp = ac["pop_per_metro"], bc["pop_per_metro"] + mode = link.get("mode", "mesh") + if mode == "one_to_one": + pairs = [(p, p) for p in range(1, min(ap, bp) + 1)] + elif mode == "mesh": + pairs = list(itertools.product(range(1, ap + 1), range(1, bp + 1))) + else: + raise ValueError(f"Unknown inter-metro mode: {mode}") + cost = math.ceil(edge["length_km"]) + adjacency = f"inter_metro:{min(ai, bi)}-{max(ai, bi)}:path{path_index}" + labels: list[str] = [] + attribute = f"stripe_im_{min(ai, bi)}_{max(ai, bi)}" + if striping := link.get("striping"): + left = stripes.groups(ac["site_blueprint"], striping, match) + right = stripes.groups(bc["site_blueprint"], striping, match) + if left.keys() != right.keys(): raise ValueError( - f"Inter-metro striping by_attr mismatch: {s_name} labels={sorted(map_A)} {t_name} labels={sorted(map_B)}" + f"Inter-metro striping mismatch: {a}={list(left)}, {b}={list(right)}" ) - label_to_names_A = {lab: map_A[lab] for lab in sorted(map_A)} - label_to_names_B = {lab: map_B[lab] for lab in sorted(map_B)} - else: - raise ValueError(f"Unknown striping.mode: {mode}") - - stripe_attr = _stripe_attr_name(f"im_{min(s_idx, t_idx)}_{max(s_idx, t_idx)}") - G.graph.setdefault("node_overrides", []) - for p_ord in range(1, s_sites + 1): - site = _site_node_id(s_idx, "pop", p_ord) - G.graph["node_overrides"].extend( - _stripe_build_overrides(site, stripe_attr, label_to_names_A) - ) - for q_ord in range(1, t_sites + 1): - site = _site_node_id(t_idx, "pop", q_ord) - G.graph["node_overrides"].extend( - _stripe_build_overrides(site, stripe_attr, label_to_names_B) + labels = list(left) + for idx, count, groups in ((ai, ap, left), (bi, bp, right)): + for ordinal in range(1, count + 1): + stripes.attach( + _site_node_id(idx, "pop", ordinal), attribute, groups + ) + for ordinal, (p, q) in enumerate(pairs): + label = labels[ordinal % len(labels)] if labels else None + _add_site_edge( + G, + _site_node_id(ai, "pop", p), + _site_node_id(bi, "pop", q), + f"{adjacency}:{p}-{q}", + link, + "inter_metro_corridor", + cost, + f"{adjacency}:{label}" if label is not None else adjacency, + a, + b, + _stripe_match(attribute, label) if label is not None else match, + risk_groups=list(edge.get("risk_groups", [])), + euclidean_km=edge.get("euclidean_km"), + detour_ratio=edge.get("detour_ratio"), ) - labels = list(label_to_names_A.keys()) - # Deterministic assignment: cycle sites then groups across corridor edges - link_index = 0 - for p in range(1, s_sites + 1): - for q in range(1, t_sites + 1): - label = labels[(link_index) % len(labels)] - u = _site_node_id(s_idx, "pop", p) - v = _site_node_id(t_idx, "pop", q) - G.add_edge( - u, - v, - key=f"{adj_id}:{p}-{q}", - link_type="inter_metro_corridor", - base_capacity=base_capacity, - target_capacity=base_capacity, - cost=cost, - adjacency_id=f"{adj_id}:{label}", - distance_km=cost, - source_metro=s_name, - target_metro=t_name, - risk_groups=risk_groups, - euclidean_km=euclid_km, - detour_ratio=detour_ratio, - match={ - "conditions": [ - { - "attr": stripe_attr, - "op": "==", - "value": label, - } - ] - }, - role_pairs=( - src_cfg.get("role_pairs", []) - if isinstance(src_cfg, dict) - else getattr(src_cfg, "role_pairs", []) - ), - ) - link_index += 1 - def build_site_graph( metros: list[dict[str, Any]], metro_settings: dict[str, dict[str, Any]], - integrated_graph: nx.Graph, - config: TopologyConfig, + integrated_graph: nx.MultiGraph, + blueprints: dict[str, Any], ) -> nx.MultiGraph: """Construct a site-level MultiGraph with nodes and edges per adjacency. @@ -742,7 +281,7 @@ def build_site_graph( their blueprints attached as node attributes. Edges are added for three adjacency families: - - intra_metro: PoP↔PoP within a metro on a ring with arc-length-based cost. + - intra_metro: full mesh of PoPs within a metro, with shortest ring-arc costs. - dc_to_pop: DC region↔PoP within a metro, optionally striped. - inter_metro_corridor: PoP↔PoP across metros along discovered corridors, optionally striped. @@ -757,9 +296,9 @@ def build_site_graph( for metro in metros: name = metro["name"] - s = int(metro_settings[name]["pop_per_metro"]) # type: ignore[index] - d = int(metro_settings[name]["dc_regions_per_metro"]) # type: ignore[index] - if (s > 1 or d > 0) and float(metro.get("radius_km", 0.0)) <= 0.0: + s = metro_settings[name]["pop_per_metro"] + d = metro_settings[name]["dc_regions_per_metro"] + if (s > 1 or d > 0) and float(metro["radius_km"]) <= 0.0: logger.error( "Metro %s requires positive radius_km for ring-based adjacency (pop_per_metro=%d, dc_regions=%d)", name, @@ -773,10 +312,10 @@ def build_site_graph( for metro in metros: name = metro["name"] idx = metro_idx_map[name] - s = int(metro_settings[name]["pop_per_metro"]) # type: ignore[index] - d = int(metro_settings[name]["dc_regions_per_metro"]) # type: ignore[index] - pop_blueprint = str(metro_settings[name]["site_blueprint"]) # type: ignore[index] - dc_blueprint = str(metro_settings[name]["dc_region_blueprint"]) # type: ignore[index] + s = metro_settings[name]["pop_per_metro"] + d = metro_settings[name]["dc_regions_per_metro"] + pop_blueprint = metro_settings[name]["site_blueprint"] + dc_blueprint = metro_settings[name]["dc_region_blueprint"] for p in range(1, s + 1): node_id = _site_node_id(idx, "pop", p) G.add_node( @@ -800,17 +339,19 @@ def build_site_graph( _assign_site_positions(G, metros, metro_idx_map) - _add_intra_metro_edges(G, metros, metro_settings, config, metro_idx_map) - _add_dc_to_pop_edges(G, metros, metro_settings, config, metro_idx_map) + stripes = StripePlanner(blueprints) + _add_intra_metro_edges(G, metros, metro_settings, metro_idx_map) + _add_dc_to_pop_edges(G, metros, metro_settings, metro_idx_map, stripes) _add_inter_metro_edges( G, - metros, metro_settings, integrated_graph, - config, metro_idx_map, metro_by_node, + stripes, ) + if rules := stripes.node_rules(): + G.graph["node_overrides"] = rules logger.info( "MultiGraph built: %d nodes, %d edges (multi-edges counted)", @@ -818,692 +359,3 @@ def build_site_graph( G.number_of_edges(), ) return G - - -def assign_per_link_capacity(G: nx.MultiGraph, config: TopologyConfig) -> None: - """Assign per-link capacity based on final DSL expansion link count. - - The configured ``base_capacity`` on an edge is interpreted as the total - capacity budget for that site-to-site adjacency. We compute how many - concrete links the NetGraph DSL will produce for this adjacency (given the - endpoint site blueprints, match filters, and the ``one_to_one`` pattern), - then set per-link capacity to ``base_capacity / num_links``. - - Link counts come from NetGraph expansion of the two endpoint sites and - the adjacency under evaluation. - - Args: - G: Site-level MultiGraph with edges carrying ``base_capacity`` and node - attributes ``site_blueprint``. - config: Topology configuration; unused by this stage. - - Raises: - ValueError: If an edge lacks ``base_capacity`` or the expansion yields - zero links. - """ - logger.info("Assigning per-link capacities by splitting base over expansion size") - - builtins = _get_builtins() - - def _estimate_link_count(u_id: str, v_id: str, edge_data: dict[str, Any]) -> int: - bp_u = str(G.nodes[u_id].get("site_blueprint", "")) - bp_v = str(G.nodes[v_id].get("site_blueprint", "")) - if not bp_u or not bp_v: - raise ValueError(f"Missing site_blueprint for nodes {u_id} or {v_id}") - - # Build a minimal DSL using only the two endpoints and one adjacency. - # Tag links from our adjacency so we can count them precisely. - raw_match = edge_data.get("match", {}) - match_dict: dict[str, Any] = raw_match if isinstance(raw_match, dict) else {} - - dsl = { - "blueprints": builtins, - "network": { - "nodes": { - u_id: {"blueprint": bp_u}, - v_id: {"blueprint": bp_v}, - }, - "links": [ - { - "source": {"path": u_id, "match": match_dict}, - "target": {"path": v_id, "match": match_dict}, - "pattern": "one_to_one", - "capacity": 1.0, - "cost": 1.0, - "attrs": {"_tg_tmp": "count_me"}, - } - ], - }, - } - try: - overrides = G.graph.get("node_overrides", []) - if isinstance(overrides, list) and overrides: - dsl["network"]["node_rules"] = overrides - except Exception: - pass - net = _ng_expand(dsl) - # Count only links created by our adjacency (tagged with _tg_tmp) - count = sum( - 1 - for _lid, link in net.links.items() - if link.attrs.get("_tg_tmp") == "count_me" - ) - return int(count) - - for u, v, k, data in G.edges(keys=True, data=True): - base_val = data.get("base_capacity") - if base_val is None: - raise ValueError(f"Edge {u}-{v} missing base_capacity") - try: - base_capacity = float(base_val) - except Exception as exc: # noqa: BLE001 - raise ValueError( - f"Edge {u}-{v} has non-numeric base_capacity: {base_val!r}" - ) from exc - if base_capacity <= 0: - raise ValueError( - f"Edge {u}-{v} (type={data.get('link_type')}) has invalid base_capacity={base_capacity}" - ) - - num_links = _estimate_link_count(u, v, data) - if num_links <= 0: - raise ValueError( - f"Adjacency expansion produced zero links for {u}↔{v} (pattern=one_to_one, match={data.get('match')})" - ) - - per_link = base_capacity / float(num_links) - G.edges[u, v, k]["capacity"] = per_link - - -def _parse_tm_endpoint_to_metro_idx(endpoint: str) -> int | None: - """Extract metro index from a traffic matrix endpoint regex/path. - - Accepts strings like '^metro3/dc2/.*' or 'metro3/dc2'. Returns 3. - """ - try: - s = endpoint.lstrip("^") - if not s.startswith("metro"): - return None - rest = s[5:] - digits: list[str] = [] - for ch in rest: - if ch.isdigit(): - digits.append(ch) - else: - break - return int("".join(digits)) if digits else None - except Exception: - return None - - -def _resolve_flow_placement(flow_placement: str) -> FlowPlacement: - """Map config string to FlowPlacement enum.""" - fp_str = str(flow_placement).upper() - if fp_str == "EQUAL_BALANCED": - return FlowPlacement.EQUAL_BALANCED - return FlowPlacement.PROPORTIONAL - - -def _tm_metro_demands( - demands: list[dict[str, Any]], dc_counts: dict[int, int] -) -> Iterator[tuple[int, int, float]]: - """Collapse the generator's concrete and uniform demands to metro pairs.""" - uniform_selector = "(metro[0-9]+/dc[0-9]+)" - for demand in demands: - volume = float(demand.get("demand", 0.0)) - if volume <= 0.0: - continue - source = str(demand.get("source_path", "")) - target = str(demand.get("sink_path", "")) - if source == target == uniform_selector and demand.get("mode") == "pairwise": - total_dc = sum(dc_counts.values()) - if total_dc < 2: - raise ValueError( - "TM sizing: pairwise traffic requires at least two DCs" - ) - # NetGraph splits volume over all ordered, distinct DC node pairs. - # Same-metro pairs remain in the denominator but use no corridors. - per_pair = volume / (total_dc * (total_dc - 1)) - for src, src_count in dc_counts.items(): - for dst, dst_count in dc_counts.items(): - if src != dst and src_count > 0 and dst_count > 0: - yield src, dst, per_pair * src_count * dst_count - continue - src = _parse_tm_endpoint_to_metro_idx(source) - dst = _parse_tm_endpoint_to_metro_idx(target) - if src is None or dst is None: - raise ValueError( - f"TM sizing: unsupported demand endpoint ({source!r} -> {target!r})" - ) - if src != dst: - yield src, dst, volume - - -def tm_based_size_capacities( - G: nx.MultiGraph, - metros: list[dict[str, Any]], - metro_settings: dict[str, dict[str, Any]], - config: TopologyConfig, -) -> None: - """Size links from generated traffic routed on a collapsed metro graph. - - Preserve parallel corridors and route each demand with the configured flow - split. Size each corridor for its peak directional load, then apply headroom - and round up to capacity increments. Derive DC-to-PoP and intra-metro - capacities from PoP egress, respecting configured minimums when enabled. - """ - sizing_cfg = getattr(getattr(config, "build", None), "tm_sizing", None) - if getattr(sizing_cfg, "enabled", False) is not True: - return - - from topogen.traffic_matrix import generate_traffic_matrix - - logger.info("TM sizing: generating early traffic matrix") - - traffic_cfg = getattr(config, "traffic", None) - if not getattr(traffic_cfg, "enabled", False): - raise ValueError( - "TM sizing is enabled but traffic generation is disabled in configuration" - ) - - tm_map = generate_traffic_matrix(metros, metro_settings, config) - matrix_name = getattr(sizing_cfg, "matrix_name", None) or getattr( - getattr(config, "traffic", object()), "matrix_name", "default" - ) - demands = list(tm_map.get(matrix_name, [])) - try: - dc_counts = { - idx: int(metro_settings[metro["name"]].get("dc_regions_per_metro", 0)) - for idx, metro in enumerate(metros, 1) - } - except Exception as exc: - raise ValueError("TM sizing: failed to determine DC region count") from exc - if sum(dc_counts.values()) <= 0: - raise ValueError( - "TM sizing is enabled but no DC regions are configured (dc_regions_per_metro == 0)" - ) - if not demands: - raise ValueError( - f"TM sizing: traffic matrix '{matrix_name}' is empty despite enabled traffic" - ) - - metro_idx_map = {m["name"]: idx for idx, m in enumerate(metros, 1)} - for idx, _ in enumerate(metros, 1): - metro_idx_map.setdefault(f"metro{idx}", idx) - - # Build temporary NetworkX MultiDiGraph with metro nodes and inter-metro corridors. - # MultiDiGraph is required to preserve parallel edges between the same metro pair - # (e.g., striped corridors with multiple links per corridor). - H: nx.MultiDiGraph = nx.MultiDiGraph() - for idx in set(metro_idx_map.values()): - H.add_node(idx) - - # Map to track correspondence between H edges and G edges. - # Key is (src_metro_idx, dst_metro_idx, h_edge_key) to handle parallel edges. - # Both directions reference the same physical edge, sized once for peak load. - g_edge_refs: dict[tuple[int, int, Any], tuple[Any, Any, Any]] = {} - - for u_g, v_g, k_g, data in G.edges(keys=True, data=True): - if str(data.get("link_type")) != "inter_metro_corridor": - continue - src_name = data.get("source_metro") - tgt_name = data.get("target_metro") - if not isinstance(src_name, str) or not isinstance(tgt_name, str): - raise ValueError( - "TM sizing: inter-metro edge missing source_metro/target_metro attributes" - ) - - s_idx = metro_idx_map.get(src_name) - t_idx = metro_idx_map.get(tgt_name) - if s_idx is None or t_idx is None: - raise ValueError( - f"TM sizing: unknown metro name(s) on inter-metro edge: {src_name!r}, {tgt_name!r}" - ) - - cost = int(data.get("cost", 1)) - # Let H allocate keys: G keys can repeat across different PoP pairs - # that collapse to the same metro pair. - h_key_fwd = H.add_edge(s_idx, t_idx, capacity=1e15, cost=cost) - g_edge_refs[(s_idx, t_idx, h_key_fwd)] = (u_g, v_g, k_g) - - h_key_rev = H.add_edge(t_idx, s_idx, capacity=1e15, cost=cost) - g_edge_refs[(t_idx, s_idx, h_key_rev)] = (u_g, v_g, k_g) - - if H.number_of_edges() == 0: - raise ValueError( - "TM sizing: no inter-metro corridor edges present in site graph" - ) - - # Convert NetworkX graph to netgraph_core format. - # bidirectional=False because we already added explicit reverse edges above. - multidigraph, node_map, edge_map = _from_networkx(H, bidirectional=False) - num_nodes = multidigraph.num_nodes() - - backend = netgraph_core.Backend.cpu() - algorithms = netgraph_core.Algorithms(backend) - handle = algorithms.build_graph(multidigraph) - - flow_state = netgraph_core.FlowState(multidigraph) - - fp_enum = _resolve_flow_placement( - getattr(sizing_cfg, "flow_placement", "EQUAL_BALANCED") - ) - core_fp = ( - netgraph_core.FlowPlacement.EQUAL_BALANCED - if fp_enum == FlowPlacement.EQUAL_BALANCED - else netgraph_core.FlowPlacement.PROPORTIONAL - ) - - edge_selection = netgraph_core.EdgeSelection( - multi_edge=True, - require_capacity=False, # IP-style routing based on cost only - tie_break=netgraph_core.EdgeTieBreak.DETERMINISTIC, - ) - - for s_metro, t_metro, demand_val in _tm_metro_demands(demands, dc_counts): - s_idx = node_map.to_index.get(s_metro) - t_idx = node_map.to_index.get(t_metro) - if s_idx is None or t_idx is None: - raise ValueError( - f"TM sizing: metro index out of range (src={s_metro}, dst={t_metro}, num_nodes={num_nodes})" - ) - - try: - dists, pred_dag = algorithms.spf( - handle, - src=s_idx, - dst=t_idx, - selection=edge_selection, - multipath=True, - ) - except Exception as exc: - raise ValueError( - f"TM sizing: SPF failed for metro {s_metro}->{t_metro}: {exc}" - ) from exc - - placed = flow_state.place_on_dag( - src=s_idx, - dst=t_idx, - dag=pred_dag, - requested_flow=demand_val, - flow_placement=core_fp, - ) - - try: - cost = int(dists[t_idx]) - logger.debug( - "TM sizing: placed %s Gbps from metro%d->metro%d (cost=%s)", - f"{placed:,.1f}", - s_metro, - t_metro, - f"{cost:,}", - ) - except Exception: - pass - - edge_flows_arr = flow_state.edge_flow_view() - ext_edge_ids_view = multidigraph.ext_edge_ids_view() - edge_loads: dict[tuple[Any, Any, Any], float] = {} - for edge_idx in range(len(edge_flows_arr)): - flow_val = float(edge_flows_arr[edge_idx]) - if flow_val <= 0.0: - continue - ext_id = int(ext_edge_ids_view[edge_idx]) - h_edge_ref = edge_map.to_ref.get(ext_id) - if h_edge_ref: - src_idx, dst_idx, h_key = h_edge_ref - if isinstance(src_idx, int) and isinstance(dst_idx, int): - # FlowState has already summed demands in each direction. A - # full-duplex corridor must cover the larger directional load. - g_edge_ref = g_edge_refs.get((src_idx, dst_idx, h_key)) - if g_edge_ref: - edge_loads[g_edge_ref] = max( - edge_loads.get(g_edge_ref, 0.0), flow_val - ) - - Q = float(getattr(sizing_cfg, "quantum_gbps", 3200.0)) - h_factor = float(getattr(sizing_cfg, "headroom", 1.3)) - respect_min = bool(getattr(sizing_cfg, "respect_min_base_capacity", True)) - - # Snapshot pre-adjustment aggregated capacities per directed metro corridor - prev_corridor_caps: dict[tuple[str, str], float] = {} - for _u, _v, _k, data in G.edges(keys=True, data=True): - if str(data.get("link_type")) != "inter_metro_corridor": - continue - src_name = data.get("source_metro") - tgt_name = data.get("target_metro") - if not isinstance(src_name, str) or not isinstance(tgt_name, str): - continue - prev_cap = float(data.get("base_capacity", data.get("capacity", 0.0))) - key = (src_name, tgt_name) - prev_corridor_caps[key] = prev_corridor_caps.get(key, 0.0) + prev_cap - - for (u_g, v_g, k_g), L in edge_loads.items(): - try: - data = G.edges[(u_g, v_g, k_g)] - except Exception: - try: - data = G.get_edge_data(u_g, v_g, k_g) or {} - except Exception: - continue - if str(data.get("link_type")) != "inter_metro_corridor": - continue - # c_e = Q * ceil(h * L / Q) - import math as _m - - sized = Q * int(_m.ceil((h_factor * L) / Q)) if Q > 0 else h_factor * L - prev = float(data.get("base_capacity", 0.0)) - new_base = max(prev, sized) if respect_min else sized - G.edges[u_g, v_g, k_g]["base_capacity"] = new_base - G.edges[u_g, v_g, k_g]["target_capacity"] = new_base - - # Aggregate post-adjustment capacities per directed metro corridor - post_corridor_caps: dict[tuple[str, str], float] = {} - for _u2, _v2, _k2, data in G.edges(keys=True, data=True): - if str(data.get("link_type")) != "inter_metro_corridor": - continue - src_name = data.get("source_metro") - tgt_name = data.get("target_metro") - if not isinstance(src_name, str) or not isinstance(tgt_name, str): - continue - cap = float(data.get("base_capacity", data.get("capacity", 0.0))) - key = (src_name, tgt_name) - post_corridor_caps[key] = post_corridor_caps.get(key, 0.0) + cap - - if post_corridor_caps: - try: - total_before = sum( - prev_corridor_caps.get(k, 0.0) for k in post_corridor_caps - ) - total_after = sum(post_corridor_caps.values()) - logger.info( - "TM sizing: corridor capacity totals (Gbps) before=%s after=%s delta=%s", - f"{total_before:,.1f}", - f"{total_after:,.1f}", - f"{(total_after - total_before):,.1f}", - ) - for pair in sorted(post_corridor_caps): - before = float(prev_corridor_caps.get(pair, 0.0)) - after = float(post_corridor_caps[pair]) - if abs(after - before) <= 0.0: - continue - factor = (after / before) if before > 0.0 else float("inf") - src, dst = pair - logger.info( - "TM sizing: %s -> %s corridor capacity %s -> %s (delta=%s, x%s)", - src, - dst, - f"{before:,.1f}", - f"{after:,.1f}", - f"{(after - before):,.1f}", - "inf" if not (factor < float("inf")) else f"{factor:.2f}", - ) - except Exception: - pass - - pop_egress: dict[str, float] = {} - for u_g, v_g, data in G.edges(data=True): - if str(data.get("link_type")) != "inter_metro_corridor": - continue - cap = float(data.get("base_capacity", data.get("capacity", 0.0))) - pop_egress[u_g] = pop_egress.get(u_g, 0.0) + cap - pop_egress[v_g] = pop_egress.get(v_g, 0.0) + cap - - alpha = float(getattr(sizing_cfg, "alpha_dc_to_pop", 1.2)) - beta = float(getattr(sizing_cfg, "beta_intra_pop", 0.8)) - - for u_g, v_g, k_g, data in G.edges(keys=True, data=True): - if str(data.get("link_type")) != "dc_to_pop": - continue - # Endpoint PoP is either u or v depending on orientation - try: - u_kind = str(G.nodes[u_g].get("site_kind", "")) - v_kind = str(G.nodes[v_g].get("site_kind", "")) - except Exception as exc: - raise ValueError( - f"TM sizing: missing site_kind on DC->PoP edge endpoints: {u_g}, {v_g}" - ) from exc - if v_kind == "pop": - pop_node = v_g - elif u_kind == "pop": - pop_node = u_g - else: - raise ValueError( - f"TM sizing: DC->PoP edge does not connect to a PoP endpoint: {u_g}<->{v_g}" - ) - egress = float(pop_egress.get(pop_node, 0.0)) - target = alpha * egress - import math as _m - - sized = Q * int(_m.ceil(target / Q)) if Q > 0 else target - prev = float(data.get("base_capacity", 0.0)) - new_base = max(prev, sized) if respect_min else sized - G.edges[u_g, v_g, k_g]["base_capacity"] = new_base - G.edges[u_g, v_g, k_g]["target_capacity"] = new_base - - for u_g, v_g, k_g, data in G.edges(keys=True, data=True): - if str(data.get("link_type")) != "intra_metro": - continue - eg_u = float(pop_egress.get(u_g, 0.0)) - eg_v = float(pop_egress.get(v_g, 0.0)) - target = beta * min(eg_u, eg_v) - import math as _m - - sized = Q * int(_m.ceil(target / Q)) if Q > 0 else target - prev = float(data.get("base_capacity", 0.0)) - new_base = max(prev, sized) if respect_min else sized - G.edges[u_g, v_g, k_g]["base_capacity"] = new_base - G.edges[u_g, v_g, k_g]["target_capacity"] = new_base - - logger.info( - "TM sizing: applied capacities (Q=%s Gb/s, h=%s, alpha=%s, beta=%s)", - f"{Q:.0f}", - f"{h_factor:.3f}", - f"{alpha:.3f}", - f"{beta:.3f}", - ) - - -def to_network_sections( - G: nx.MultiGraph, - metros: list[dict[str, Any]], - metro_settings: dict[str, dict[str, Any]], - config: TopologyConfig, -) -> tuple[dict[str, Any], list[dict[str, Any]]]: - """Serialize MultiGraph to NetGraph 'nodes' and 'links' sections.""" - logger.info("Serializing MultiGraph to scenario network sections") - groups: dict[str, Any] = {} - metro_idx_map = {m["name"]: idx for idx, m in enumerate(metros, 1)} - for metro in metros: - name = metro["name"] - idx = metro_idx_map[name] - s = int(metro_settings[name]["pop_per_metro"]) # type: ignore[index] - d = int(metro_settings[name]["dc_regions_per_metro"]) # type: ignore[index] - pop_group_path = f"metro{idx}/pop[1-{s}]" - groups[pop_group_path] = { - "blueprint": metro_settings[name]["site_blueprint"], - "attrs": { - "metro_name": name, - "metro_name_orig": metro.get("name_orig", name), - "metro_id": metro.get("metro_id", ""), - "location_x": metro.get("x", 0.0), - "location_y": metro.get("y", 0.0), - "radius_km": metro.get("radius_km", 0.0), - "node_type": "pop", - }, - } - try: - logger.debug( - "group: path=%s use_blueprint=%s node_type=pop metro=%s count=%d", - pop_group_path, - metro_settings[name]["site_blueprint"], - name, - s, - ) - except Exception: - pass - if d > 0: - dc_group_path = f"metro{idx}/dc[1-{d}]" - groups[dc_group_path] = { - "blueprint": metro_settings[name]["dc_region_blueprint"], - "attrs": { - "metro_name": name, - "metro_name_orig": metro.get("name_orig", name), - "metro_id": metro.get("metro_id", ""), - "location_x": metro.get("x", 0.0), - "location_y": metro.get("y", 0.0), - "radius_km": metro.get("radius_km", 0.0), - "node_type": "dc_region", - # Required by validation and traffic - "mw_per_dc_region": float( - getattr( - getattr(config, "traffic", None), "mw_per_dc_region", 0.0 - ) - ), - "gbps_per_mw": float( - getattr(getattr(config, "traffic", None), "gbps_per_mw", 0.0) - ), - }, - } - try: - logger.debug( - "group: path=%s use_blueprint=%s node_type=dc_region metro=%s count=%d", - dc_group_path, - metro_settings[name]["dc_region_blueprint"], - name, - d, - ) - except Exception: - pass - - adjacency: list[dict[str, Any]] = [] - for u, v, k, data in G.edges(keys=True, data=True): - cap = data.get("capacity", data.get("base_capacity", 1)) - cost = int(data.get("cost", 1)) - attrs = { - "link_type": data.get("link_type", "unknown"), - "source_metro": data.get("source_metro"), - "target_metro": data.get("target_metro"), - } - # Emit total expected capacity for the adjacency (pre per-link split) - try: - tcap = float(data.get("target_capacity", data.get("base_capacity", 0.0))) - except Exception: - tcap = float(data.get("base_capacity", 0.0)) - attrs["target_capacity"] = tcap - # Tag this adjacency with a stable site-edge key and optional adjacency id - attrs["site_edge"] = f"{u}|{v}|{k}" - if "adjacency_id" in data: - attrs["adjacency_id"] = data.get("adjacency_id") - if "distance_km" in data: - attrs["distance_km"] = int( - data["distance_km"] - ) # ensure int for YAML stability - if "euclidean_km" in data and data["euclidean_km"] is not None: - attrs["euclidean_km"] = float( - data["euclidean_km"] - ) # corridor straight-line distance - if "detour_ratio" in data and data["detour_ratio"] is not None: - attrs["detour_ratio"] = float( - data["detour_ratio"] - ) # corridor length / euclidean - hw = data.get("hardware") - if isinstance(hw, dict) and hw: - attrs["hardware"] = hw - - link_entry: dict[str, Any] = { - # Preserve role-based match filters on endpoints - # so DSL expansion restricts nodes within each site. - "source": ( - {"path": u, "match": data.get("match")} - if isinstance(data.get("match"), dict) and data.get("match") - else u - ), - "target": ( - {"path": v, "match": data.get("match")} - if isinstance(data.get("match"), dict) and data.get("match") - else v - ), - "pattern": "one_to_one", - "capacity": cap, - "cost": cost, - "attrs": attrs, - } - if "risk_groups" in data and data["risk_groups"]: - link_entry["risk_groups"] = data["risk_groups"] - - adjacency.append(link_entry) - - # Deduplicate and store node_overrides on graph for assembly to embed - try: - node_overrides = G.graph.get("node_overrides", []) - if isinstance(node_overrides, list) and node_overrides: - seen: set[tuple[str, str, str]] = set() - dedup: list[dict[str, Any]] = [] - for ov in node_overrides: - path = str(ov.get("path", "")) - attrs = ( - ov.get("attrs", {}) if isinstance(ov.get("attrs", {}), dict) else {} - ) - for k, v in attrs.items(): - key = (path, str(k), str(v)) - if key in seen: - continue - seen.add(key) - dedup.append({"path": path, "attrs": {k: v}}) - G.graph["__emitted_node_overrides__"] = dedup - except Exception: - pass - - logger.info( - "Serialized network: %d groups, %d adjacency entries", - len(groups), - len(adjacency), - ) - return groups, adjacency - - -def save_site_graph_json(G: nx.MultiGraph, path: Path, *, json_indent: int = 2) -> None: - """Save site nodes and edges as JSON. - - String node IDs and edge keys preserve parallel links. - """ - logger.info(f"Saving site-level network graph to JSON: {path}") - - def _to_jsonable(obj: Any) -> Any: - if isinstance(obj, (set, tuple)): - try: - return list(sorted(obj)) if isinstance(obj, set) else list(obj) - except Exception: - return list(obj) - try: - import numpy as np # type: ignore - - if isinstance(obj, (np.floating, np.integer)): - return obj.item() - except Exception: - pass - return obj - - out: dict[str, Any] = {"graph_type": "site_network", "nodes": [], "edges": []} - - for node_id, data in G.nodes(data=True): - out["nodes"].append( - {"id": str(node_id), **{k: _to_jsonable(v) for k, v in data.items()}} - ) - - for u, v, k, data in G.edges(keys=True, data=True): - out["edges"].append( - { - "source": str(u), - "target": str(v), - "key": str(k), - **{k2: _to_jsonable(v2) for k2, v2 in data.items()}, - } - ) - - path.parent.mkdir(parents=True, exist_ok=True) - with path.open("w") as f: - json.dump(out, f, indent=int(json_indent)) - size_mb = path.stat().st_size / (1024 * 1024) - logger.info(f"Saved site network graph JSON ({size_mb:.2f} MB) → {path}") diff --git a/topogen/scenario/libraries.py b/topogen/scenario/libraries.py index 9002ddd..203109d 100644 --- a/topogen/scenario/libraries.py +++ b/topogen/scenario/libraries.py @@ -6,44 +6,41 @@ from topogen.blueprints_lib import get_builtin_blueprints from topogen.components_lib import get_builtin_components -from topogen.log_config import get_logger if TYPE_CHECKING: # pragma: no cover - import-time types only from topogen.config import TopologyConfig -logger = get_logger(__name__) - def _build_components_section( - config: "TopologyConfig", used_blueprints: set[str] + config: "TopologyConfig", blueprints: dict[str, Any] ) -> dict[str, Any]: """Build the components section of the NetGraph scenario. Uses merged component library (built-ins + lib/components.yml) and includes - only component definitions that are actually referenced by the configuration - (platforms from ``hw_component`` and optics from ``optics``). + components referenced by configuration and finalized blueprint assignments. + Scan after role overrides so replaced hardware does not remain a dependency. """ components = get_builtin_components() - referenced_components: set[str] = set() - role_to_platform = getattr(config.components, "hw_component", {}) or {} - optics_map = getattr(config.components, "optics", {}) or {} - if isinstance(role_to_platform, dict): - for v in role_to_platform.values(): - if isinstance(v, str) and v: - referenced_components.add(v) - if isinstance(optics_map, dict): - for v in optics_map.values(): - if isinstance(v, str) and v: - referenced_components.add(v) - result: dict[str, Any] = {} - for comp_name in sorted(referenced_components): - if comp_name in components: - result[comp_name] = components[comp_name] - else: - logger.warning( - f"Referenced component '{comp_name}' not found in component library" - ) + referenced_components = set(config.components.hw_component.values()) | set( + config.components.optics.values() + ) + + def references(value): + if isinstance(value, dict): + if "component" in value: + referenced_components.add(value["component"]) + for child in value.values(): + references(child) + elif isinstance(value, list): + for child in value: + references(child) + references(blueprints) + referenced_components.discard("") # Explicitly unassigned roles have no component. + missing = referenced_components - components.keys() + if missing: + raise ValueError(f"Unknown components: {sorted(missing)}") + result = {name: components[name] for name in sorted(referenced_components)} return result @@ -54,50 +51,35 @@ def _build_blueprints_section( from copy import deepcopy builtin_blueprints = get_builtin_blueprints() - role_to_platform = getattr(config.components, "hw_component", {}) or {} + role_to_platform = config.components.hw_component if not isinstance(role_to_platform, dict): - role_to_platform = {} + raise ValueError("components.hw_component must be a mapping") result: dict[str, Any] = {} - total_groups = 0 - groups_with_role = 0 - groups_with_hw = 0 - for blueprint_name in sorted(used_blueprints): - if blueprint_name not in builtin_blueprints: - raise ValueError(f"Unknown blueprint: {blueprint_name}") - blueprint = deepcopy(builtin_blueprints[blueprint_name]) - if "nodes" in blueprint: - for group_name, group_def in blueprint["nodes"].items(): - total_groups += 1 - if "attrs" not in group_def: - group_def["attrs"] = {} - role = group_def["attrs"].get("role") - if not isinstance(role, str) or not role: - raise ValueError( - f"Blueprint '{blueprint_name}' group '{group_name}' is missing required 'role' attribute" - ) - groups_with_role += 1 - hw_name = role_to_platform.get(role, "") - if hw_name: - group_def["attrs"]["hardware"] = { - "component": hw_name, - "count": 1, - } - groups_with_hw += 1 - logger.info( - "HW: node blueprint=%s group=%s role=%s platform=%s", - blueprint_name, - group_name, - role, - hw_name, - ) - result[blueprint_name] = blueprint - try: - logger.info( - "Node hardware assigned for %d of %d blueprint groups (with_role=%d)", - groups_with_hw, - total_groups, - groups_with_role, - ) - except Exception: - pass + visiting: set[str] = set() + + def include(name: str) -> None: + if name in visiting: + raise ValueError(f"Cyclic blueprint reference: {name}") + if name in result: + return + if name not in builtin_blueprints: + raise ValueError(f"Unknown blueprint: {name}") + visiting.add(name) + blueprint = deepcopy(builtin_blueprints[name]) + for group_name, group in blueprint["nodes"].items(): + if "blueprint" in group: + include(group["blueprint"]) + attrs = group.setdefault("attrs", {}) + role = attrs.get("role") + if (not isinstance(role, str) or not role) and "blueprint" not in group: + raise ValueError( + f"Blueprint '{name}' group '{group_name}' is missing required 'role' attribute" + ) + if hardware := role_to_platform.get(role): + attrs["hardware"] = {"component": hardware, "count": 1} + visiting.remove(name) + result[name] = blueprint + + for name in sorted(used_blueprints): + include(name) return result diff --git a/topogen/scenario/network.py b/topogen/scenario/network.py index 5d572e5..8da8fc6 100644 --- a/topogen/scenario/network.py +++ b/topogen/scenario/network.py @@ -1,45 +1,191 @@ -"""Extract metro attributes for scenario assembly.""" +"""Extract metro descriptors and serialize site graphs to NetGraph DSL and JSON.""" from __future__ import annotations -from typing import TYPE_CHECKING, Any +import json +import math +from pathlib import Path +from typing import Any import networkx as nx +import numpy as np +from topogen.config import TopologyConfig from topogen.log_config import get_logger - -if TYPE_CHECKING: # pragma: no cover - import-time types only - pass +from topogen.naming import site_edge_id logger = get_logger(__name__) -def _extract_metros_from_graph(graph: nx.Graph) -> list[dict[str, Any]]: - """Extract metro nodes and their identity, coordinates, and radius. - - Raise ValueError if a metro lacks ``name``, ``metro_id``, or ``radius_km``. - """ - metros: list[dict[str, Any]] = [] +def _extract_metros_from_graph(graph: nx.MultiGraph) -> list[dict[str, Any]]: + if not graph.is_multigraph() or graph.is_directed(): + raise TypeError("Expected an undirected corridor MultiGraph") + required = ("name", "name_orig", "metro_id", "x", "y", "radius_km") + metros = [] + names: set[str] = set() + ids: set[str] = set() for node, data in graph.nodes(data=True): - if data.get("node_type") in ["metro", "metro+highway"]: - required_attrs = ["name", "metro_id", "radius_km"] - for attr in required_attrs: - if attr not in data: - raise ValueError( - f"Metro node {node} missing required attribute '{attr}'" - ) - metros.append( - { - "node_key": node, - "name": data["name"], - "name_orig": data.get("name_orig", data["name"]), - "metro_id": data["metro_id"], - "x": data.get("x", 0.0), - "y": data.get("y", 0.0), - "radius_km": data["radius_km"], - } + if data.get("node_type") != "metro": + raise ValueError(f"Expected metro node at {node}") + missing = set(required) - data.keys() + if missing: + raise ValueError( + f"Metro node {node} missing required attributes: {sorted(missing)}" ) + for key, seen in (("name", names), ("metro_id", ids)): + value = data[key] + if not isinstance(value, str) or not value.strip(): + raise ValueError(f"Metro {node} requires nonempty string '{key}'") + if value in seen: + raise ValueError(f"Duplicate metro {key}: {value}") + seen.add(value) + for key in ("x", "y", "radius_km"): + value = data[key] + if ( + isinstance(value, bool) + or not isinstance(value, (int, float)) + or not math.isfinite(value) + ): + raise ValueError(f"Metro {node} requires finite numeric '{key}'") + if data["radius_km"] < 0: + raise ValueError(f"Metro {node} has negative radius_km") + metros.append({"node_key": node, **{key: data[key] for key in required}}) return metros -__all__ = ["_extract_metros_from_graph"] +def to_network_sections( + G: nx.MultiGraph, + metros: list[dict[str, Any]], + metro_settings: dict[str, dict[str, Any]], + config: TopologyConfig, +) -> tuple[dict[str, Any], list[dict[str, Any]]]: + """Serialize MultiGraph to NetGraph 'nodes' and 'links' sections.""" + logger.info("Serializing MultiGraph to scenario network sections") + groups: dict[str, Any] = {} + for idx, metro in enumerate(metros, 1): + name = metro["name"] + settings = metro_settings[name] + attrs = { + "metro_name": name, + "metro_name_orig": metro["name_orig"], + "metro_id": metro["metro_id"], + "location_x": metro["x"], + "location_y": metro["y"], + "radius_km": metro["radius_km"], + } + for kind, count, blueprint in ( + ("pop", settings["pop_per_metro"], settings["site_blueprint"]), + ("dc", settings["dc_regions_per_metro"], settings["dc_region_blueprint"]), + ): + if count == 0: + continue + site_attrs = {**attrs, "node_type": "pop" if kind == "pop" else "dc_region"} + if kind == "dc": + site_attrs.update( + mw_per_dc_region=float(config.traffic.mw_per_dc_region), + gbps_per_mw=float(config.traffic.gbps_per_mw), + ) + groups[f"metro{idx}/{kind}[1-{count}]"] = { + "blueprint": blueprint, + "attrs": site_attrs, + } + + adjacency: list[dict[str, Any]] = [] + for u, v, k, data in G.edges(keys=True, data=True): + cap = data["base_capacity"] + cost = int(data.get("cost", 1)) + attrs = { + **data.get("attrs", {}), + "link_type": data.get("link_type", "unknown"), + "source_metro": data.get("source_metro"), + "target_metro": data.get("target_metro"), + } + # Emit total expected capacity for the adjacency (pre per-link split) + tcap = float(data.get("target_capacity", data.get("base_capacity", 0.0))) + attrs["target_capacity"] = tcap + # Tag this adjacency with a stable site-edge key and optional adjacency id + attrs["site_edge"] = site_edge_id(u, v, k) + if "adjacency_id" in data: + attrs["adjacency_id"] = data.get("adjacency_id") + if "distance_km" in data: + attrs["distance_km"] = int( + data["distance_km"] + ) # ensure int for YAML stability + if "euclidean_km" in data and data["euclidean_km"] is not None: + attrs["euclidean_km"] = float( + data["euclidean_km"] + ) # corridor straight-line distance + if "detour_ratio" in data and data["detour_ratio"] is not None: + attrs["detour_ratio"] = float( + data["detour_ratio"] + ) # corridor length / euclidean + hw = data.get("hardware") + if isinstance(hw, dict) and hw: + attrs["hardware"] = hw + + link_entry: dict[str, Any] = { + # Preserve role-based match filters on endpoints + # so DSL expansion restricts nodes within each site. + "source": ( + {"path": u, "match": data.get("match")} + if isinstance(data.get("match"), dict) and data.get("match") + else u + ), + "target": ( + {"path": v, "match": data.get("match")} + if isinstance(data.get("match"), dict) and data.get("match") + else v + ), + "pattern": "one_to_one", + "capacity": cap, + "cost": cost, + "attrs": attrs, + } + if config.corridors.risk_groups.enabled and data.get("risk_groups"): + link_entry["risk_groups"] = data["risk_groups"] + + adjacency.append(link_entry) + + logger.info( + "Serialized network: %d groups, %d adjacency entries", + len(groups), + len(adjacency), + ) + return groups, adjacency + + +def save_site_graph_json(G: nx.MultiGraph, path: Path, *, json_indent: int = 2) -> None: + """Save site nodes and edges as JSON. + + String node IDs and edge keys preserve parallel links. + """ + logger.info(f"Saving site-level network graph to JSON: {path}") + + def _to_jsonable(obj: Any) -> Any: + if isinstance(obj, set): + return sorted(obj) + + if isinstance(obj, (np.floating, np.integer)): + return obj.item() + raise TypeError(f"Cannot serialize {type(obj).__name__}") + + out: dict[str, Any] = {"graph_type": "site_network", "nodes": [], "edges": []} + + for node_id, data in G.nodes(data=True): + out["nodes"].append({"id": str(node_id), **data}) + + for u, v, k, data in G.edges(keys=True, data=True): + out["edges"].append( + { + "source": str(u), + "target": str(v), + "key": str(k), + **data, + } + ) + + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("w") as f: + json.dump(out, f, indent=json_indent, default=_to_jsonable, allow_nan=False) + size_mb = path.stat().st_size / (1024 * 1024) + logger.info(f"Saved site network graph JSON ({size_mb:.2f} MB) → {path}") diff --git a/topogen/scenario/policies.py b/topogen/scenario/policies.py index 527c6aa..9fae75a 100644 --- a/topogen/scenario/policies.py +++ b/topogen/scenario/policies.py @@ -11,7 +11,9 @@ from topogen.config import TopologyConfig -def _build_failure_policy_set_section(config: "TopologyConfig") -> dict[str, Any]: +def _build_failure_policy_set_section( + config: "TopologyConfig", workflow: list[dict[str, Any]] +) -> dict[str, Any]: """Build the ``failures`` section of the scenario.""" builtin_policies = get_builtin_failure_policies() policies: dict[str, Any] = {} @@ -25,35 +27,22 @@ def _build_failure_policy_set_section(config: "TopologyConfig") -> dict[str, Any f"Default failure policy '{default_policy_name}' not found. Available built-in policies: {available}" ) - workflows_cfg = getattr(config, "workflows", None) - if ( - workflows_cfg is not None - and getattr(workflows_cfg, "assignments", None) is not None - ): - workflows = get_builtin_workflows() - workflow_name = workflows_cfg.assignments.default - steps = workflows.get(workflow_name, []) - for step in steps: - policy_name = step.get("failure_policy") - if not policy_name: - continue - if policy_name not in builtin_policies: - available2 = list(builtin_policies.keys()) - raise ValueError( - f"Workflow '{workflow_name}' references unknown failure policy '{policy_name}'. Available built-in policies: {available2}" - ) - policies[policy_name] = builtin_policies[policy_name] + workflow_name = config.workflows.assignments.default + for step in workflow: + policy_name = step.get("failure_policy") + if policy_name is None: + continue + if policy_name not in builtin_policies: + raise ValueError( + f"Workflow '{workflow_name}' references unknown failure policy '{policy_name}'" + ) + policies[policy_name] = builtin_policies[policy_name] return policies def _build_workflow_section(config: "TopologyConfig") -> list[dict[str, Any]]: """Build the ``workflow`` section of the scenario.""" - matrix_name = getattr(getattr(config, "traffic", None), "matrix_name", None) - builtin_workflows = get_builtin_workflows( - matrix_name=matrix_name - if isinstance(matrix_name, str) - else "baseline_traffic_matrix" - ) + builtin_workflows = get_builtin_workflows(matrix_name=config.traffic.matrix_name) default_workflow_name = config.workflows.assignments.default if default_workflow_name in builtin_workflows: return builtin_workflows[default_workflow_name] diff --git a/topogen/scenario/risk.py b/topogen/scenario/risk.py index dd4fdf2..d0d275e 100644 --- a/topogen/scenario/risk.py +++ b/topogen/scenario/risk.py @@ -1,4 +1,4 @@ -"""Risk groups section for scenario building.""" +"""Risk definitions use the length of their owning corridor path.""" from __future__ import annotations @@ -7,87 +7,40 @@ import networkx as nx -from topogen.log_config import get_logger +from topogen.corridors import corridor_risk_name -if TYPE_CHECKING: # pragma: no cover - import-time types only +if TYPE_CHECKING: from topogen.config import TopologyConfig -logger = get_logger(__name__) - def _build_risk_groups_section( - graph: nx.Graph, config: "TopologyConfig" + graph: nx.MultiGraph, config: "TopologyConfig" ) -> list[dict[str, Any]]: - """Build the ``risk_groups`` section from corridor-tagged edges. - - This reads risk-group tags attached to corridor edges and creates scenario - risk group entries with a canonical per-pair distance in kilometers. - """ if not config.corridors.risk_groups.enabled: return [] - - prefix = getattr(config.corridors.risk_groups, "group_prefix", "corridor_risk") - pair_distance_km: dict[str, int] = {} - for source, target, data in graph.edges(data=True): - src = graph.nodes[source] - tgt = graph.nodes[target] - if src.get("node_type") in ["metro", "metro+highway"] and tgt.get( - "node_type" - ) in [ - "metro", - "metro+highway", - ]: - src_name = str(src.get("name", "")).strip() - tgt_name = str(tgt.get("name", "")).strip() - a_name, b_name = ( - (src_name, tgt_name) if src_name < tgt_name else (tgt_name, src_name) - ) - pair_rg = f"{prefix}_{a_name}_{b_name}" - dist_km = int(math.ceil(float(data.get("length_km", 0.0)))) - prev = pair_distance_km.get(pair_rg) - pair_distance_km[pair_rg] = dist_km if prev is None else max(prev, dist_km) - - risk_group_distance_km: dict[str, int] = {} - for _source, _target, data in graph.edges(data=True): - src = graph.nodes[_source] - tgt = graph.nodes[_target] - if src.get("node_type") in ["metro", "metro+highway"] and tgt.get( - "node_type" - ) in [ - "metro", - "metro+highway", - ]: - edge_dist_km = int(math.ceil(float(data.get("length_km", 0.0)))) - edge_risk_groups = data.get("risk_groups", []) or [] - for rg in edge_risk_groups: - name = str(rg) - base = name - idx = base.rfind("_path") - if idx != -1 and idx + 5 <= len(base): - tail = base[idx + 5 :] - if tail.isdigit(): - base = base[:idx] - canonical = pair_distance_km.get(base) - chosen = canonical if canonical is not None else edge_dist_km - prev = risk_group_distance_km.get(name) - risk_group_distance_km[name] = ( - chosen if prev is None else max(prev, chosen) - ) - - if not risk_group_distance_km: - logger.info("No risk groups found in corridor edges") - return [] - - risk_groups: list[dict[str, Any]] = [] - for rg_name in sorted(risk_group_distance_km.keys()): - risk_groups.append( - { - "name": rg_name, - "attrs": { - "type": "corridor_risk", - "distance_km": int(risk_group_distance_km.get(rg_name, 0)), - }, - } + distances: dict[str, int] = {} + referenced: set[str] = set() + for source, target, key, data in graph.edges(keys=True, data=True): + name = corridor_risk_name( + config.corridors.risk_groups.group_prefix, + graph.nodes[source]["name"], + graph.nodes[target]["name"], + key, ) - logger.info(f"Generated {len(risk_groups)} risk group definitions") - return risk_groups + if name in distances: + raise ValueError(f"Duplicate corridor risk identity: {name}") + length = float(data["length_km"]) + if not math.isfinite(length) or length <= 0: + raise ValueError(f"Corridor '{name}' must have positive finite length") + distances[name] = math.ceil(length) + referenced.update(data.get("risk_groups", [])) + unknown = referenced - distances.keys() + if unknown: + raise ValueError(f"Risk groups have no owning corridor path: {sorted(unknown)}") + return [ + { + "name": name, + "attrs": {"type": "corridor_risk", "distance_km": distances[name]}, + } + for name in sorted(referenced) + ] diff --git a/topogen/scenario/sizing.py b/topogen/scenario/sizing.py new file mode 100644 index 0000000..a193b29 --- /dev/null +++ b/topogen/scenario/sizing.py @@ -0,0 +1,321 @@ +"""Size site links from traffic routed across the metro corridor graph.""" + +from __future__ import annotations + +import math +import re +from collections import defaultdict +from collections.abc import Hashable, Iterator +from typing import Any + +import netgraph_core +import networkx as nx +from ngraph.lib.nx import from_networkx as _from_networkx + +from topogen.config import TopologyConfig +from topogen.log_config import get_logger + +logger = get_logger(__name__) + + +def _parse_tm_endpoint_to_metro_idx(endpoint: str) -> int | None: + """Extract metro index from a traffic matrix endpoint regex/path. + + Accepts strings like '^metro3/dc2/.*' or 'metro3/dc2'. Returns 3. + """ + match = re.fullmatch(r"\^?metro([1-9][0-9]*)/dc[1-9][0-9]*(?:/\.\*)?", endpoint) + return int(match[1]) if match else None + + +def _resolve_flow_placement(flow_placement: str) -> netgraph_core.FlowPlacement: + """Resolve the configured split directly to the Core routing enum.""" + try: + return { + "EQUAL_BALANCED": netgraph_core.FlowPlacement.EQUAL_BALANCED, + "PROPORTIONAL": netgraph_core.FlowPlacement.PROPORTIONAL, + }[flow_placement] + except KeyError as exc: + raise ValueError(f"Unknown flow_placement: {flow_placement}") from exc + + +def _tm_metro_demands( + demands: list[dict[str, Any]], dc_counts: dict[int, int] +) -> Iterator[tuple[int, int, float]]: + """Collapse the generator's concrete and uniform demands to metro pairs.""" + uniform_selector = "(metro[0-9]+/dc[0-9]+)" + for demand in demands: + volume = float(demand.get("demand", 0.0)) + if not math.isfinite(volume) or volume < 0: + raise ValueError("TM sizing: demand volume must be finite non-negative") + if volume == 0: + continue + source = str(demand.get("source_path", "")) + target = str(demand.get("sink_path", "")) + if ( + source == target == uniform_selector + and demand.get("mode") == "pairwise" + and demand.get("group_mode") == "group_pairwise" + ): + total_dc = sum(dc_counts.values()) + if total_dc < 2: + raise ValueError( + "TM sizing: pairwise traffic requires at least two DCs" + ) + # NetGraph splits volume over all ordered, distinct DC site pairs. + # Same-metro pairs remain in the denominator but use no corridors. + per_pair = volume / (total_dc * (total_dc - 1)) + for src, src_count in dc_counts.items(): + for dst, dst_count in dc_counts.items(): + if src != dst and src_count > 0 and dst_count > 0: + yield src, dst, per_pair * src_count * dst_count + continue + src = _parse_tm_endpoint_to_metro_idx(source) + dst = _parse_tm_endpoint_to_metro_idx(target) + if src is None or dst is None: + raise ValueError( + f"TM sizing: unsupported demand endpoint ({source!r} -> {target!r})" + ) + if src != dst: + yield src, dst, volume + + +def tm_based_size_capacities( + G: nx.MultiGraph, + metros: list[dict[str, Any]], + metro_settings: dict[str, dict[str, Any]], + config: TopologyConfig, + traffic_matrices: dict[str, list[dict[str, Any]]], +) -> None: + """Size links from generated traffic routed on a collapsed metro graph. + + Preserve parallel corridors and route each demand with the configured flow + split. Size each corridor for its peak directional load, then apply headroom + and round up to capacity increments. Derive DC-to-PoP and intra-metro + capacities from PoP egress, respecting configured minimums when enabled. + """ + sizing_cfg = config.build.tm_sizing + if sizing_cfg.enabled is not True: + return + + traffic_cfg = config.traffic + if not traffic_cfg.enabled: + raise ValueError( + "TM sizing is enabled but traffic generation is disabled in configuration" + ) + + matrix_name = sizing_cfg.matrix_name or config.traffic.matrix_name + demands = traffic_matrices.get(matrix_name, []) + dc_counts = { + idx: metro_settings[metro["name"]]["dc_regions_per_metro"] + for idx, metro in enumerate(metros, 1) + } + if sum(dc_counts.values()) <= 0: + raise ValueError( + "TM sizing is enabled but no DC regions are configured (dc_regions_per_metro == 0)" + ) + if not demands: + raise ValueError( + f"TM sizing: traffic matrix '{matrix_name}' is empty despite enabled traffic" + ) + + metro_idx_map = {m["name"]: idx for idx, m in enumerate(metros, 1)} + # Build temporary NetworkX MultiDiGraph with metro nodes and inter-metro corridors. + # MultiDiGraph is required to preserve parallel edges between the same metro pair + # (e.g., striped corridors with multiple links per corridor). + H: nx.MultiDiGraph = nx.MultiDiGraph() + H.add_nodes_from(metro_idx_map.values()) + + # Map to track correspondence between H edges and G edges. + # Key is (src_metro_idx, dst_metro_idx, h_edge_key) to handle parallel edges. + # Both directions reference the same physical edge, sized once for peak load. + g_edge_refs: dict[tuple[Hashable, Hashable, Any], tuple[Any, Any, Any]] = {} + + corridor_edges = [ + (u, v, key, data) + for u, v, key, data in G.edges(keys=True, data=True) + if data["link_type"] == "inter_metro_corridor" + ] + for u_g, v_g, k_g, data in corridor_edges: + src_name = data.get("source_metro") + tgt_name = data.get("target_metro") + if not isinstance(src_name, str) or not isinstance(tgt_name, str): + raise ValueError( + "TM sizing: inter-metro edge missing source_metro/target_metro attributes" + ) + + s_idx = metro_idx_map.get(src_name) + t_idx = metro_idx_map.get(tgt_name) + if s_idx is None or t_idx is None: + raise ValueError( + f"TM sizing: unknown metro name(s) on inter-metro edge: {src_name!r}, {tgt_name!r}" + ) + + cost = int(data.get("cost", 1)) + # Let H allocate keys: G keys can repeat across different PoP pairs + # that collapse to the same metro pair. + h_key_fwd = H.add_edge(s_idx, t_idx, capacity=1e15, cost=cost) + g_edge_refs[(s_idx, t_idx, h_key_fwd)] = (u_g, v_g, k_g) + + h_key_rev = H.add_edge(t_idx, s_idx, capacity=1e15, cost=cost) + g_edge_refs[(t_idx, s_idx, h_key_rev)] = (u_g, v_g, k_g) + + if H.number_of_edges() == 0: + raise ValueError( + "TM sizing: no inter-metro corridor edges present in site graph" + ) + + # Convert NetworkX graph to netgraph_core format. + # bidirectional=False because we already added explicit reverse edges above. + multidigraph, node_map, edge_map = _from_networkx(H, bidirectional=False) + num_nodes = multidigraph.num_nodes() + + backend = netgraph_core.Backend.cpu() + algorithms = netgraph_core.Algorithms(backend) + handle = algorithms.build_graph(multidigraph) + + flow_state = netgraph_core.FlowState(multidigraph) + + core_fp = _resolve_flow_placement(sizing_cfg.flow_placement) + + edge_selection = netgraph_core.EdgeSelection( + multi_edge=True, + require_capacity=False, # IP-style routing based on cost only + tie_break=netgraph_core.EdgeTieBreak.DETERMINISTIC, + ) + + by_pair: dict[tuple[int, int], list[float]] = defaultdict(list) + for source, target, volume in _tm_metro_demands(demands, dc_counts): + by_pair[source, target].append(volume) + for (s_metro, t_metro), volumes in by_pair.items(): + demand_val = math.fsum(volumes) + s_idx = node_map.to_index.get(s_metro) + t_idx = node_map.to_index.get(t_metro) + if s_idx is None or t_idx is None: + raise ValueError( + f"TM sizing: metro index out of range (src={s_metro}, dst={t_metro}, num_nodes={num_nodes})" + ) + + try: + dists, pred_dag = algorithms.spf( + handle, + src=s_idx, + dst=t_idx, + selection=edge_selection, + multipath=True, + ) + except Exception as exc: + raise ValueError( + f"TM sizing: SPF failed for metro {s_metro}->{t_metro}: {exc}" + ) from exc + + placed = flow_state.place_on_dag( + src=s_idx, + dst=t_idx, + dag=pred_dag, + requested_flow=demand_val, + flow_placement=core_fp, + ) + + if not math.isclose(placed, demand_val, rel_tol=1e-9, abs_tol=1e-9): + raise ValueError( + f"TM sizing: could not route demand metro{s_metro}->metro{t_metro}: {placed}/{demand_val}" + ) + cost = int(dists[t_idx]) + logger.debug( + "TM sizing: placed %s Gbps from metro%d->metro%d (cost=%s)", + f"{placed:,.1f}", + s_metro, + t_metro, + f"{cost:,}", + ) + + edge_flows_arr = flow_state.edge_flow_view() + ext_edge_ids_view = multidigraph.ext_edge_ids_view() + edge_loads: dict[tuple[Any, Any, Any], float] = {} + for edge_idx in range(len(edge_flows_arr)): + flow_val = float(edge_flows_arr[edge_idx]) + if flow_val <= 0.0: + continue + ext_id = int(ext_edge_ids_view[edge_idx]) + src_idx, dst_idx, h_key = edge_map.to_ref[ext_id] + g_edge_ref = g_edge_refs[src_idx, dst_idx, h_key] + edge_loads[g_edge_ref] = max(edge_loads.get(g_edge_ref, 0.0), flow_val) + + Q = float(sizing_cfg.quantum_gbps) + h_factor = float(sizing_cfg.headroom) + respect_min = bool(sizing_cfg.respect_min_base_capacity) + + def apply_capacity(data: dict[str, Any], target: float) -> None: + sized = Q * math.ceil(target / Q) if Q > 0 else target + capacity = max(float(data["base_capacity"]), sized) if respect_min else sized + data["base_capacity"] = data["target_capacity"] = capacity + + # Each undirected site edge is sized once for its peak directional load. + # Accumulate metro-pair totals and PoP egress in the same pass. + previous: dict[tuple[str, str], float] = defaultdict(float) + current: dict[tuple[str, str], float] = defaultdict(float) + pop_egress: dict[str, float] = defaultdict(float) + for source, target, key, data in corridor_edges: + pair = (data["source_metro"], data["target_metro"]) + previous[pair] += float(data["base_capacity"]) + if (source, target, key) in edge_loads: + apply_capacity(data, h_factor * edge_loads[source, target, key]) + capacity = float(data["base_capacity"]) + current[pair] += capacity + pop_egress[source] += capacity + pop_egress[target] += capacity + + total_before, total_after = sum(previous.values()), sum(current.values()) + logger.info( + "TM sizing: corridor capacity totals (Gbps) before=%.1f after=%.1f delta=%.1f", + total_before, + total_after, + total_after - total_before, + ) + for pair in sorted(current): + before, after = previous[pair], current[pair] + if after != before: + logger.info( + "TM sizing: %s <-> %s corridor capacity %.1f -> %.1f (delta=%.1f)", + *pair, + before, + after, + after - before, + ) + + alpha = float(sizing_cfg.alpha_dc_to_pop) + beta = float(sizing_cfg.beta_intra_pop) + + for u_g, v_g, data in G.edges(data=True): + if str(data.get("link_type")) != "dc_to_pop": + continue + # An undirected edge may expose either endpoint first. + u_kind = G.nodes[u_g]["site_kind"] + v_kind = G.nodes[v_g]["site_kind"] + if v_kind == "pop": + pop_node = v_g + elif u_kind == "pop": + pop_node = u_g + else: + raise ValueError( + f"TM sizing: DC->PoP edge does not connect to a PoP endpoint: {u_g}<->{v_g}" + ) + egress = float(pop_egress.get(pop_node, 0.0)) + target = alpha * egress + apply_capacity(data, target) + + for u_g, v_g, data in G.edges(data=True): + if str(data.get("link_type")) != "intra_metro": + continue + eg_u = float(pop_egress.get(u_g, 0.0)) + eg_v = float(pop_egress.get(v_g, 0.0)) + target = beta * min(eg_u, eg_v) + apply_capacity(data, target) + + logger.info( + "TM sizing: applied capacities (Q=%s Gb/s, h=%s, alpha=%s, beta=%s)", + f"{Q:.0f}", + f"{h_factor:.3f}", + f"{alpha:.3f}", + f"{beta:.3f}", + ) diff --git a/topogen/scenario/striping.py b/topogen/scenario/striping.py index b38d47e..e7990c1 100644 --- a/topogen/scenario/striping.py +++ b/topogen/scenario/striping.py @@ -1,144 +1,92 @@ -"""Helpers for deterministic striping of inter-site adjacencies. - -Compute stripe groupings (by fixed width or by blueprint attribute) and emit -``node_overrides`` that assign per-adjacency stripe identifiers to nodes. The -graph pipeline then matches on a single stripe attribute, so role-based -eligibility is encoded upfront in the grouping selection. -""" +"""Stripe actual blueprint devices and emit one exact node rule per device.""" from __future__ import annotations -from typing import Any, Dict, List, Tuple - - -def _natural_key(name: str) -> Tuple[str, int]: - """Return the non-digit characters and concatenated digits as a sort key. - - For example, ``leaf12`` becomes ``("leaf", 12)``. No digits gives 0. - """ - - prefix_chars: List[str] = [] - digits: List[str] = [] - for ch in name: - if ch.isdigit(): - digits.append(ch) - else: - prefix_chars.append(ch) - prefix = "".join(prefix_chars) - try: - num = int("".join(digits)) if digits else 0 - except Exception: - num = 0 - return prefix, num - - -def eligible_device_names_from_blueprint( - blueprint: Dict[str, Any], roles: set[str] | None -) -> List[str]: - """List eligible device names from a blueprint filtered by roles. - - Args: - blueprint: Blueprint definition with a ``nodes`` mapping. - roles: Allowed roles or None for all roles. - - Returns: - Sorted device names (e.g., ["leaf1", "leaf2"]). - """ - - names: List[str] = [] - groups = blueprint.get("nodes", {}) if isinstance(blueprint, dict) else {} - for _gname, gdef in groups.items(): - attrs = gdef.get("attrs", {}) if isinstance(gdef, dict) else {} - role = str(attrs.get("role", "")) - if roles and role not in roles: - continue - try: - n = int(gdef.get("count", 0)) - except Exception: - n = 0 - template = str(gdef.get("template", "")) - for i in range(1, n + 1): - names.append(template.replace("{n}", str(i))) - names.sort(key=_natural_key) - return names - - -def group_by_width(names: List[str], width: int) -> List[List[str]]: - """Partition names into contiguous groups of size ``width``. - - Raises: - ValueError: If ``width`` <= 0 or ``len(names) % width != 0``. - """ - - if width <= 0: - raise ValueError("striping.width must be positive") - total = len(names) - if total % width != 0: - raise ValueError( - f"Eligible device count {total} is not divisible by width {width}" +import re +from typing import Any + +from ngraph import Network +from ngraph.dsl.blueprints.expand import expand_network_dsl +from ngraph.dsl.selectors import normalize_selector +from ngraph.model.selectors import select_nodes + + +def _natural_key(path: str) -> tuple: + return tuple( + int(part) if part.isdigit() else part for part in re.split(r"(\d+)", path) + ) + + +class StripePlanner: + """Cache each blueprint inventory and merge stripe attributes within one build.""" + + def __init__(self, blueprints: dict[str, Any]) -> None: + self._blueprints = blueprints + self._inventories: dict[str, Network] = {} + self._rules: dict[str, dict[str, str]] = {} + + def groups( + self, blueprint: str, striping: dict[str, Any], match: dict[str, Any] + ) -> dict[str, list[str]]: + if blueprint not in self._inventories: + network = expand_network_dsl( + { + "blueprints": self._blueprints, + "network": {"nodes": {"stripe_probe": {"blueprint": blueprint}}}, + } + ) + self._inventories[blueprint] = network + groups = select_nodes( + self._inventories[blueprint], + normalize_selector({"path": "stripe_probe", "match": match}, "link"), + default_active_only=False, ) - groups: List[List[str]] = [] - for i in range(0, total, width): - groups.append(names[i : i + width]) - return groups - - -def group_by_attr( - blueprint: Dict[str, Any], *, attr: str, roles: set[str] | None -) -> Dict[str, List[str]]: - """Group device names by a blueprint group attribute value. - - Uses per-blueprint group attribute (constant for the subgroup) as label. - Returns a mapping label -> list of device names sorted by natural order. - """ - - labels: Dict[str, List[str]] = {} - groups = blueprint.get("nodes", {}) if isinstance(blueprint, dict) else {} - for _gname, gdef in groups.items(): - attrs = gdef.get("attrs", {}) if isinstance(gdef, dict) else {} - role = str(attrs.get("role", "")) - if roles and role not in roles: - continue - label = attrs.get(attr) - if label is None: - continue - label_str = str(label) - try: - n = int(gdef.get("count", 0)) - except Exception: - n = 0 - template = str(gdef.get("template", "")) - for i in range(1, n + 1): - namestr = template.replace("{n}", str(i)) - labels.setdefault(label_str, []).append(namestr) - for lab in list(labels.keys()): - labels[lab].sort(key=_natural_key) - return labels - - -def build_node_overrides_for_site( - site_path: str, stripe_attr: str, label_to_names: Dict[str, List[str]] -) -> List[Dict[str, Any]]: - """Create DSL ``node_overrides`` to assign a stripe attribute to nodes. - - Matches nodes by anchored regex on full path with exactly one subgroup level - (blueprint group name), e.g., ``^metro3/pop1/[^/]+/leaf3$``. - """ - - overrides: List[Dict[str, Any]] = [] - for label, names in label_to_names.items(): - for nodename in names: - pattern = f"^{site_path}/[^/]+/{nodename}$" - overrides.append({"path": pattern, "attrs": {stripe_attr: label}}) - return overrides - - -def make_stripe_attr_name(scope: str) -> str: - """Return a stable stripe attribute name for a scope string. - - Example scopes: - - ``im_5_7`` for inter-metro between indices 5 and 7 - - ``dc_5`` for dc-to-pop inside metro 5 - """ - - return f"stripe_{scope}" + selected = { + node.name.removeprefix("stripe_probe/"): node.attrs + for nodes in groups.values() + for node in nodes + } + if not selected: + raise ValueError(f"No eligible stripe devices in blueprint '{blueprint}'") + names = sorted(selected, key=_natural_key) + mode = striping.get("mode", "width") + if mode == "width": + width = striping["width"] + if ( + not isinstance(width, int) + or isinstance(width, bool) + or width <= 0 + or len(names) % width + ): + raise ValueError( + f"Eligible device count {len(names)} requires a positive divisible striping.width" + ) + return { + f"g{offset // width + 1}": names[offset : offset + width] + for offset in range(0, len(names), width) + } + if mode != "by_attr": + raise ValueError(f"Unknown striping.mode: {mode}") + attribute = striping["attribute"] + labels: dict[str, list[str]] = {} + for name in names: + if attribute not in selected[name]: + raise ValueError( + f"Stripe device '{blueprint}/{name}' lacks attribute '{attribute}'" + ) + labels.setdefault(str(selected[name][attribute]), []).append(name) + return {label: labels[label] for label in sorted(labels)} + + def attach(self, site: str, attribute: str, groups: dict[str, list[str]]) -> None: + for label, members in groups.items(): + for member in members: + path = f"^{re.escape(site + '/' + member)}$" + attrs = self._rules.setdefault(path, {}) + if attribute in attrs and attrs[attribute] != label: + raise ValueError( + f"Conflicting stripe assignment for {site}/{member}" + ) + attrs[attribute] = label + + def node_rules(self) -> list[dict[str, Any]]: + return [{"path": path, "attrs": attrs} for path, attrs in self._rules.items()] diff --git a/topogen/scenario/traffic.py b/topogen/scenario/traffic.py index 92c8266..16178f5 100644 --- a/topogen/scenario/traffic.py +++ b/topogen/scenario/traffic.py @@ -1,23 +1,12 @@ -"""Map generated traffic matrices to the NetGraph ``demands`` section.""" +"""Map generated traffic matrices to the NetGraph demands contract.""" -from __future__ import annotations +from typing import Any -from typing import TYPE_CHECKING, Any -from topogen.traffic_matrix import generate_traffic_matrix - -if TYPE_CHECKING: # pragma: no cover - import-time types only - from topogen.config import TopologyConfig - - -def _build_traffic_matrix_section( - metros: list[dict[str, Any]], - metro_settings: dict[str, dict[str, Any]], - config: "TopologyConfig", +def to_demand_sets( + matrices: dict[str, list[dict[str, Any]]], ) -> dict[str, list[dict[str, Any]]]: - """Return NetGraph demand sets, or {} when traffic is disabled or no DCs exist.""" - - # Map traffic generator fields to NetGraph demand fields. + """Translate the same matrices used for sizing without regenerating traffic.""" field_names = { "source_path": "source", "sink_path": "target", @@ -29,7 +18,5 @@ def _build_traffic_matrix_section( {field_names.get(key, key): value for key, value in demand.items()} for demand in demands ] - for name, demands in generate_traffic_matrix( - metros, metro_settings, config - ).items() + for name, demands in matrices.items() } diff --git a/topogen/scenario_builder.py b/topogen/scenario_builder.py deleted file mode 100644 index d22a1e6..0000000 --- a/topogen/scenario_builder.py +++ /dev/null @@ -1,30 +0,0 @@ -"""Public scenario-building entry points, re-exported from ``topogen.scenario``.""" - -from __future__ import annotations - -from topogen.scenario.assembly import _add_adjacency_comments, build_scenario -from topogen.scenario.config import _determine_metro_settings -from topogen.scenario.libraries import ( - _build_blueprints_section, - _build_components_section, -) -from topogen.scenario.network import _extract_metros_from_graph -from topogen.scenario.policies import ( - _build_failure_policy_set_section, - _build_workflow_section, -) -from topogen.scenario.risk import _build_risk_groups_section -from topogen.scenario.traffic import _build_traffic_matrix_section - -__all__ = [ - "build_scenario", - "_add_adjacency_comments", - "_determine_metro_settings", - "_build_blueprints_section", - "_build_components_section", - "_extract_metros_from_graph", - "_build_failure_policy_set_section", - "_build_workflow_section", - "_build_risk_groups_section", - "_build_traffic_matrix_section", -] diff --git a/topogen/schemas/topogen_config.json b/topogen/schemas/topogen_config.json index 92502fc..e94fa97 100644 --- a/topogen/schemas/topogen_config.json +++ b/topogen/schemas/topogen_config.json @@ -102,16 +102,8 @@ "type": "string" } }, - "min_cycle_nodes": { - "type": "integer", - "minimum": 0 - }, "filter_largest_component": { "type": "boolean" - }, - "validation_sample_size": { - "type": "integer", - "minimum": 0 } } }, @@ -182,25 +174,9 @@ "type": "object", "additionalProperties": false, "required": [ - "scenario_metadata", "formatting" ], "properties": { - "scenario_metadata": { - "type": "object", - "additionalProperties": false, - "properties": { - "title": { - "type": "string" - }, - "description": { - "type": "string" - }, - "version": { - "type": "string" - } - } - }, "formatting": { "type": "object", "additionalProperties": false, @@ -330,10 +306,10 @@ "minimum": 0 }, "flow_placement": { - "type": "string" - }, - "edge_select": { - "type": "string" + "enum": [ + "EQUAL_BALANCED", + "PROPORTIONAL" + ] }, "respect_min_base_capacity": { "type": "boolean" @@ -342,9 +318,57 @@ } } }, - "components": {}, - "failure_policies": {}, - "workflows": {}, + "components": { + "type": "object", + "additionalProperties": false, + "properties": { + "hw_component": { + "type": "object", + "additionalProperties": { + "type": "string" + } + }, + "optics": { + "type": "object", + "additionalProperties": { + "type": "string" + }, + "propertyNames": { + "pattern": "^[^|>]+->[^|>]+$" + } + } + } + }, + "failure_policies": { + "type": "object", + "additionalProperties": false, + "properties": { + "assignments": { + "type": "object", + "additionalProperties": false, + "properties": { + "default": { + "type": "string" + } + } + } + } + }, + "workflows": { + "type": "object", + "additionalProperties": false, + "properties": { + "assignments": { + "type": "object", + "additionalProperties": false, + "properties": { + "default": { + "type": "string" + } + } + } + } + }, "traffic": { "type": "object", "additionalProperties": false, @@ -394,15 +418,15 @@ "properties": { "alpha": { "type": "number", - "minimum": 0 + "exclusiveMinimum": 0 }, "beta": { "type": "number", - "minimum": 0 + "exclusiveMinimum": 0 }, "min_distance_km": { "type": "number", - "minimum": 0 + "exclusiveMinimum": 0 }, "max_partners_per_dc": { "type": [ @@ -430,8 +454,12 @@ "mw_per_dc_region_overrides": { "type": "object", "additionalProperties": { - "type": "number" + "type": "number", + "minimum": 0 } + }, + "exclude_same_metro": { + "type": "boolean" } } }, @@ -445,11 +473,11 @@ }, "beta": { "type": "number", - "minimum": 0 + "exclusiveMinimum": 0 }, "min_distance_km": { "type": "number", - "minimum": 0 + "exclusiveMinimum": 0 }, "exclude_same_metro": { "type": "boolean" @@ -533,23 +561,65 @@ "role_pairs": { "type": "array", "items": { - "oneOf": [ - { - "type": "string" - }, - { - "type": "array", - "items": { - "type": "string" - }, - "minItems": 2, - "maxItems": 2 - } - ] + "type": "string", + "pattern": "^[^|]+\\|[^|]+$" } }, "striping": { - "type": "object" + "type": "object", + "additionalProperties": false, + "properties": { + "mode": { + "enum": [ + "width", + "by_attr" + ] + }, + "width": { + "type": "integer", + "minimum": 1 + }, + "attribute": { + "type": "string", + "minLength": 1 + } + }, + "oneOf": [ + { + "maxProperties": 0 + }, + { + "required": [ + "width" + ], + "properties": { + "mode": { + "const": "width" + } + }, + "not": { + "required": [ + "attribute" + ] + } + }, + { + "required": [ + "mode", + "attribute" + ], + "properties": { + "mode": { + "const": "by_attr" + } + }, + "not": { + "required": [ + "width" + ] + } + } + ] }, "mode": { "type": "string", @@ -558,22 +628,6 @@ "one_to_one" ] } - }, - "required": [ - "capacity", - "cost" - ] - }, - "ComponentAssignment": { - "type": "object", - "additionalProperties": false, - "properties": { - "hw_component": { - "type": "string" - }, - "optics": { - "type": "string" - } } } } diff --git a/topogen/traffic_matrix.py b/topogen/traffic_matrix.py index 6acf7c6..60cde8e 100644 --- a/topogen/traffic_matrix.py +++ b/topogen/traffic_matrix.py @@ -1,950 +1,358 @@ -"""Generate DC-to-DC traffic using uniform, gravity, or hose models. +"""Generate deterministic uniform, gravity and hose demand sets. -Uniform emits a pairwise selector demand. Gravity allocates traffic by DC power -and Euclidean distance, with optional jitter, pruning, and rounding. Hose uses -iterative proportional fitting to target per-DC totals, then emits symmetric -pairs. Rounding can change the final totals. - -Gravity and hose enumerate DC pairs and use O(N_dc^2) intermediate storage. +Gravity and hose share pair selection, rounding and symmetric emission. Their +pair inventories are quadratic in DC count; uniform emits one selector per class. """ from __future__ import annotations +import heapq import json +import logging import math -import random as _random -from pathlib import Path +import random +from dataclasses import dataclass from typing import Any -from topogen.config import TopologyConfig, _validate_flow_policy_names +import numpy as np + +from topogen.config import TopologyConfig, TrafficConfig, _validate_flow_policy_names from topogen.log_config import get_logger logger = get_logger(__name__) - - -def _fmt(value: float, *, decimals: int = 2) -> str: - """Format a number with thousands separators and the requested precision.""" - - fmt = f"{{:,.{decimals}f}}" - return fmt.format(float(value)) - - -def _safe_metro_to_path(metro_name: str) -> str: - """Normalize a metro name for use in a DC path, such as ``salt-lake-city/dc1``.""" - - return metro_name.lower().replace(" ", "-") - - -def generate_traffic_matrix( - metros: list[dict[str, Any]], - metro_settings: dict[str, dict[str, Any]], - config: TopologyConfig, -) -> dict[str, list[dict[str, Any]]]: - """Generate traffic matrices from configuration and metros. - - Builds DC inventory from ``metro_settings``, computes the offered load, and - emits traffic matrices according to the configured model. When the gravity - model is selected, per-pair allocations are proportional to - (m_i^alpha * m_j^alpha) / (dist_ij^beta) using Euclidean distances in km. - - Args: - metros: Extracted metro descriptors with keys ``name``, ``x``, ``y``. - metro_settings: Per-metro settings including ``dc_regions_per_metro``. - config: Full topology configuration. - - Returns: - Mapping from matrix name to list of demand dicts. Returns empty dict - when traffic generation is disabled or no DCs exist. Demand fields are - ``source_path``, ``sink_path``, ``demand``, and optional - ``flow_policy_config``; scenario assembly maps these to NetGraph fields. - - Raises: - ValueError: If the gravity model yields zero total weight or if top-K - pruning removes all pairs. - """ - - traffic_cfg = getattr(config, "traffic", None) - if not traffic_cfg or not getattr(traffic_cfg, "enabled", False): - logger.debug("Traffic generation disabled or no traffic configuration present") - return {} - - # Configs can also be constructed or mutated through the Python API. - fpc = traffic_cfg.flow_policy_config - _validate_flow_policy_names(fpc) - - dc_nodes: list[tuple[str, int]] = [] # (metro_name, dc_index) - for metro_name, settings in metro_settings.items(): - dc_count = int(settings.get("dc_regions_per_metro", 0)) - for dc_idx in range(1, dc_count + 1): - dc_nodes.append((metro_name, dc_idx)) - - if not dc_nodes: - logger.debug("No DC regions configured; skipping traffic matrix generation") - return {} - - model = str(getattr(traffic_cfg, "model", "uniform")).strip() - - def _power_for_dc(metro_name: str, dc_path: str) -> float: - # Override by full path or by metro name; else default - overrides = getattr(traffic_cfg.gravity, "mw_per_dc_region_overrides", {}) - if dc_path in overrides: - return float(overrides[dc_path]) - if metro_name in overrides: - return float(overrides[metro_name]) - return float(traffic_cfg.mw_per_dc_region) - - dc_mass: dict[tuple[str, int], float] = {} - total_power_mw = 0.0 - for metro_name, dc_idx in dc_nodes: - dc_path = f"{_safe_metro_to_path(metro_name)}/dc{dc_idx}" - mw = _power_for_dc(metro_name, dc_path) - dc_mass[(metro_name, dc_idx)] = mw - total_power_mw += mw - - offered_gbps = float(traffic_cfg.gbps_per_mw) * float(total_power_mw) - - try: - logger.debug( - "Traffic model=%s matrix_name=%s gbps_per_mw=%s mw_per_dc_region=%s", - getattr(traffic_cfg, "model", "uniform"), - getattr(traffic_cfg, "matrix_name", "default"), - _fmt(float(traffic_cfg.gbps_per_mw), decimals=3), - _fmt(float(traffic_cfg.mw_per_dc_region), decimals=3), - ) - logger.debug( - "DC inventory (MW) total=%s: %s", - _fmt(total_power_mw, decimals=3), - ", ".join( - f"{m}/dc{d}={_fmt(dc_mass[(m, d)], decimals=3)}" - for (m, d) in sorted(dc_mass.keys()) - ), +Pair = tuple[int, int] + + +def _fit_hose_matrix( + matrix: list[list[float]], targets: list[float] +) -> list[list[float]]: + """Fit supported pairs to both margins, rejecting infeasible or unfinished fits.""" + values = np.asarray(matrix, dtype=float) + margins = np.asarray(targets, dtype=float) + if ( + not np.all(np.isfinite(values)) + or not np.all(np.isfinite(margins)) + or np.any(values < 0) + or np.any(margins < 0) + ): + raise ValueError("hose requires finite non-negative weights and totals") + total = math.fsum(targets) + largest = int(np.argmax(margins)) + rest = math.fsum(value for index, value in enumerate(targets) if index != largest) + if margins[largest] > rest and not math.isclose( + margins[largest], rest, rel_tol=1e-12, abs_tol=0 + ): + raise ValueError( + "hose margins are infeasible: one DC exceeds all peers combined" ) - logger.debug("Offered traffic (Gbps)=%s", _fmt(offered_gbps)) - except Exception: # pragma: no cover - logging only - pass - - if model == "uniform": - # Uniform model emission using regex selection across all DCs - source_regex = "(metro[0-9]+/dc[0-9]+)" - sink_regex = "(metro[0-9]+/dc[0-9]+)" - demands: list[dict[str, Any]] = [] - for priority, ratio in sorted(traffic_cfg.priority_ratios.items()): - class_demand = offered_gbps * float(ratio) - if class_demand <= 0.0: - # Skip zero or negative class demand entries + # At equality, only a star through the largest DC can satisfy both margins. + if math.isclose(margins[largest], rest, rel_tol=1e-12, abs_tol=0): + star = np.zeros_like(values) + for index, target in enumerate(targets): + if index == largest or target == 0: continue - entry = { - "source_path": source_regex, - "sink_path": sink_regex, - "mode": "pairwise", - "priority": int(priority), - "demand": float(class_demand), - } - if int(priority) in fpc: - entry["flow_policy_config"] = fpc[int(priority)] - demands.append(entry) - try: - for priority, ratio in sorted(traffic_cfg.priority_ratios.items()): - logger.debug( - "uniform: class priority=%s ratio=%s class_demand_gbps=%s", - int(priority), - _fmt(float(ratio), decimals=6), - _fmt(offered_gbps * float(ratio)), + if values[largest, index] <= 0 or values[index, largest] <= 0: + raise ValueError("hose support is infeasible for the requested margins") + star[largest, index] = star[index, largest] = target + return star.tolist() + for _ in range(2000): + for axis in (1, 0): + sums = values.sum(axis=axis) + if np.any((sums == 0) & (margins > 0)): + raise ValueError( + "hose support is infeasible: a DC has no eligible peers" ) - pretty = { - str(traffic_cfg.matrix_name): [ - { - "source_path": d["source_path"], - "sink_path": d["sink_path"], - "mode": d["mode"], - "priority": int(d["priority"]), - "demand_gbps": _fmt(float(d["demand"])) - if isinstance(d.get("demand"), (int, float)) - else str(d.get("demand")), - } - for d in demands - ] - } - logger.debug( - "traffic_matrix pretty:\n%s", - json.dumps(pretty, indent=2, ensure_ascii=True), - ) - except Exception: # pragma: no cover - logging only - pass - return {traffic_cfg.matrix_name: demands} - - if model == "hose": - # Fit randomized hose matrices toward the per-DC totals. - if total_power_mw <= 0.0: - logger.debug("hose: total_power_mw is zero; skipping traffic generation") - return {} - - T: dict[tuple[str, int], float] = { - k: offered_gbps * (float(mw) / float(total_power_mw)) - for k, mw in dc_mass.items() - } - - # Map metro name to 1-based index for regex paths - metro_idx_map_hose = {m["name"]: idx for idx, m in enumerate(metros, 1)} - - # Hose tilt configuration (gravity-tilted initialization) - hcfg = getattr(traffic_cfg, "hose", object()) - try: - tilt_exp = float(getattr(hcfg, "tilt_exponent", 0.0)) - hose_beta = float(getattr(hcfg, "beta", 1.0)) - hose_min_km = float(getattr(hcfg, "min_distance_km", 1.0)) - hose_excl_same = bool(getattr(hcfg, "exclude_same_metro", False)) - carve_top_k = getattr(hcfg, "carve_top_k", None) - except Exception: - tilt_exp = 0.0 - hose_beta = 1.0 - hose_min_km = 1.0 - hose_excl_same = False - carve_top_k = None - - coords_hose: dict[str, tuple[float, float]] = { - m["name"]: (float(m.get("x", 0.0)), float(m.get("y", 0.0))) for m in metros - } - - def _hose_distance_km(name_a: str, name_b: str) -> float: - if name_a == name_b: - return max(hose_min_km, 0.0) - (x1, y1) = coords_hose.get(name_a, (0.0, 0.0)) - (x2, y2) = coords_hose.get(name_b, (0.0, 0.0)) - return max(math.hypot(x1 - x2, y1 - y2) / 1000.0, hose_min_km) - - # Seeded RNG for reproducibility; vary with sample index - try: - base_seed = int( - getattr(getattr(config, "output", object()), "scenario_seed", 42) + factors = np.divide( + margins, sums, out=np.zeros_like(margins), where=sums > 0 ) - except Exception: - base_seed = 42 - - def _ipf_directed_matrix(rng: _random.Random) -> list[list[float]]: - """Fit a zero-diagonal matrix toward per-DC row and column totals. + values *= factors[:, None] if axis == 1 else factors[None, :] + if all( + np.allclose(values.sum(axis=axis), margins, rtol=1e-9, atol=total * 1e-12) + for axis in (0, 1) + ): + return values.tolist() + raise ValueError( + "hose fitting did not converge after 2000 iterations; adjust DC totals or retained partners" + ) - Stop at absolute error 1e-6 or after 2,000 iterations; return the last fit. - """ - n = len(dc_nodes) - t_vec = [T[dc_nodes[i]] for i in range(n)] - D = [[0.0 for _ in range(n)] for _ in range(n)] - for i in range(n): - (mi, di) = dc_nodes[i] - for j in range(n): - if i == j: - continue - (mj, dj) = dc_nodes[j] - # Random base > 0 - base = 1e-9 + rng.random() - if tilt_exp > 0.0: - if hose_excl_same and mi == mj: - weight = 0.0 - else: - dist_km = _hose_distance_km(mi, mj) - weight = 1.0 / (dist_km**hose_beta) if dist_km > 0 else 0.0 - if weight <= 0.0: - # Keep strictly positive but very small to retain feasibility - tilt = 1e-12 - else: - tilt = weight**tilt_exp - D[i][j] = base * tilt - else: - D[i][j] = base +@dataclass(frozen=True, slots=True) +class _DC: + metro: str + path: str + mass: float - max_iter = 2000 - tol = 1e-6 - for _ in range(max_iter): - for i in range(n): - row_sum = sum(D[i][j] for j in range(n) if j != i) - if row_sum > 0.0: - factor = t_vec[i] / row_sum - if factor != 1.0: - for j in range(n): - if i != j: - D[i][j] *= factor - max_dev = 0.0 - for j in range(n): - col_sum = sum(D[i][j] for i in range(n) if i != j) - if col_sum > 0.0: - factor = t_vec[j] / col_sum - if factor != 1.0: - for i in range(n): - if i != j: - D[i][j] *= factor - col_sum2 = sum(D[i][j] for i in range(n) if i != j) - max_dev = max(max_dev, abs(col_sum2 - t_vec[j])) - for i in range(n): - row_sum2 = sum(D[i][j] for j in range(n) if j != i) - max_dev = max(max_dev, abs(row_sum2 - t_vec[i])) - if max_dev <= tol: - break - return D - num_samples = int(getattr(traffic_cfg, "samples", 1)) - base_name = str(getattr(traffic_cfg, "matrix_name", "default")) - # Reuse gravity rounding settings for undirected totals, then split equally - rounding = float( - getattr(getattr(traffic_cfg, "gravity", object()), "rounding_gbps", 0.0) - ) - rounding_policy = str( - getattr( - getattr(traffic_cfg, "gravity", object()), "rounding_policy", "nearest" +def _inventory( + metros: list[dict[str, Any]], + settings: dict[str, dict[str, Any]], + config: TrafficConfig, +) -> list[_DC]: + inventory = [] + overrides = config.gravity.mw_per_dc_region_overrides + for metro_index, metro in enumerate(metros, 1): + name = metro["name"] + prefix = name.lower().replace(" ", "-") + for ordinal in range(1, settings[name]["dc_regions_per_metro"] + 1): + mass = float( + overrides.get( + f"{prefix}/dc{ordinal}", + overrides.get(name, config.mw_per_dc_region), + ) ) - ) - - result_hose: dict[str, list[dict[str, Any]]] = {} - for s_idx in range(1, num_samples + 1): - rng = _random.Random(base_seed * 1000003 + s_idx) - D = _ipf_directed_matrix(rng) - # Optional gravity carve step: keep top-K partners per DC by undirected total - if carve_top_k is not None: - k = int(carve_top_k) - n = len(dc_nodes) - und: dict[tuple[int, int], float] = {} - for i in range(n): - for j in range(i + 1, n): - u = float(D[i][j] + D[j][i]) - if u > 0.0: - und[(i, j)] = u - partner_map: dict[int, list[tuple[int, float]]] = {} - for (i, j), u in und.items(): - partner_map.setdefault(i, []).append((j, u)) - partner_map.setdefault(j, []).append((i, u)) - keep_pairs: set[tuple[int, int]] = set() - for i, lst in partner_map.items(): - lst_sorted = sorted(lst, key=lambda t: t[1], reverse=True)[:k] - for j, _ in lst_sorted: - a, b = (i, j) if i < j else (j, i) - keep_pairs.add((a, b)) - # Reinitialize retained pairs and refit toward the per-DC totals. - D = [[0.0 for _ in range(n)] for _ in range(n)] - for i in range(n): - (mi, di) = dc_nodes[i] - for j in range(n): - if i == j: - continue - (mj, dj) = dc_nodes[j] - a, b = (i, j) if i < j else (j, i) - if (a, b) not in keep_pairs: - continue - base = 1e-9 + rng.random() - if tilt_exp > 0.0: - if hose_excl_same and mi == mj: - weight = 0.0 - else: - dist_km = _hose_distance_km(mi, mj) - weight = ( - 1.0 / (dist_km**hose_beta) if dist_km > 0 else 0.0 - ) - tilt = (weight**tilt_exp) if weight > 0.0 else 1e-12 - D[i][j] = base * tilt - else: - D[i][j] = base - max_iter = 2000 - tol = 1e-6 - for _ in range(max_iter): - for i in range(n): - row_sum = sum(D[i][j] for j in range(n) if j != i) - if row_sum > 0.0: - factor = T[dc_nodes[i]] / row_sum - if factor != 1.0: - for j in range(n): - if i != j: - D[i][j] *= factor - max_dev = 0.0 - for j in range(n): - col_sum = sum(D[i][j] for i in range(n) if i != j) - if col_sum > 0.0: - factor = T[dc_nodes[j]] / col_sum - if factor != 1.0: - for i in range(n): - if i != j: - D[i][j] *= factor - col_sum2 = sum(D[i][j] for i in range(n) if i != j) - max_dev = max(max_dev, abs(col_sum2 - T[dc_nodes[j]])) - for i in range(n): - row_sum2 = sum(D[i][j] for j in range(n) if j != i) - max_dev = max(max_dev, abs(row_sum2 - T[dc_nodes[i]])) - if max_dev <= tol: - break - n = len(dc_nodes) - undirected: list[tuple[tuple[int, int], float]] = [] - for i in range(n): - for j in range(i + 1, n): - total_ij = float(D[i][j] + D[j][i]) - if total_ij > 0.0: - undirected.append(((i, j), total_ij)) - - demands: list[dict[str, Any]] = [] - for priority, ratio in sorted(traffic_cfg.priority_ratios.items()): - D_c = offered_gbps * float(ratio) - allocs_hose = [((i, j), u * float(ratio)) for ((i, j), u) in undirected] - - # Round undirected totals and distribute positive residual by largest remainder. - if rounding > 0.0: - floored_hose: list[tuple[tuple[int, int], float, float]] = [] - total_floor = 0.0 - for (i, j), v in allocs_hose: - if rounding_policy == "ceil": - q = math.ceil(v / rounding) * rounding - elif rounding_policy == "floor": - q = math.floor(v / rounding) * rounding - else: - q = round(v / rounding) * rounding - rem = v - q - floored_hose.append(((i, j), q, rem)) - total_floor += q - remainder = D_c - total_floor - steps = int(round(remainder / rounding)) if rounding > 0 else 0 - floored_hose.sort(key=lambda t: t[2], reverse=True) - final_map_hose: dict[tuple[int, int], float] = { - key: q for key, q, _ in floored_hose - } - idx = 0 - while steps > 0 and idx < len(floored_hose): - key, q, _ = floored_hose[idx] - final_map_hose[key] = q + rounding - steps -= 1 - idx += 1 - allocs_hose = list(final_map_hose.items()) - - dir_step = max(rounding / 2.0, 0.0) - for (i, j), v in allocs_hose: - (m1, d1) = dc_nodes[i] - (m2, d2) = dc_nodes[j] - i1 = metro_idx_map_hose[m1] - i2 = metro_idx_map_hose[m2] - src = f"^metro{i1}/dc{d1}/.*" - dst = f"^metro{i2}/dc{d2}/.*" - dist_km = _hose_distance_km(m1, m2) - demand_each_raw = float(v) / 2.0 - if dir_step > 0.0: - demand_each = round(demand_each_raw / dir_step) * dir_step - else: - demand_each = round(demand_each_raw, 2) - if demand_each <= 0.0: - continue - entry_fwd = { - "source_path": src, - "sink_path": dst, - "mode": "pairwise", - "priority": int(priority), - "demand": float(demand_each), - "attrs": {"euclidean_km": int(math.ceil(float(dist_km)))}, - } - entry_rev = { - "source_path": dst, - "sink_path": src, - "mode": "pairwise", - "priority": int(priority), - "demand": float(demand_each), - "attrs": {"euclidean_km": int(math.ceil(float(dist_km)))}, - } - if int(priority) in fpc: - entry_fwd["flow_policy_config"] = fpc[int(priority)] - entry_rev["flow_policy_config"] = fpc[int(priority)] - demands.append(entry_fwd) - demands.append(entry_rev) - - matrix_name = base_name if num_samples == 1 else f"{base_name}_{s_idx}" - result_hose[matrix_name] = demands - - try: - first = next(iter(result_hose.keys())) if result_hose else base_name - pretty = { - str(first): [ - { - "source_path": d["source_path"], - "sink_path": d["sink_path"], - "mode": d["mode"], - "priority": int(d["priority"]), - "demand_gbps": _fmt(float(d["demand"])) - if isinstance(d.get("demand"), (int, float)) - else str(d.get("demand")), - } - for d in result_hose.get(first, []) - ] - } - logger.debug( - "traffic_matrix pretty (hose sample):\n%s", - json.dumps(pretty, indent=2, ensure_ascii=True), + if not math.isfinite(mass) or mass < 0: + raise ValueError( + f"DC power for {name}/dc{ordinal} must be finite and non-negative" + ) + inventory.append( + _DC( + name, + f"^metro{metro_index}/dc{ordinal}/.*", + mass, + ) ) - except Exception: # pragma: no cover - logging only - pass - - return result_hose + return inventory + + +def _distances( + dcs: list[_DC], metros: list[dict[str, Any]], minimum: float +) -> np.ndarray: + positions = {metro["name"]: (metro["x"], metro["y"]) for metro in metros} + xy = np.array([positions[dc.metro] for dc in dcs], dtype=float) + return np.maximum( + np.hypot(xy[:, None, 0] - xy[None, :, 0], xy[:, None, 1] - xy[None, :, 1]) + / 1000, + minimum, + ) - gcfg = traffic_cfg.gravity - # Metro coordinates (EPSG:5070 meters); convert to km for distance - coords: dict[str, tuple[float, float]] = { - m["name"]: (float(m.get("x", 0.0)), float(m.get("y", 0.0))) for m in metros +def _top_pairs(weights: dict[Pair, float], k: int) -> dict[Pair, float]: + """Keep the union of each DC's strongest K partners, with stable ties.""" + if k <= 0: + raise ValueError("Partner count must be positive") + partners: dict[int, list[tuple[Pair, float]]] = {} + for pair, value in weights.items(): + for endpoint in pair: + partners.setdefault(endpoint, []).append((pair, value)) + keep = { + pair + for entries in partners.values() + for pair, _ in heapq.nlargest(k, entries, key=lambda item: item[1]) + } + return {pair: value for pair, value in weights.items() if pair in keep} + + +def _quantize( + allocations: dict[Pair, float], total: float, step: float, policy: str +) -> dict[Pair, float]: + """Quantize pair totals and distribute positive residual by largest remainder.""" + if step == 0: + return allocations + quantizer = {"nearest": round, "ceil": math.ceil, "floor": math.floor}[policy] + rounded = [ + (pair, quantizer(value / step) * step, value) + for pair, value in allocations.items() + ] + rounded.sort(key=lambda item: item[2] - item[1], reverse=True) + steps = max(0, int(round((total - sum(value for _, value, _ in rounded)) / step))) + return { + pair: value + (step if index < steps else 0) + for index, (pair, value, _) in enumerate(rounded) } - def _distance_km(m1: str, m2: str) -> float: - if m1 == m2: - return max(float(gcfg.min_distance_km), 0.0) - (x1, y1) = coords.get(m1, (0.0, 0.0)) - (x2, y2) = coords.get(m2, (0.0, 0.0)) - dx = x1 - x2 - dy = y1 - y2 - return max(math.hypot(dx, dy) / 1000.0, float(gcfg.min_distance_km)) - weights: dict[tuple[tuple[str, int], tuple[str, int]], float] = {} - total_w = 0.0 - logger.debug( - "Distance model: Euclidean straight-line km (EPSG:5070 coords), min_distance_km=%s", - _fmt(float(gcfg.min_distance_km), decimals=3), - ) - logger.debug("traffic: diagnostics begin") - for i, (m1, d1) in enumerate(dc_nodes): - for j in range(i + 1, len(dc_nodes)): - m2, d2 = dc_nodes[j] - if gcfg.exclude_same_metro and m1 == m2: +def _emit_pairs( + dcs: list[_DC], + pairs: dict[Pair, float], + distances: np.ndarray, + config: TrafficConfig, + offered: float, + *, + rng: random.Random | None = None, +) -> list[dict[str, Any]]: + demands = [] + gravity = config.gravity + step = gravity.rounding_gbps + for priority, ratio in sorted(config.priority_ratios.items()): + total = offered * ratio + if total <= 0: + continue + allocations = {pair: value * ratio for pair, value in pairs.items()} + if rng is not None and gravity.jitter_stddev > 0: + sigma = gravity.jitter_stddev + allocations = { + pair: value * rng.lognormvariate(-0.5 * sigma * sigma, sigma) + for pair, value in allocations.items() + } + jittered_total = sum(allocations.values()) + if not math.isfinite(jittered_total) or jittered_total <= 0: + raise ValueError("Gravity jitter produced an invalid total") + allocations = { + pair: value * (total / jittered_total) + for pair, value in allocations.items() + } + allocations = _quantize(allocations, total, step, gravity.rounding_policy) + for (source, target), value in allocations.items(): + each = value / 2 + if step > 0: + each = round(each / (step / 2)) * (step / 2) + if each <= 0: continue - m_i = dc_mass[(m1, d1)] - m_j = dc_mass[(m2, d2)] - dist = _distance_km(m1, m2) - dist_eff = max(dist, float(gcfg.min_distance_km)) - w = ( - (m_i ** float(gcfg.alpha)) - * (m_j ** float(gcfg.alpha)) - / (dist_eff ** float(gcfg.beta)) - ) - if w <= 0.0: + distance = math.ceil(float(distances[source, target])) + for a, b in ((source, target), (target, source)): + entry = { + "source_path": dcs[a].path, + "sink_path": dcs[b].path, + "mode": "pairwise", + "priority": priority, + "demand": each, + "attrs": {"euclidean_km": distance}, + } + if priority in config.flow_policy_config: + entry["flow_policy_config"] = config.flow_policy_config[priority] + demands.append(entry) + return demands + + +def _gravity_pairs( + dcs: list[_DC], distances: np.ndarray, config: TrafficConfig, offered: float +) -> dict[Pair, float]: + gravity = config.gravity + weights = {} + for i, a in enumerate(dcs): + for j in range(i + 1, len(dcs)): + b = dcs[j] + if gravity.exclude_same_metro and a.metro == b.metro: continue - key = ((m1, d1), (m2, d2)) - weights[key] = w - total_w += w - - if total_w <= 0.0: - raise ValueError( - "Gravity traffic model produced zero total weight across DC pairs" - ) - - try: - logger.debug( - ( - "gravity: alpha=%s beta=%s min_distance_km=%s " - "exclude_same_metro=%s jitter_stddev=%s rounding_gbps=%s " - "directional_step_gbps=%s max_partners_per_dc=%s" - ), - _fmt(float(gcfg.alpha), decimals=6), - _fmt(float(gcfg.beta), decimals=6), - _fmt(float(gcfg.min_distance_km), decimals=3), - bool(gcfg.exclude_same_metro), - _fmt(float(gcfg.jitter_stddev), decimals=6), - _fmt(float(gcfg.rounding_gbps), decimals=3), - _fmt(float(gcfg.rounding_gbps) / 2.0, decimals=3), - ( - None - if gcfg.max_partners_per_dc is None - else int(gcfg.max_partners_per_dc) - ), - ) - top_pairs = sorted(weights.items(), key=lambda kv: kv[1], reverse=True)[:10] - for (m1, d1), (m2, d2) in [p[0] for p in top_pairs]: - w = weights[((m1, d1), (m2, d2))] - frac = w / total_w if total_w > 0 else 0.0 - logger.debug( - "gravity: pair %s/dc%s <-> %s/dc%s weight=%s frac=%s", - m1, - d1, - m2, - d2, - _fmt(w, decimals=6), - _fmt(frac, decimals=6), - ) - except Exception: # pragma: no cover - logging only - pass - - # Optional top-K pruning per DC - if gcfg.max_partners_per_dc is not None: - k = int(gcfg.max_partners_per_dc) - partners: dict[tuple[str, int], list[tuple[tuple[str, int], float]]] = {} - for (a, b), w in weights.items(): - partners.setdefault(a, []).append((b, w)) - partners.setdefault(b, []).append((a, w)) - keep: set[tuple[tuple[str, int], tuple[str, int]]] = set() - for node, lst in partners.items(): - lst_sorted = sorted(lst, key=lambda t: t[1], reverse=True)[:k] - for other, _w in lst_sorted: - pair = tuple(sorted([node, other])) - keep.add((pair[0], pair[1])) - weights = {pair: w for pair, w in weights.items() if pair in keep} - total_w = sum(weights.values()) - if total_w <= 0.0: - raise ValueError( - "After top-K pruning, no DC pairs remain for gravity model" + value = ( + (a.mass**gravity.alpha) + * (b.mass**gravity.alpha) + / (float(distances[i, j]) ** gravity.beta) ) - - logger.debug( - "gravity: applied top-K pruning with k=%d; remaining_pairs=%d", - k, - len(weights), - ) - - # Map metro name to 1-based index consistent with network group naming - metro_idx_map = {m["name"]: idx for idx, m in enumerate(metros, 1)} - try: - logger.debug( - "regex index map: %s", - ", ".join( - f"metro{idx} -> {name}" - for name, idx in sorted(metro_idx_map.items(), key=lambda x: x[1]) - ), + if value > 0: + weights[i, j] = value + if gravity.max_partners_per_dc is not None: + weights = _top_pairs(weights, gravity.max_partners_per_dc) + total = sum(weights.values()) + if not math.isfinite(total) or total <= 0: + raise ValueError( + "Gravity traffic model produced zero or non-finite total weight across DC pairs" ) - except Exception: # pragma: no cover - logging only - pass - - # Emit per-pair demands with class-specific jitter and rounding. - demands: list[dict[str, Any]] = [] - debug_entries: list[dict[str, Any]] = [] - for priority, ratio in sorted(traffic_cfg.priority_ratios.items()): - D_c = offered_gbps * float(ratio) - allocs: list[tuple[tuple[tuple[str, int], tuple[str, int]], float]] = [] - for pair, w in weights.items(): - alloc = D_c * (w / total_w) - allocs.append((pair, alloc)) + return {pair: offered * (value / total) for pair, value in weights.items()} + + +def _random_hose( + dcs: list[_DC], + distances: np.ndarray, + config: TrafficConfig, + rng: random.Random, + keep: set[Pair] | None = None, +) -> list[list[float]]: + hose = config.hose + n = len(dcs) + matrix = [[0.0] * n for _ in range(n)] + for i, a in enumerate(dcs): + for j, b in enumerate(dcs): + if i == j or (keep is not None and (min(i, j), max(i, j)) not in keep): + continue + value = 1e-9 + rng.random() + if hose.tilt_exponent > 0: + # This setting excludes same-metro pairs from the tilt kernel, + # while keeping positive support for marginal fitting. + tilt = ( + 1e-12 + if hose.exclude_same_metro and a.metro == b.metro + else (1.0 / float(distances[i, j]) ** hose.beta) + ** hose.tilt_exponent + ) + value *= tilt + matrix[i][j] = value + return _fit_hose_matrix(matrix, [dc.mass * config.gbps_per_mw for dc in dcs]) - # Optional jitter (lognormal with sigma=jitter_stddev, mu set for mean=1) - if gcfg.jitter_stddev > 0.0: - import random as _r - sigma = float(gcfg.jitter_stddev) - mu = -0.5 * sigma * sigma - jittered: list[tuple[tuple[tuple[str, int], tuple[str, int]], float]] = [] - total_after = 0.0 - for pair, v in allocs: - factor = _r.lognormvariate(mu, sigma) - val = v * factor - jittered.append((pair, val)) - total_after += val - if total_after > 0: - allocs = [(pair, v * (D_c / total_after)) for pair, v in jittered] +def _hose_pairs(matrix: list[list[float]]) -> dict[Pair, float]: + return { + (i, j): matrix[i][j] + matrix[j][i] + for i in range(len(matrix)) + for j in range(i + 1, len(matrix)) + if matrix[i][j] + matrix[j][i] > 0 + } - logger.debug( - "gravity: class priority=%d ratio=%s class_total_gbps=%s", - int(priority), - _fmt(float(ratio), decimals=6), - _fmt(D_c), - ) - # Round pair totals and distribute positive residual by largest remainder. - rounding = float(gcfg.rounding_gbps) - if rounding > 0.0: - floored: list[ - tuple[tuple[tuple[str, int], tuple[str, int]], float, float] - ] = [] - total_floor = 0.0 - for pair, v in allocs: - if getattr(gcfg, "rounding_policy", "nearest") == "ceil": - q = math.ceil(v / rounding) * rounding - elif getattr(gcfg, "rounding_policy", "nearest") == "floor": - q = math.floor(v / rounding) * rounding - else: - q = round(v / rounding) * rounding - rem = v - q - floored.append((pair, q, rem)) - total_floor += q - remainder = D_c - total_floor - steps = int(round(remainder / rounding)) if rounding > 0 else 0 - # Distribute leftover to pairs with largest remainders - floored.sort(key=lambda t: t[2], reverse=True) - final_map: dict[tuple[tuple[str, int], tuple[str, int]], float] = { - pair: q for pair, q, _ in floored - } - idx = 0 - while steps > 0 and idx < len(floored): - pair, q, _rem = floored[idx] - final_map[pair] = q + rounding - steps -= 1 - idx += 1 - allocs = list(final_map.items()) - try: - sum_final = sum(final_map.values()) - delta = D_c - sum_final - logger.debug( - ( - "gravity: class priority=%d rounding delta_gbps=%s " - "(exact-total - rounded-total) policy=%s" - ), - int(priority), - _fmt(delta), - getattr(gcfg, "rounding_policy", "nearest"), - ) - except Exception: # pragma: no cover - logging only - pass - logger.debug( - "gravity: class priority=%d applied rounding_gbps=%s", - int(priority), - _fmt(rounding, decimals=3), - ) +def generate_traffic_matrix( + metros: list[dict[str, Any]], + metro_settings: dict[str, dict[str, Any]], + config: TopologyConfig, +) -> dict[str, list[dict[str, Any]]]: + """Return named demand lists, without file I/O or global PRNG mutation. - class_directed_sum = 0.0 - dir_step = max(float(gcfg.rounding_gbps) / 2.0, 0.0) - for pair, v in allocs: - (m1, d1), (m2, d2) = pair - i1 = metro_idx_map[m1] - i2 = metro_idx_map[m2] - src = f"^metro{i1}/dc{d1}/.*" - dst = f"^metro{i2}/dc{d2}/.*" - # Symmetric split: half each direction; align to configured rounding step per direction - demand_each_raw = float(v) / 2.0 - if dir_step > 0.0: - demand_each = round(demand_each_raw / dir_step) * dir_step - else: - demand_each = round(demand_each_raw, 2) - if demand_each <= 0.0: + Uniform shares offered volume over ordered DC pairs through NetGraph's + selectors. Gravity weights pairs by power/distance; hose fits each DC's + margins. Both emit symmetric pairs with shared quantization semantics. + """ + traffic = config.traffic + if not traffic.enabled: + return {} + _validate_flow_policy_names(traffic.flow_policy_config) + if traffic.model not in {"uniform", "gravity", "hose"}: + raise ValueError(f"Unknown traffic model: {traffic.model}") + dcs = _inventory(metros, metro_settings, traffic) + if not dcs: + return {} + offered = sum(dc.mass for dc in dcs) * traffic.gbps_per_mw + if not math.isfinite(offered) or offered < 0: + raise ValueError("Offered traffic must be finite and non-negative") + name = traffic.matrix_name + if traffic.model == "uniform": + demands = [] + for priority, ratio in sorted(traffic.priority_ratios.items()): + if (volume := offered * ratio) <= 0: continue - dist_km = _distance_km(m1, m2) - try: - w = weights[pair] - frac = w / total_w if total_w > 0 else 0.0 - m_i = dc_mass[(m1, d1)] - m_j = dc_mass[(m2, d2)] - logger.debug( - ( - "entry: prio=%d src=%s dst=%s demand_gbps=%s " - "m_i_MW=%s m_j_MW=%s euclidean_km=%s weight=%s frac=%s " - "alpha=%s beta=%s class_total_gbps=%s rounding_gbps=%s " - "jitter_stddev=%s" - ), - int(priority), - src, - dst, - _fmt(demand_each), - _fmt(m_i, decimals=3), - _fmt(m_j, decimals=3), - _fmt(dist_km, decimals=3), - _fmt(w, decimals=6), - _fmt(frac, decimals=6), - _fmt(float(gcfg.alpha), decimals=6), - _fmt(float(gcfg.beta), decimals=6), - _fmt(D_c), - _fmt(float(gcfg.rounding_gbps), decimals=3), - _fmt(float(gcfg.jitter_stddev), decimals=6), - ) - debug_entries.append( - { - "priority": int(priority), - "source_path": src, - "sink_path": dst, - "demand_gbps": float(demand_each), - "m_i_MW": float(m_i), - "m_j_MW": float(m_j), - "euclidean_km": float(dist_km), - "weight": float(w), - "weight_fraction": float(frac), - "alpha": float(gcfg.alpha), - "beta": float(gcfg.beta), - "class_total_gbps": float(D_c), - "rounding_gbps": float(gcfg.rounding_gbps), - "jitter_stddev": float(gcfg.jitter_stddev), - } - ) - logger.debug( - ( - "entry: prio=%d src=%s dst=%s demand_gbps=%s " - "m_i_MW=%s m_j_MW=%s euclidean_km=%s weight=%s frac=%s " - "alpha=%s beta=%s class_total_gbps=%s rounding_gbps=%s " - "jitter_stddev=%s" - ), - int(priority), - dst, - src, - _fmt(demand_each), - _fmt(m_j, decimals=3), - _fmt(m_i, decimals=3), - _fmt(dist_km, decimals=3), - _fmt(w, decimals=6), - _fmt(frac, decimals=6), - _fmt(float(gcfg.alpha), decimals=6), - _fmt(float(gcfg.beta), decimals=6), - _fmt(D_c), - _fmt(float(gcfg.rounding_gbps), decimals=3), - _fmt(float(gcfg.jitter_stddev), decimals=6), - ) - debug_entries.append( - { - "priority": int(priority), - "source_path": dst, - "sink_path": src, - "demand_gbps": float(demand_each), - "m_i_MW": float(m_j), - "m_j_MW": float(m_i), - "euclidean_km": float(dist_km), - "weight": float(w), - "weight_fraction": float(frac), - "alpha": float(gcfg.alpha), - "beta": float(gcfg.beta), - "class_total_gbps": float(D_c), - "rounding_gbps": float(gcfg.rounding_gbps), - "jitter_stddev": float(gcfg.jitter_stddev), - } - ) - except Exception: # pragma: no cover - logging only - pass - demands.append( - { - "source_path": src, - "sink_path": dst, - "mode": "pairwise", - "priority": int(priority), - "demand": demand_each, - "attrs": {"euclidean_km": int(math.ceil(float(dist_km)))}, - } - ) - if int(priority) in fpc: - demands[-1]["flow_policy_config"] = fpc[int(priority)] - demands.append( - { - "source_path": dst, - "sink_path": src, - "mode": "pairwise", - "priority": int(priority), - "demand": demand_each, - "attrs": {"euclidean_km": int(math.ceil(float(dist_km)))}, - } - ) - if int(priority) in fpc: - demands[-1]["flow_policy_config"] = fpc[int(priority)] - class_directed_sum += 2.0 * demand_each - - # Post-split rounding delta relative to class exact total (using directional rounding quanta) - try: - delta_dir = D_c - class_directed_sum - logger.debug( - "gravity: class priority=%d post-split rounding delta_gbps=%s (exact-total - sum(directed))", - int(priority), - _fmt(delta_dir), - ) - except Exception: # pragma: no cover - logging only - pass - - result = {traffic_cfg.matrix_name: demands} - try: - logger.debug( - "traffic: counts dc_regions=%d undirected_pairs=%d directed_entries=%d", - len(dc_nodes), - len(weights), - len(demands), - ) - values = [float(d.get("demand", 0.0)) for d in demands] - if values: - svals = sorted(values) - n = len(svals) - total = sum(svals) - mean = total / n - median = ( - svals[n // 2] - if n % 2 == 1 - else 0.5 * (svals[n // 2 - 1] + svals[n // 2]) - ) - logger.debug( - "traffic: stats min=%s median=%s mean=%s max=%s", - _fmt(svals[0]), - _fmt(median), - _fmt(mean), - _fmt(svals[-1]), - ) - by_class: dict[int, list[float]] = {} - for d in demands: - by_class.setdefault(int(d["priority"]), []).append(float(d["demand"])) - for prio in sorted(by_class): - lst = sorted(by_class[prio]) - n_c = len(lst) - tot_c = sum(lst) - mean_c = tot_c / n_c - med_c = ( - lst[n_c // 2] - if n_c % 2 == 1 - else 0.5 * (lst[n_c // 2 - 1] + lst[n_c // 2]) - ) - logger.debug( - "class %d: entries=%d total=%s min=%s median=%s mean=%s max=%s", - prio, - n_c, - _fmt(tot_c), - _fmt(lst[0]), - _fmt(med_c), - _fmt(mean_c), - _fmt(lst[-1]), - ) - debug_dir = getattr(config, "_debug_dir", None) - if debug_dir is not None: - export: dict[str, Any] = { - "model": getattr(traffic_cfg, "model", "uniform"), - "matrix_name": getattr(traffic_cfg, "matrix_name", "default"), - "gbps_per_mw": float(traffic_cfg.gbps_per_mw), - "mw_per_dc_region": float(traffic_cfg.mw_per_dc_region), - "offered_gbps": float(offered_gbps), - "gravity": { - "alpha": float(gcfg.alpha), - "beta": float(gcfg.beta), - "min_distance_km": float(gcfg.min_distance_km), - "exclude_same_metro": bool(gcfg.exclude_same_metro), - "jitter_stddev": float(gcfg.jitter_stddev), - "rounding_gbps": float(gcfg.rounding_gbps), - "max_partners_per_dc": ( - None - if gcfg.max_partners_per_dc is None - else int(gcfg.max_partners_per_dc) - ), - }, - "dc_masses_MW": { - f"{m}/dc{d}": float(dc_mass[(m, d)]) for (m, d) in sorted(dc_mass) - }, - "metro_index_map": { - f"metro{idx}": name for name, idx in metro_idx_map.items() - }, - "weights": { - f"{a[0]}/dc{a[1]} <-> {b[0]}/dc{b[1]}": float(w) - for (a, b), w in weights.items() - }, - "entries": debug_entries, + entry = { + "source_path": "(metro[0-9]+/dc[0-9]+)", + "sink_path": "(metro[0-9]+/dc[0-9]+)", + "mode": "pairwise", + "group_mode": "group_pairwise", + "priority": priority, + "demand": volume, } - out_dir = Path(str(debug_dir)) - out_dir.mkdir(parents=True, exist_ok=True) - stem = getattr(config, "_source_stem", None) - if stem is None: - src = getattr(config, "_source_path", None) - if isinstance(src, (str, Path)): - stem = Path(src).stem - else: - stem = "scenario" - out_path = out_dir / f"{stem}_traffic_debug.json" - try: - out_path.write_text(json.dumps(export, indent=2)) - logger.debug("traffic: wrote debug JSON to %s", str(out_path)) - except Exception: - logger.debug("traffic: failed to write debug JSON to %s", str(out_path)) - except Exception: # pragma: no cover - logging only - pass - try: - pretty = { - str(traffic_cfg.matrix_name): [ - { - "source_path": d["source_path"], - "sink_path": d["sink_path"], - "mode": d["mode"], - "priority": int(d["priority"]), - "demand_gbps": _fmt(float(d["demand"])) - if isinstance(d.get("demand"), (int, float)) - else str(d.get("demand")), - } - for d in demands - ] + if priority in traffic.flow_policy_config: + entry["flow_policy_config"] = traffic.flow_policy_config[priority] + demands.append(entry) + result = {name: demands} + elif traffic.model == "gravity": + distances = _distances(dcs, metros, traffic.gravity.min_distance_km) + pairs = _gravity_pairs(dcs, distances, traffic, offered) + result = { + name: _emit_pairs( + dcs, + pairs, + distances, + traffic, + offered, + rng=random.Random(config.output.scenario_seed), + ) } - logger.debug( - "traffic_matrix pretty:\n%s", - json.dumps(pretty, indent=2, ensure_ascii=True), - ) - except Exception: # pragma: no cover - logging only - pass - - logger.debug("traffic: diagnostics end") - + else: + if offered == 0: + return {} + distances = _distances(dcs, metros, traffic.hose.min_distance_km) + result = {} + for sample in range(1, traffic.samples + 1): + rng = random.Random(config.output.scenario_seed * 1000003 + sample) + pairs = _hose_pairs(_random_hose(dcs, distances, traffic, rng)) + if traffic.hose.carve_top_k is not None: + keep = set(_top_pairs(pairs, traffic.hose.carve_top_k)) + pairs = _hose_pairs(_random_hose(dcs, distances, traffic, rng, keep)) + matrix_name = name if traffic.samples == 1 else f"{name}_{sample}" + result[matrix_name] = _emit_pairs(dcs, pairs, distances, traffic, offered) + logger.info( + "Generated %s traffic: %d DCs, %d matrices, %d demands", + traffic.model, + len(dcs), + len(result), + sum(map(len, result.values())), + ) + if logger.isEnabledFor(logging.DEBUG): + logger.debug("Traffic matrices:\n%s", json.dumps(result, indent=2)) return result diff --git a/topogen/validation/audits/dc_capacity.py b/topogen/validation/audits/dc_capacity.py new file mode 100644 index 0000000..0a5cbc8 --- /dev/null +++ b/topogen/validation/audits/dc_capacity.py @@ -0,0 +1,90 @@ +"""Check each demand set against actual external capacity of each DC site.""" + +from __future__ import annotations + +import math +import re +from collections import defaultdict + +from ngraph import Network +from ngraph.analysis.demand import expand_demands +from ngraph.model.demand.matrix import DemandSet + +_DC = re.compile(r"^(metro\d+/dc\d+)(?:/|$)") + + +def check_dc_capacity(network: Network, demand_sets: DemandSet) -> list[str]: + """Resolve selectors with NetGraph and audit separate demand sets separately. + + Count external device links, not DSL selector rules or internal DC links. + Pairwise demand endpoints have exact per-site volumes. A combine endpoint + contributes a guaranteed per-DC load only when all members belong to that DC; + placement across multiple DCs requires a routing analysis. + """ + sites = { + name: match.group(1) if (match := _DC.match(name)) else None + for name in network.nodes + } + capacity: dict[str, float] = defaultdict(float) + for link in network.links.values(): + source, target = sites[link.source], sites[link.target] + if ( + source == target + or link.disabled + or network.nodes[link.source].disabled + or network.nodes[link.target].disabled + ): + continue + for site in (source, target): + if site is not None: + capacity[site] += link.capacity + issues = [] + for name, demands in demand_sets.sets.items(): + egress: dict[str, float] = defaultdict(float) + ingress: dict[str, float] = defaultdict(float) + for index, demand in enumerate(demands): + label = f"demand set '{name}' entry {index}" + if not math.isfinite(demand.volume) or demand.volume <= 0: + issues.append(f"{label}: volume must be positive and finite") + continue + try: + expanded = expand_demands(network, [demand]) + except ValueError as exc: + issues.append(f"{label}: {exc}") + continue + endpoints: dict[str, set[str | None]] = {} + for edge in expanded.augmentations: + if edge.source in sites: + endpoints.setdefault(edge.target, set()).add(sites[edge.source]) + if edge.target in sites: + endpoints.setdefault(edge.source, set()).add(sites[edge.target]) + for flow in expanded.demands: + source = ( + {sites[flow.src_name]} + if flow.src_name in sites + else endpoints[flow.src_name] + ) + target = ( + {sites[flow.dst_name]} + if flow.dst_name in sites + else endpoints[flow.dst_name] + ) + for local, remote, totals in ( + (source, target, egress), + (target, source, ingress), + ): + if len(local) == 1: + site = next(iter(local)) + if site is not None and site not in remote: + totals[site] += flow.volume + for direction, totals in (("egress", egress), ("ingress", ingress)): + for site, volume in sorted(totals.items()): + available = capacity[site] + if volume > available and not math.isclose( + volume, available, rel_tol=1e-9, abs_tol=1e-9 + ): + issues.append( + f"dc capacity: demand set '{name}', {site} {direction} demand {volume:,.6g} " + f"exceeds adjacency capacity {available:,.6g}" + ) + return issues diff --git a/topogen/validation/audits/expand_checks.py b/topogen/validation/audits/expand_checks.py index fdd64d9..b2f7f9d 100644 --- a/topogen/validation/audits/expand_checks.py +++ b/topogen/validation/audits/expand_checks.py @@ -1,8 +1,9 @@ -"""Check that network node and link rules and blueprint links expand to entities.""" +"""Check expansion coverage without changing the network's selection rules.""" from __future__ import annotations -from copy import deepcopy as _deepcopy +from collections import Counter +from copy import deepcopy from typing import Any, Callable @@ -11,161 +12,59 @@ def check_groups_adjacency_blueprints( ng_expand: Callable[[dict[str, Any]], Any], logger_obj, ) -> list[str]: - """Ensure nodes expand to nodes and link rules expand to links. + """Check all network rules in one expansion and each blueprint in isolation. - Also validates each blueprint's link rules expand to links. + Copy the complete DSL, including node/link rules and variables. Dropping + these changes selector meaning, especially for striped inter-site links. + Blueprint checks remain independent so unused definitions are checked too. """ - issues: list[str] = [] - - # Network node expansion - groups_orig = ( - _deepcopy((dsl.get("network") or {}).get("nodes", {})) - if isinstance((dsl.get("network") or {}).get("nodes", {}), dict) - else {} + tagged = deepcopy(dsl) + network = tagged.setdefault("network", {}) + groups = network.get("nodes", {}) + adjacencies = network.get("links", []) + for idx, definition in enumerate(groups.values()): + definition.setdefault("attrs", {})["_tg_group_id"] = idx + for idx, rule in enumerate(adjacencies): + rule.setdefault("attrs", {})["_tg_adj_tag"] = f"adj_{idx}" + expanded = ng_expand(tagged) + group_counts = Counter( + node.attrs.get("_tg_group_id") for node in expanded.nodes.values() ) - gid_to_path: dict[int, str] = {} - if isinstance(groups_orig, dict) and groups_orig: - for _i, (gpath, gdef) in enumerate(groups_orig.items()): - if not isinstance(gdef, dict): - continue - attrs = gdef.get("attrs") - if not isinstance(attrs, dict): - attrs = {} - gdef["attrs"] = attrs - attrs["_tg_group_id"] = _i - gid_to_path[_i] = str(gpath) - dsl_groups = { - "blueprints": dsl.get("blueprints") or {}, - "network": { - "nodes": groups_orig, - "links": (dsl.get("network") or {}).get("links", []), - }, - } - net_groups = ng_expand(dsl_groups) - counts: dict[int, int] = {k: 0 for k in gid_to_path} - for node in net_groups.nodes.values(): - try: - gid = int(getattr(node, "attrs", {}).get("_tg_group_id", -1)) - except Exception: - gid = -1 - if gid in counts: - counts[gid] += 1 - for gid, cnt in counts.items(): - if cnt <= 0: - gpath = gid_to_path[gid] - try: - logger_obj.error( - "validation: group expands to zero nodes: %s", gpath - ) - except Exception: - pass - issues.append(f"group '{gpath}' expands to 0 nodes") + adjacency_counts = Counter( + link.attrs.get("_tg_adj_tag") for link in expanded.links.values() + ) + issues = [ + f"group '{path}' expands to 0 nodes" + for idx, path in enumerate(groups) + if not group_counts[idx] + ] + for idx, rule in enumerate(adjacencies): + if not adjacency_counts[f"adj_{idx}"]: + issues.append( + f"adjacency[{idx}] expands to 0 links " + f"(source={rule.get('source')}, target={rule.get('target')}, " + f"pattern={rule.get('pattern')})" + ) - # Network link expansion - adj_list = (dsl.get("network") or {}).get("links", []) - if isinstance(adj_list, list) and adj_list: - tagged_adj: list[dict[str, Any]] = [] - tag_attr = "_tg_adj_tag" - for idx, rule in enumerate(adj_list): - if not isinstance(rule, dict): - continue - r = _deepcopy(rule) - attrs = r.get("attrs") - if not isinstance(attrs, dict): - attrs = {} - r["attrs"] = attrs - attrs[tag_attr] = f"adj_{idx}" - tagged_adj.append(r) - dsl_adj = { - "blueprints": dsl.get("blueprints") or {}, - "network": { - "nodes": ( - _deepcopy((dsl.get("network") or {}).get("nodes", {})) - if isinstance((dsl.get("network") or {}).get("nodes", {}), dict) - else {} - ), - "links": tagged_adj, - }, + for name, definition in (dsl.get("blueprints") or {}).items(): + blueprint = deepcopy(definition) + rules = blueprint.get("links", []) + if not rules: + continue + for idx, rule in enumerate(rules): + rule.setdefault("attrs", {})["_tg_bp_adj_tag"] = f"{name}#{idx}" + probe = { + **dsl, + "blueprints": {**dsl.get("blueprints", {}), name: blueprint}, + "network": {"nodes": {"probe": {"blueprint": name}}, "links": []}, } - net_adj = ng_expand(dsl_adj) - adj_counts_by_tag: dict[str, int] = {} - for link in net_adj.links.values(): - tag = getattr(link, "attrs", {}).get(tag_attr) - if isinstance(tag, str): - adj_counts_by_tag[tag] = adj_counts_by_tag.get(tag, 0) + 1 - for idx, _rule in enumerate(adj_list): - tag = f"adj_{idx}" - if adj_counts_by_tag.get(tag, 0) <= 0: - try: - src = _rule.get("source") if isinstance(_rule, dict) else None - dst = _rule.get("target") if isinstance(_rule, dict) else None - patt = _rule.get("pattern") if isinstance(_rule, dict) else None - except Exception: - src = dst = patt = None - try: - logger_obj.error( - ( - "validation: adjacency[%d] expands to zero " - "links: source=%r target=%r pattern=%r" - ), - idx, - src, - dst, - patt, - ) - except Exception: - pass - issues.append( - f"adjacency[{idx}] expands to 0 links (source={src}, target={dst}, pattern={patt})" - ) - - # Blueprint link expansion - blueprints = dsl.get("blueprints") or {} - if isinstance(blueprints, dict) and blueprints: - for bp_name, bp_def in blueprints.items(): - if not isinstance(bp_def, dict): - continue - bp_adj = bp_def.get("links", []) - if not isinstance(bp_adj, list) or not bp_adj: - continue - bp_copy = _deepcopy(bp_def) - tag_attr = "_tg_bp_adj_tag" - expected_tags: list[str] = [] - for i, rule in enumerate(bp_copy.get("links", [])): - if not isinstance(rule, dict): - continue - attrs = rule.get("attrs") - if not isinstance(attrs, dict): - attrs = {} - rule["attrs"] = attrs - tag_val = f"{bp_name}#{i}" - expected_tags.append(tag_val) - attrs[tag_attr] = tag_val - dsl_bp = { - "blueprints": {str(bp_name): bp_copy}, - "network": {"nodes": {"__check__": {"blueprint": str(bp_name)}}}, - } - net_bp = ng_expand(dsl_bp) - seen: set[str] = set() - for link in net_bp.links.values(): - tag_val = getattr(link, "attrs", {}).get(tag_attr) - if isinstance(tag_val, str): - seen.add(tag_val) - for i, tag_val in enumerate(expected_tags): - if tag_val not in seen: - try: - logger_obj.error( - ( - "validation: blueprint '%s' " - "adjacency[%d] expands to zero links" - ), - bp_name, - i, - ) - except Exception: - pass - issues.append( - f"blueprint '{bp_name}' adjacency[{i}] expands to 0 links" - ) - + expanded = ng_expand(probe) + seen = {link.attrs.get("_tg_bp_adj_tag") for link in expanded.links.values()} + issues.extend( + f"blueprint '{name}' adjacency[{idx}] expands to 0 links" + for idx in range(len(rules)) + if f"{name}#{idx}" not in seen + ) + for issue in issues: + logger_obj.error(issue) return issues diff --git a/topogen/validation/audits/hw_capacity.py b/topogen/validation/audits/hw_capacity.py index 3237b78..241461c 100644 --- a/topogen/validation/audits/hw_capacity.py +++ b/topogen/validation/audits/hw_capacity.py @@ -1,51 +1,33 @@ -"""Hardware capacity vs total attached capacity feasibility check.""" +"""Compare total attached link capacity with each declared platform budget.""" from __future__ import annotations -from typing import Any, Dict - -from ..helpers import _node_hw_from_attrs +import math +from collections import defaultdict +from typing import Any +from ngraph import Network -def check_node_hw_capacity(net: Any, comp_lib: Dict[str, Any]) -> list[str]: - """For each node with explicit hardware, ensure attached capacity <= HW capacity. +from ..helpers import _node_hw_from_attrs - Unknown components are skipped here (already reported by node_hw_presence). - """ - issues: list[str] = [] - # Pre-compute total attached capacity per node (sum of incident link capacities) - attached: dict[str, float] = {} +def check_node_hw_capacity(net: Network, comp_lib: dict[str, Any]) -> list[str]: + attached: dict[str, float] = defaultdict(float) for link in net.links.values(): - cap = float(getattr(link, "capacity", 0.0) or 0.0) - src = str(getattr(link, "source", "")) - dst = str(getattr(link, "target", "")) - if src: - attached[src] = attached.get(src, 0.0) + cap - if dst: - attached[dst] = attached.get(dst, 0.0) + cap - + attached[link.source] += link.capacity + attached[link.target] += link.capacity + issues = [] for node in net.nodes.values(): - node_name = str(getattr(node, "name", "")) - if not node_name: - continue - comp_name, count = _node_hw_from_attrs(getattr(node, "attrs", {}) or {}) - if not comp_name or count <= 0.0: - continue - comp = comp_lib.get(comp_name) - if comp is None: - # Avoid duplicate reporting; node_hw_presence handles unknown comps. - continue - hw_cap = float(comp.get("capacity", 0.0)) * float(count) - total_attached = float(attached.get(node_name, 0.0)) - if total_attached > hw_cap + 1e-9: + component, count = _node_hw_from_attrs(node.attrs) + if component is None or component not in comp_lib: + continue # Presence audit reports unknown references. + capacity = float(comp_lib[component]["capacity"]) * count + if not math.isfinite(capacity) or capacity < 0: issues.append( - ( - "hardware capacity: node '\n" - f"{node_name}\n' total attached capacity {total_attached:,.0f} " - f"exceeds hardware capacity {hw_cap:,.0f} from component " - f"'{comp_name}' (hw_count={count:g})." - ).replace("\n", "") + f"hardware capacity: node '{node.name}' has invalid capacity {capacity} from '{component}'" + ) + elif attached[node.name] > capacity + 1e-9: + issues.append( + f"hardware capacity: node '{node.name}' total attached capacity {attached[node.name]:,.0f} exceeds hardware capacity {capacity:,.0f} from component '{component}' (hw_count={count:g})." ) - return issues diff --git a/topogen/validation/audits/ngraph_schema.py b/topogen/validation/audits/ngraph_schema.py deleted file mode 100644 index f61ea92..0000000 --- a/topogen/validation/audits/ngraph_schema.py +++ /dev/null @@ -1,31 +0,0 @@ -"""Build a complete NetGraph scenario and detect isolated nodes.""" - -from __future__ import annotations - - -def check_schema_and_isolation(scenario_yaml: str) -> list[str]: - """Return NetGraph Scenario construction errors and isolated-node issues.""" - issues: list[str] = [] - try: - from ngraph.scenario import Scenario - except Exception as exc: # pragma: no cover - environment import error - return [f"ngraph explorer: {exc}"] - try: - net = Scenario.from_yaml(scenario_yaml).network - except Exception as exc: - return [f"ngraph scenario: {exc}"] - - node_names = [str(n.name) for n in net.nodes.values()] - engaged: set[str] = set() - for link in net.links.values(): - engaged.add(str(link.source)) - engaged.add(str(link.target)) - isolated_nodes = [n for n in node_names if n not in engaged] - - if isolated_nodes: - preview = ", ".join(isolated_nodes[:10]) - issues.append( - f"{len(isolated_nodes)} isolated nodes found in built network (e.g., {preview})" - ) - - return issues diff --git a/topogen/validation/audits/node_hw_presence.py b/topogen/validation/audits/node_hw_presence.py index ef304c6..906f31b 100644 --- a/topogen/validation/audits/node_hw_presence.py +++ b/topogen/validation/audits/node_hw_presence.py @@ -1,70 +1,40 @@ -"""Checks for node hardware assignment completeness and basic validity.""" +"""Validate declared platform references and optional role assignment coverage.""" from __future__ import annotations -from typing import Any, Dict +import math +from typing import Any +from ngraph import Network -def check_node_hw_presence( - net: Any, - comps_section: Dict[str, Any], - comp_lib: Dict[str, Any], -) -> list[str]: - """If role is mapped in components.hw_component, enforce that node has hardware. - - Also flags unknown components referenced by node hardware and capacity calc errors. - """ - issues: list[str] = [] - - if not isinstance(comps_section, dict) or not comps_section: - # Mapping not provided in scenario → skip this audit entirely. - return issues +from ..helpers import _node_hw_from_attrs - # Record roles present in network and compare with declared mapping keys - roles_present: set[str] = set() - try: - for node in net.nodes.values(): - r = str(getattr(node, "attrs", {}).get("role", "")).strip() - if r: - roles_present.add(r) - except Exception: - pass - declared_roles: set[str] = set(map(str, comps_section.keys())) - for role in sorted(roles_present - declared_roles): - issues.append( - f"node hardware: components.hw_component missing mapping for role '{role}'" - ) - for node in net.nodes.values(): - nname = str(getattr(node, "name", "")) - attrs = getattr(node, "attrs", {}) or {} - role = str(attrs.get("role", "")).strip() - if not role: - continue - # Explicit exemption: non-string/empty mapping value means 'do not enforce' - mapping_val = comps_section.get(role) - if not isinstance(mapping_val, str) or not mapping_val.strip(): - continue - hw = attrs.get("hardware") - if not isinstance(hw, dict) or not str(hw.get("component", "")).strip(): +def check_node_hw_presence( + net: Network, assignments: dict[str, str], comp_lib: dict[str, Any] +) -> list[str]: + issues = [] + roles = {node.attrs.get("role", "") for node in net.nodes.values()} + if assignments: + for role in sorted(roles - assignments.keys() - {""}): issues.append( - f"node hardware: role '{role}' mapped in components.hw_component but node '{nname}' has no hardware assignment" + f"node hardware: components.hw_component missing mapping for role '{role}'" ) - else: - try: - comp_name = str(hw.get("component", "")).strip() - count = float(hw.get("count", 1.0)) - comp = comp_lib.get(comp_name) - if comp is None: - issues.append( - f"node hardware: node '{nname}' references unknown component '{comp_name}'" - ) - else: - # Surface component capacity calculation errors. - _ = float(comp.get("capacity", 0.0)) * float(count) - except Exception as e2: # pragma: no cover + for node in net.nodes.values(): + role = node.attrs.get("role", "") + component, count = _node_hw_from_attrs(node.attrs) + if component is None: + if assignments.get(role): issues.append( - f"node hardware: capacity calculation error for node '{nname}': {e2}" + f"node hardware: role '{role}' mapped in components.hw_component but node '{node.name}' has no hardware assignment" ) - + continue + if component not in comp_lib: + issues.append( + f"node hardware: node '{node.name}' references unknown component '{component}'" + ) + if not math.isfinite(count) or count <= 0: + issues.append( + f"node hardware: node '{node.name}' has invalid hardware count {count}" + ) return issues diff --git a/topogen/validation/audits/node_role.py b/topogen/validation/audits/node_role.py index daac39f..a6b2c51 100644 --- a/topogen/validation/audits/node_role.py +++ b/topogen/validation/audits/node_role.py @@ -1,32 +1,15 @@ -"""Node role presence checks.""" +"""Require a nonempty string role on every device.""" -from __future__ import annotations +from ngraph import Network -from typing import Any - -def check_node_roles(net: Any) -> list[str]: - """Ensure each node has a non-empty role attribute.""" - issues: list[str] = [] - missing_role_count = 0 - missing_role_samples: list[str] = [] - for node in net.nodes.values(): - nname = str(getattr(node, "name", "")) - role = str(getattr(node, "attrs", {}).get("role", "")).strip() - if not role: - missing_role_count += 1 - if ( - nname - and len(missing_role_samples) < 5 - and nname not in missing_role_samples - ): - missing_role_samples.append(nname) - if missing_role_count: - if missing_role_samples: - examples = ", ".join("'" + n + "'" for n in missing_role_samples) - issues.append( - f"node roles: {missing_role_count} node(s) missing role (e.g., {examples})" - ) - else: - issues.append("node roles: nodes missing role detected") - return issues +def check_node_roles(net: Network) -> list[str]: + missing = [ + node.name + for node in net.nodes.values() + if not isinstance(node.attrs.get("role"), str) or not node.attrs["role"].strip() + ] + if not missing: + return [] + examples = ", ".join(repr(name) for name in missing[:5]) + return [f"node roles: {len(missing)} node(s) missing role (e.g., {examples})"] diff --git a/topogen/validation/audits/optics_checks.py b/topogen/validation/audits/optics_checks.py index 11f34cf..e11e35e 100644 --- a/topogen/validation/audits/optics_checks.py +++ b/topogen/validation/audits/optics_checks.py @@ -1,304 +1,64 @@ -"""Link optics mapping and blueprint-provided hardware checks.""" +"""Audit explicit link hardware and optional expected role-pair coverage.""" from __future__ import annotations -from collections import defaultdict as _dd -from typing import Any, Dict +import math +from collections import Counter +from typing import Any +from ngraph import Network -def _role_of(net: Any, node_name: str) -> str: - try: - return str(getattr(net.nodes[node_name], "attrs", {}).get("role", "")) - except Exception: - return "" - - -def _build_optics_map( - optics_cfg: Dict[str, Any], -) -> tuple[ - Dict[tuple[str, str], str], - Dict[tuple[str, str], set[str]], - Dict[tuple[str, str], set[str]], -]: - """Return (optics_map, optics_values, mapping_conflicts).""" - optics_map: dict[tuple[str, str], str] = {} - optics_values: dict[tuple[str, str], set[str]] = {} - mapping_conflicts: dict[tuple[str, str], set[str]] = {} - if isinstance(optics_cfg, dict) and optics_cfg: - unordered_by_pair: dict[frozenset[str], list[tuple[str, str, str]]] = {} - for key, val in optics_cfg.items(): - if not isinstance(val, str): - continue - k = str(key) - if "-" in k and "|" not in k: - a, b = [p.strip() for p in k.split("-", 1)] - if a and b: - key2 = (a, b) - ov = str(val) - if key2 not in optics_map: - optics_map[key2] = ov - if key2 not in optics_values: - optics_values[key2] = set() - optics_values[key2].add(ov) - elif "|" in k: - a, b = [p.strip() for p in k.split("|", 1)] - if a and b: - sig = frozenset((a, b)) - unordered_by_pair.setdefault(sig, []).append((a, b, str(val))) - # Resolve unordered declarations - for _sig, entries in unordered_by_pair.items(): - if len(entries) == 1: - a, b, ov = entries[0] - for key2 in ((a, b), (b, a)): - if key2 not in optics_map: - optics_map[key2] = ov - if key2 not in optics_values: - optics_values[key2] = set() - optics_values[key2].add(ov) - else: - for a, b, ov in entries: - key2 = (a, b) - if key2 not in optics_map: - optics_map[key2] = ov - if key2 not in optics_values: - optics_values[key2] = set() - optics_values[key2].add(ov) - - for key2, values in optics_values.items(): - if len(values) > 1: - mapping_conflicts[key2] = values - - return optics_map, optics_values, mapping_conflicts +from topogen.roles import role_optics def check_link_optics( - net: Any, d: Dict[str, Any], comp_lib: Dict[str, Any] + net: Network, assignments: dict[str, str], comp_lib: dict[str, Any] ) -> list[str]: - """Aggregate optics mapping, blueprint hardware presence/capacity checks.""" - issues: list[str] = [] - - optics_cfg = (d.get("components") or {}).get("optics", {}) - optics_map, _optics_values, mapping_conflicts = _build_optics_map(optics_cfg) - - if not optics_map and not mapping_conflicts: - return issues - - miss_src: dict[tuple[str, str], int] = _dd(int) - miss_tgt: dict[tuple[str, str], int] = _dd(int) - miss_src_samples: dict[tuple[str, str], list[str]] = _dd(list) - miss_tgt_samples: dict[tuple[str, str], list[str]] = _dd(list) - # Missing mapping when hardware absent - miss_map_src: dict[tuple[str, str], int] = _dd(int) - miss_map_tgt: dict[tuple[str, str], int] = _dd(int) - miss_map_src_samples: dict[tuple[str, str], list[str]] = _dd(list) - miss_map_tgt_samples: dict[tuple[str, str], list[str]] = _dd(list) - # Aggregation for blueprint-provided hardware checks - bhw_missing_src: dict[tuple[str, str], int] = _dd(int) - bhw_missing_tgt: dict[tuple[str, str], int] = _dd(int) - bhw_cap_short: dict[tuple[str, str], int] = _dd(int) - bhw_unknown_src: dict[tuple[str, str], int] = _dd(int) - bhw_unknown_tgt: dict[tuple[str, str], int] = _dd(int) - bhw_calc_err: dict[tuple[str, str], int] = _dd(int) - bhw_samples: dict[tuple[str, str], list[str]] = _dd(list) - + try: + expected = role_optics(assignments) + except ValueError as exc: + return [f"optics mapping: {exc}"] + failures: Counter[str] = Counter() for link in net.links.values(): - src = str(getattr(link, "source", "")) - dst = str(getattr(link, "target", "")) - if not src or not dst: - continue - rs = _role_of(net, src) - rd = _role_of(net, dst) - if not rs or not rd: + hardware = link.attrs.get("hardware") + if hardware is None and not expected: continue - attrs_link = getattr(link, "attrs", {}) or {} - aid = str(attrs_link.get("adjacency_id", "")).strip() - if not aid: - aid = str(attrs_link.get("link_type", "")).strip() - hw = attrs_link.get("hardware") - # If hardware present on the link, validate presence and capacity - if isinstance(hw, dict): - src_hw = hw.get("source") if isinstance(hw.get("source"), dict) else None - tgt_hw = hw.get("target") if isinstance(hw.get("target"), dict) else None - if not src_hw: - key = (rs, rd) - bhw_missing_src[key] += 1 - if aid and len(bhw_samples[key]) < 3 and aid not in bhw_samples[key]: - bhw_samples[key].append(aid) + for end, node_name, other_name in ( + ("source", link.source, link.target), + ("target", link.target, link.source), + ): + role = net.nodes[node_name].attrs.get("role", "") + other = net.nodes[other_name].attrs.get("role", "") + label = f"for roles ({role},{other})" + if hardware is None: + issue = ( + f"optics: missing hardware required by mapping on {end} end" + if (role, other) in expected + else f"optics mapping: missing {end}-end mapping" + ) + failures[f"{issue} {label}"] += 1 continue - if not tgt_hw: - key = (rd, rs) - bhw_missing_tgt[key] += 1 - if aid and len(bhw_samples[key]) < 3 and aid not in bhw_samples[key]: - bhw_samples[key].append(aid) + assignment = hardware.get(end) + if not assignment: + failures[ + f"optics (blueprint): missing hardware on {end} end {label}" + ] += 1 continue - try: - comp_src = str(src_hw.get("component", "")).strip() - comp_tgt = str(tgt_hw.get("component", "")).strip() - c_src = comp_lib.get(comp_src) if comp_src else None - c_tgt = comp_lib.get(comp_tgt) if comp_tgt else None - if comp_src and not c_src: - key = (rs, rd) - bhw_unknown_src[key] += 1 - if ( - aid - and len(bhw_samples[key]) < 3 - and aid not in bhw_samples[key] - ): - bhw_samples[key].append(aid) - if comp_tgt and not c_tgt: - key = (rd, rs) - bhw_unknown_tgt[key] += 1 - if ( - aid - and len(bhw_samples[key]) < 3 - and aid not in bhw_samples[key] - ): - bhw_samples[key].append(aid) - cnt_src = float(src_hw.get("count", 1.0)) - cnt_tgt = float(tgt_hw.get("count", 1.0)) - av_src = (float(c_src.get("capacity", 0.0)) * cnt_src) if c_src else 0.0 - av_tgt = (float(c_tgt.get("capacity", 0.0)) * cnt_tgt) if c_tgt else 0.0 - need = float(getattr(link, "capacity", 0.0) or 0.0) - if av_src + 1e-9 < need or av_tgt + 1e-9 < need: - key = (rs, rd) - bhw_cap_short[key] += 1 - if ( - aid - and len(bhw_samples[key]) < 3 - and aid not in bhw_samples[key] - ): - bhw_samples[key].append(aid) - except Exception: - key = (rs, rd) - bhw_calc_err[key] += 1 - if aid and len(bhw_samples[key]) < 3 and aid not in bhw_samples[key]: - bhw_samples[key].append(aid) - continue - - # Enforce mapping only when hardware is not specified on the link - req_src = optics_map.get((rs, rd)) - req_tgt = optics_map.get((rd, rs)) - if isinstance(req_src, str): - key = (rs, rd) - miss_src[key] += 1 - if ( - aid - and len(miss_src_samples[key]) < 3 - and aid not in miss_src_samples[key] - ): - miss_src_samples[key].append(aid) - else: - key = (rs, rd) - miss_map_src[key] += 1 - if ( - aid - and len(miss_map_src_samples[key]) < 3 - and aid not in miss_map_src_samples[key] - ): - miss_map_src_samples[key].append(aid) - if isinstance(req_tgt, str): - key = (rd, rs) - miss_tgt[key] += 1 - if ( - aid - and len(miss_tgt_samples[key]) < 3 - and aid not in miss_tgt_samples[key] - ): - miss_tgt_samples[key].append(aid) - else: - key = (rd, rs) - miss_map_tgt[key] += 1 + name = assignment["component"] + if name not in comp_lib: + failures[ + f"optics (blueprint): unknown component on {end} end {label}: {name}" + ] += 1 + continue + count = float(assignment.get("count", 1)) + capacity = float(comp_lib[name]["capacity"]) * count if ( - aid - and len(miss_map_tgt_samples[key]) < 3 - and aid not in miss_map_tgt_samples[key] + not math.isfinite(count) + or count <= 0 + or not math.isfinite(capacity) + or capacity < link.capacity ): - miss_map_tgt_samples[key].append(aid) - - for (a, b), vals in sorted(mapping_conflicts.items()): - vstr = ", ".join(sorted(vals)) - issues.append( - f"optics mapping: conflicting values for roles ({a},{b}) -> {{{vstr}}}" - ) - for (a, b), cnt in sorted(bhw_missing_src.items()): - samples = bhw_samples.get((a, b), []) - prefix = f"optics (blueprint): missing hardware on source end for roles ({a},{b}) - {cnt} links" - if samples: - sample_str = ", ".join("'" + s + "'" for s in samples) - issues.append(f"{prefix} (e.g., adj={sample_str})") - else: - issues.append(prefix) - for (a, b), cnt in sorted(bhw_missing_tgt.items()): - samples = bhw_samples.get((a, b), []) - prefix = f"optics (blueprint): missing hardware on target end for roles ({a},{b}) - {cnt} links" - if samples: - sample_str = ", ".join("'" + s + "'" for s in samples) - issues.append(f"{prefix} (e.g., adj={sample_str})") - else: - issues.append(prefix) - for (a, b), cnt in sorted(bhw_unknown_src.items()): - samples = bhw_samples.get((a, b), []) - prefix = f"optics (blueprint): unknown component on source end for roles ({a},{b}) - {cnt} links" - if samples: - sample_str = ", ".join("'" + s + "'" for s in samples) - issues.append(f"{prefix} (e.g., adj={sample_str})") - else: - issues.append(prefix) - for (a, b), cnt in sorted(bhw_unknown_tgt.items()): - samples = bhw_samples.get((a, b), []) - prefix = f"optics (blueprint): unknown component on target end for roles ({a},{b}) - {cnt} links" - if samples: - sample_str = ", ".join("'" + s + "'" for s in samples) - issues.append(f"{prefix} (e.g., adj={sample_str})") - else: - issues.append(prefix) - for (a, b), cnt in sorted(bhw_cap_short.items()): - samples = bhw_samples.get((a, b), []) - prefix = f"optics (blueprint): hardware capacity shortfall for roles ({a},{b}) - {cnt} links" - if samples: - sample_str = ", ".join("'" + s + "'" for s in samples) - issues.append(f"{prefix} (e.g., adj={sample_str})") - else: - issues.append(prefix) - for (a, b), cnt in sorted(bhw_calc_err.items()): - samples = bhw_samples.get((a, b), []) - prefix = f"optics (blueprint): capacity calculation error for roles ({a},{b}) - {cnt} links" - if samples: - sample_str = ", ".join("'" + s + "'" for s in samples) - issues.append(f"{prefix} (e.g., adj={sample_str})") - else: - issues.append(prefix) - for (a, b), cnt in sorted(miss_map_src.items()): - samples = miss_map_src_samples.get((a, b), []) - prefix = f"optics mapping: missing source-end mapping for roles ({a},{b}) - {cnt} links" - if samples: - sample_str = ", ".join("'" + s + "'" for s in samples) - issues.append(f"{prefix} (e.g., adj={sample_str})") - else: - issues.append(prefix) - for (a, b), cnt in sorted(miss_map_tgt.items()): - samples = miss_map_tgt_samples.get((a, b), []) - prefix = f"optics mapping: missing target-end mapping for roles ({a},{b}) - {cnt} links" - if samples: - sample_str = ", ".join("'" + s + "'" for s in samples) - issues.append(f"{prefix} (e.g., adj={sample_str})") - else: - issues.append(prefix) - for (a, b), cnt in sorted(miss_src.items()): - samples = miss_src_samples.get((a, b), []) - prefix = f"optics: missing hardware required by mapping on source end for roles ({a},{b}) - {cnt} links" - if samples: - sample_str = ", ".join(f"'{s}'" for s in samples) - issues.append(f"{prefix} (e.g., adj={sample_str})") - else: - issues.append(prefix) - for (a, b), cnt in sorted(miss_tgt.items()): - samples = miss_tgt_samples.get((a, b), []) - prefix = f"optics: missing hardware required by mapping on target end for roles ({a},{b}) - {cnt} links" - if samples: - sample_str = ", ".join(f"'{s}'" for s in samples) - issues.append(f"{prefix} (e.g., adj={sample_str})") - else: - issues.append(prefix) - - return issues + failures[ + f"optics (blueprint): hardware capacity shortfall {label}" + ] += 1 + return [f"{issue} - {count} link ends" for issue, count in sorted(failures.items())] diff --git a/topogen/validation/audits/pipeline.py b/topogen/validation/audits/pipeline.py index dbe528e..2a81cfe 100644 --- a/topogen/validation/audits/pipeline.py +++ b/topogen/validation/audits/pipeline.py @@ -1,14 +1,19 @@ -"""Run NetGraph construction, topology, and hardware audits.""" +"""Construct the complete NetGraph scenario once and audit its network.""" from __future__ import annotations +from typing import Any + import yaml +from jsonschema import ValidationError +from ngraph.dsl.blueprints.expand import expand_network_dsl +from ngraph.scenario import Scenario from topogen.log_config import get_logger +from .dc_capacity import check_dc_capacity from .expand_checks import check_groups_adjacency_blueprints from .hw_capacity import check_node_hw_capacity -from .ngraph_schema import check_schema_and_isolation from .node_hw_presence import check_node_hw_presence from .node_role import check_node_roles from .optics_checks import check_link_optics @@ -20,83 +25,48 @@ def run_ngraph_audits( scenario_yaml: str, *, - hw_component_map: dict[str, object] | None = None, - optics_map: dict[str, object] | None = None, + hw_component_map: dict[str, str] | None = None, + optics_map: dict[str, str] | None = None, ) -> list[str]: - """Validate Scenario construction and rule expansion, then audit the network. - - Check node roles, hardware assignments, optics, attached capacity, and ports. - Return all collected issue strings. - """ - issues: list[str] = [] - - try: - issues.extend(check_schema_and_isolation(scenario_yaml)) - except Exception as e: - issues.append(f"ngraph schema: {e}") - + """Validate schema/workflows/failures, then audit the constructed network.""" try: - from ngraph.dsl.blueprints.expand import ( # type: ignore[import-untyped] - expand_network_dsl as _ng_expand, - ) - - from topogen.components_lib import ( # type: ignore[import-untyped] - get_builtin_components as _get_components_lib, + scenario = Scenario.from_yaml(scenario_yaml) + except ValidationError as exc: + return [f"ngraph schema: {exc.message}"] + except Exception as exc: + return [f"ngraph scenario: {exc}"] + data = yaml.safe_load(scenario_yaml) + net = scenario.network + issues: list[str] = [] + engaged = { + endpoint + for link in net.links.values() + for endpoint in (link.source, link.target) + } + isolated = [node for node in net.nodes if node not in engaged] + if isolated: + issues.append( + f"{len(isolated)} isolated nodes found in built network " + f"(e.g., {', '.join(isolated[:10])})" ) - - d = yaml.safe_load(scenario_yaml) or {} - dsl = { - "blueprints": (d.get("blueprints") or {}), - "network": (d.get("network") or {}), - } - - try: - issues.extend(check_groups_adjacency_blueprints(dsl, _ng_expand, logger)) - except Exception as e: - issues.append(f"adjacency/group expansion audit failed: {e}") - - net = _ng_expand(dsl) - comp_lib = _get_components_lib() - - try: - issues.extend(check_node_roles(net)) - except Exception as e: - issues.append(f"node roles audit failed: {e}") - - try: - # Prefer explicit mapping provided by caller (from config) - if isinstance(hw_component_map, dict) and hw_component_map: - comps_section = hw_component_map - else: - comps_section = (d.get("components") or {}).get("hw_component", {}) - issues.extend(check_node_hw_presence(net, comps_section, comp_lib)) - except Exception as e: - issues.append(f"node hardware audit failed: {e}") - + components = data.get("components", {}) + role_map = {} if hw_component_map is None else hw_component_map + optic_assignments = {} if optics_map is None else optics_map + checks: list[tuple[str, Any]] = [ + ("DC capacity", lambda: check_dc_capacity(net, scenario.demand_set)), + ( + "adjacency/group expansion", + lambda: check_groups_adjacency_blueprints(data, expand_network_dsl, logger), + ), + ("node roles", lambda: check_node_roles(net)), + ("node hardware", lambda: check_node_hw_presence(net, role_map, components)), + ("link optics", lambda: check_link_optics(net, optic_assignments, components)), + ("hardware capacity", lambda: check_node_hw_capacity(net, components)), + ("port budget", lambda: audit_port_budget(net, components)), + ] + for label, check in checks: try: - # When override provided, inject into a shallow copy for optics checks - if isinstance(optics_map, dict) and optics_map: - d2 = dict(d) - comps = dict(d2.get("components") or {}) - comps["optics"] = optics_map - d2["components"] = comps - issues.extend(check_link_optics(net, d2, comp_lib)) - else: - issues.extend(check_link_optics(net, d, comp_lib)) - except Exception as e: - issues.append(f"link optics audit failed: {e}") - - try: - issues.extend(check_node_hw_capacity(net, comp_lib)) - except Exception as e: - issues.append(f"hardware capacity audit failed: {e}") - - try: - issues.extend(audit_port_budget(net, d, comp_lib)) - except Exception as e: - issues.append(f"port budget audit failed: {e}") - - except Exception as e: - issues.append(f"ngraph explorer: {e}") - + issues.extend(check()) + except Exception as exc: + issues.append(f"{label} audit failed: {exc}") return issues diff --git a/topogen/validation/audits/port_budget.py b/topogen/validation/audits/port_budget.py index 37c92ae..a403cb4 100644 --- a/topogen/validation/audits/port_budget.py +++ b/topogen/validation/audits/port_budget.py @@ -1,149 +1,33 @@ -"""Compare per-link optic requirements with node platform port budgets.""" +"""Audit declared optic module counts against declared platform port budgets.""" from __future__ import annotations +from collections import defaultdict from typing import Any -from topogen.log_config import get_logger +from ngraph import Network from ..helpers import _node_hw_from_attrs -logger = get_logger(__name__) - -def audit_port_budget( - net: Any, scenario_dict: dict[str, Any], comp_lib: dict[str, Any] -) -> list[str]: - """Audit node platform port budgets against per-link optics. - - For each link end, compute modules = ceil(link_capacity / optic.capacity) - and multiply by optic.ports to get required platform ports. Sum per node, - compare to available ports on node platform (component.ports × count). - """ - issues: list[str] = [] - try: - from math import ceil as _ceil - - def _role_of(node_name: str) -> str: - try: - return str(getattr(net.nodes[node_name], "attrs", {}).get("role", "")) - except Exception: - return "" - - # Build optics mapping from components.optics config - optics_cfg = (scenario_dict.get("components") or {}).get("optics", {}) - optics_map: dict[tuple[str, str], str] = {} - if isinstance(optics_cfg, dict) and optics_cfg: - for key, val in optics_cfg.items(): - if not isinstance(val, str): - continue - k = str(key) - if "|" in k: - a, b = [p.strip() for p in k.split("|", 1)] - if a and b: - optics_map[(a, b)] = str(val) - optics_map[(b, a)] = str(val) - elif "-" in k: - a, b = [p.strip() for p in k.split("-", 1)] - if a and b: - optics_map[(a, b)] = str(val) - - # Available platform ports per node - available_ports: dict[str, int] = {} - for node in net.nodes.values(): - node_name = str(getattr(node, "name", "")) - if not node_name: - continue - comp_name, count = _node_hw_from_attrs(getattr(node, "attrs", {}) or {}) - if not comp_name or count <= 0.0: - continue - comp = comp_lib.get(comp_name) - if comp is None: - continue - try: - ports_per_platform = int(comp.get("ports", 0)) - except Exception: - ports_per_platform = 0 - if ports_per_platform <= 0: - continue - total_ports = int(ports_per_platform * float(count)) - if total_ports > 0: - available_ports[node_name] = total_ports - - if not available_ports: - return issues - - # Sum required ports per node across incident links - required_ports: dict[str, int] = {} - for link in net.links.values(): - cap = float(getattr(link, "capacity", 0.0) or 0.0) - if cap <= 0.0: - continue - src = str(getattr(link, "source", "")) - dst = str(getattr(link, "target", "")) - if not src or not dst: - continue - - src_optic: str | None = None - tgt_optic: str | None = None - hw = getattr(link, "attrs", {}).get("hardware") - if isinstance(hw, dict): - s_hw = hw.get("source") if isinstance(hw.get("source"), dict) else None - t_hw = hw.get("target") if isinstance(hw.get("target"), dict) else None - if s_hw: - src_optic = str(s_hw.get("component", "")).strip() or None - if t_hw: - tgt_optic = str(t_hw.get("component", "")).strip() or None - - if src_optic is None or tgt_optic is None: - rs = _role_of(src) - rd = _role_of(dst) - if not src_optic: - src_optic = optics_map.get((rs, rd)) - if not tgt_optic: - tgt_optic = optics_map.get((rd, rs)) - - def _accumulate( - node_name: str, optic_name: str | None, link_cap: float - ) -> None: - if not optic_name: - return - comp = comp_lib.get(optic_name) - if comp is None: - return - try: - module_capacity = float(comp.get("capacity", 0.0)) - ports_per_optic = int(comp.get("ports", 1)) - except Exception: - module_capacity = 0.0 - ports_per_optic = 1 - if module_capacity <= 0.0: - return - modules_needed = int(_ceil(link_cap / module_capacity)) - ports_needed = modules_needed * max(ports_per_optic, 1) - if ports_needed <= 0: - return - required_ports[node_name] = ( - required_ports.get(node_name, 0) + ports_needed - ) - - _accumulate(src, src_optic, cap) - _accumulate(dst, tgt_optic, cap) - - for node_name, need_ports in sorted(required_ports.items()): - avail_ports = int(available_ports.get(node_name, 0)) - if avail_ports <= 0: - continue - if need_ports > avail_ports: - attrs = getattr(net.nodes[node_name], "attrs", {}) or {} - comp_name, count = _node_hw_from_attrs(attrs) - issues.append( - ( - "hardware ports: node '\n" - f"{node_name}\n' requires {need_ports} ports for link optics but only " - f"{avail_ports} ports are available on '{comp_name}' (count={count:g})." - ).replace("\n", "") - ) - except Exception as e: # pragma: no cover - issues.append(f"port budget audit failed: {e}") +def audit_port_budget(net: Network, comp_lib: dict[str, Any]) -> list[str]: + required: dict[str, float] = defaultdict(float) + for link in net.links.values(): + hardware = link.attrs.get("hardware", {}) + for end, node in (("source", link.source), ("target", link.target)): + optic = hardware.get(end) + if optic is None or optic["component"] not in comp_lib: + continue # Optics audit reports incomplete/unknown assignments. + component = comp_lib[optic["component"]] + required[node] += float(optic.get("count", 1)) * component["ports"] + issues = [] + for node, needed in sorted(required.items()): + platform, count = _node_hw_from_attrs(net.nodes[node].attrs) + if platform is None or platform not in comp_lib: + continue # Hardware presence audit reports this separately. + available = comp_lib[platform]["ports"] * count + if needed > available: + issues.append( + f"hardware ports: node '{node}' requires {needed:g} ports for link optics but only {available:g} ports are available on '{platform}' (count={count:g})." + ) return issues diff --git a/topogen/validation/helpers.py b/topogen/validation/helpers.py index c94becb..3301f42 100644 --- a/topogen/validation/helpers.py +++ b/topogen/validation/helpers.py @@ -7,31 +7,18 @@ def _build_ig_coord_map(ig_json: dict[str, Any]) -> dict[str, tuple[float, float]]: """Map metro names to projected (x, y) coordinates from integrated graph JSON.""" - mapping: dict[str, tuple[float, float]] = {} - for node in ig_json.get("nodes", []) or []: - try: - node_type = node.get("node_type") - except AttributeError: - continue - if node_type not in ("metro", "metro+highway"): - continue - name = str(node.get("name", "")).strip() - if not name: - continue - try: - x = float(node.get("x", 0.0)) - y = float(node.get("y", 0.0)) - except Exception: - continue - mapping[name] = (x, y) - return mapping + return { + node["name"]: (float(node["x"]), float(node["y"])) + for node in ig_json["nodes"] + if node["node_type"] == "metro" + } def _float_or_nan(value: Any) -> float: """Convert value to float or return NaN if conversion fails.""" try: - return float(value) # type: ignore[arg-type] - except Exception: + return float(value) + except (TypeError, ValueError, OverflowError): return float("nan") @@ -46,9 +33,6 @@ def _node_hw_from_attrs(node_attrs: dict[str, object]) -> tuple[str | None, floa if isinstance(hw, dict): comp_name = str(hw.get("component", "")).strip() if comp_name: - try: - count = float(hw.get("count", 1.0)) - except Exception: - count = 1.0 + count = float(hw.get("count", 1.0)) return comp_name, count return None, 0.0 diff --git a/topogen/validation/scenario_dict.py b/topogen/validation/scenario_dict.py index c77c3b8..be62b87 100644 --- a/topogen/validation/scenario_dict.py +++ b/topogen/validation/scenario_dict.py @@ -5,12 +5,8 @@ import re from typing import Any -from topogen.log_config import get_logger - from .helpers import _float_or_nan -logger = get_logger(__name__) - def validate_scenario_dict( data: dict[str, Any], ig_coords: dict[str, tuple[float, float]] | None = None @@ -28,9 +24,9 @@ def validate_scenario_dict( issues: list[str] = [] network = (data or {}).get("network", {}) - groups: dict[str, Any] = ( - network.get("nodes", {}) if isinstance(network, dict) else {} - ) + if not isinstance(network, dict): + return ["network must be a mapping"] + groups: dict[str, Any] = network.get("nodes", {}) if not isinstance(groups, dict) or not groups: issues.append("No network.nodes found in scenario") groups = {} @@ -40,6 +36,9 @@ def validate_scenario_dict( dc_groups: dict[str, dict[str, Any]] = {} for name, entry in groups.items(): + if not isinstance(name, str) or not isinstance(entry, dict): + issues.append("network.nodes must map string paths to node definitions") + continue m = metro_pattern.match(str(name)) if not m: continue @@ -56,51 +55,53 @@ def validate_scenario_dict( if not pop: issues.append(f"metro{idx}: missing pop group") continue - if dc is None: + pa = pop.get("attrs", {}) + da = dc.get("attrs", {}) if dc is not None else {} + if not isinstance(pa, dict) or not isinstance(da, dict): + issues.append(f"metro{idx}: site attrs must be mappings") continue - pa = (pop.get("attrs", {}) if isinstance(pop, dict) else {}) or {} - da = (dc.get("attrs", {}) if isinstance(dc, dict) else {}) or {} - + if ig_coords is not None: + for kind, attrs in (("pop", pa), ("dc", da)): + if kind == "dc" and dc is None: + continue + name = str(attrs.get("metro_name", "")) + if name not in ig_coords: + issues.append( + f"metro{idx}: {kind} metro '{name}' missing from integrated graph" + ) + elif ( + _float_or_nan(attrs.get("location_x")), + _float_or_nan(attrs.get("location_y")), + ) != ig_coords[name]: + issues.append( + f"metro{idx}: {kind} location differs from integrated graph for {name}" + ) + if dc is None: + continue for key in ("metro_name", "metro_name_orig", "metro_id"): if pa.get(key) != da.get(key): issues.append( f"metro{idx}: attribute mismatch for {key}: pop={pa.get(key)} dc={da.get(key)}" ) - for key in ("location_x", "location_y"): - pv = _float_or_nan(pa.get(key)) - dv = _float_or_nan(da.get(key)) - if not (pv == dv): + if _float_or_nan(pa.get(key)) != _float_or_nan(da.get(key)): issues.append( f"metro{idx}: {key} mismatch: pop={pa.get(key)} dc={da.get(key)}" ) - - if ig_coords: - name = str(pa.get("metro_name", "")).strip() - if name in ig_coords: - ix, iy = ig_coords[name] - px = _float_or_nan(pa.get("location_x", 0.0)) - py = _float_or_nan(pa.get("location_y", 0.0)) - if not (px == ix and py == iy): - issues.append( - f"metro{idx}: pop location differs from integrated graph for {name}" - ) - if dc is not None: - dx = _float_or_nan(da.get("location_x", 0.0)) - dy = _float_or_nan(da.get("location_y", 0.0)) - if not (dx == ix and dy == iy): - issues.append( - f"metro{idx}: dc location differs from integrated graph for {name}" - ) - - if dc is not None: - for key in ("mw_per_dc_region", "gbps_per_mw"): - if key not in da: - issues.append(f"metro{idx}: dc attrs missing required '{key}'") + for key in ("mw_per_dc_region", "gbps_per_mw"): + if key not in da: + issues.append(f"metro{idx}: dc attrs missing required '{key}'") failure_set = (data or {}).get("failures") or {} traffic_set = (data or {}).get("demands") or {} + for label, section in (("failures", failure_set), ("demands", traffic_set)): + if not isinstance(section, dict): + issues.append(f"{label} must be a mapping") + if not isinstance(failure_set, dict): + failure_set = {} + if not isinstance(traffic_set, dict): + traffic_set = {} workflows = (data or {}).get("workflow") or [] if isinstance(workflows, list): for step in workflows: @@ -108,28 +109,20 @@ def validate_scenario_dict( continue step_name = str(step.get("name") or step.get("type") or "step").strip() policy_ref = step.get("failure_policy") - if policy_ref and policy_ref not in failure_set: + if policy_ref and ( + not isinstance(policy_ref, str) or policy_ref not in failure_set + ): issues.append( f"workflow step '{step_name}' references missing failure_policy '{policy_ref}'" ) matrix_ref = step.get("demand_set") - if matrix_ref and matrix_ref not in traffic_set: + if matrix_ref and ( + not isinstance(matrix_ref, str) or matrix_ref not in traffic_set + ): issues.append( f"workflow step '{step_name}' references missing traffic matrix '{matrix_ref}'" ) - try: - tcfg = (data or {}).get("traffic", {}) or {} - t_enabled = bool(tcfg.get("enabled", False)) - if t_enabled and isinstance(traffic_set, dict): - for mname, entries in traffic_set.items(): - if isinstance(entries, list) and len(entries) == 0: - issues.append( - f"traffic matrix '{mname}' is empty despite enabled traffic" - ) - except Exception: - pass - adjacency = network.get("links", []) or [] if not adjacency: key_re = re.compile(r"^/?(metro\d+)/(pop|dc)\b") @@ -137,101 +130,4 @@ def validate_scenario_dict( if key_re.match(str(gkey)): issues.append(f"{gkey} appears isolated (no adjacency references)") - # Compare DC ingress and egress with attached capacity. - try: - - def _endpoint_path(ep: Any) -> str: - if isinstance(ep, str): - return ep - if isinstance(ep, dict): - p = ep.get("path") - return str(p) if p is not None else "" - return "" - - dc_re = re.compile(r"\^?/?(metro\d+)/dc(\d+)") - - def _parse_dc_key(path: str) -> str | None: - m = dc_re.search(str(path)) - if not m: - return None - metro = m.group(1) - dc_idx = m.group(2) - return f"{metro}/dc{int(dc_idx)}" - - dc_capacity: dict[str, float] = {} - if isinstance(adjacency, list): - for rule in adjacency: - if not isinstance(rule, dict): - continue - src = _endpoint_path(rule.get("source")) - dst = _endpoint_path(rule.get("target")) - attrs = ( - rule.get("attrs", {}) if isinstance(rule.get("attrs"), dict) else {} - ) - try: - cap_val = float( - attrs.get("target_capacity", rule.get("capacity", 0.0)) - ) - except Exception: - cap_val = float(rule.get("capacity", 0.0)) - for endpoint in (src, dst): - key = _parse_dc_key(endpoint) - if key: - dc_capacity[key] = dc_capacity.get(key, 0.0) + cap_val - - dc_egress: dict[str, float] = {} - dc_ingress: dict[str, float] = {} - tm_set = (data or {}).get("demands") or {} - if isinstance(tm_set, dict): - for _mname, entries in tm_set.items(): - if not isinstance(entries, list): - continue - for d in entries: - if not isinstance(d, dict): - continue - try: - demand_val = float(d.get("volume", 0.0)) - except Exception: - demand_val = 0.0 - if demand_val <= 0.0: - issues.append( - "demands contains zero or negative 'volume' entry" - ) - continue - s_path = str(d.get("source", "")) - t_path = str(d.get("target", "")) - s_dc = _parse_dc_key(s_path) - t_dc = _parse_dc_key(t_path) - if s_dc: - dc_egress[s_dc] = dc_egress.get(s_dc, 0.0) + demand_val - if t_dc: - dc_ingress[t_dc] = dc_ingress.get(t_dc, 0.0) + demand_val - - all_dcs = set(dc_capacity) | set(dc_egress) | set(dc_ingress) - for dc_key in sorted(all_dcs): - cap = float(dc_capacity.get(dc_key, 0.0)) - eg = float(dc_egress.get(dc_key, 0.0)) - ing = float(dc_ingress.get(dc_key, 0.0)) - try: - logger.info( - "dc capacity check: %s capacity=%s egress_demand=%s ingress_demand=%s", - dc_key, - f"{cap:,.1f}", - f"{eg:,.1f}", - f"{ing:,.1f}", - ) - except Exception: - pass - eps = 1e-9 - if eg > cap + eps: - issues.append( - f"dc capacity: {dc_key} egress demand {eg:,.1f} exceeds adjacency capacity {cap:,.1f}" - ) - if ing > cap + eps: - issues.append( - f"dc capacity: {dc_key} ingress demand {ing:,.1f} exceeds adjacency capacity {cap:,.1f}" - ) - except Exception as e: - issues.append(f"dc capacity audit failed: {e}") - return issues diff --git a/topogen/validation/yaml_validation.py b/topogen/validation/yaml_validation.py index 571fcb9..ab96593 100644 --- a/topogen/validation/yaml_validation.py +++ b/topogen/validation/yaml_validation.py @@ -3,7 +3,6 @@ from __future__ import annotations import json -from importlib import resources as res from pathlib import Path import yaml @@ -17,13 +16,13 @@ logger = get_logger(__name__) -def validate_scenario_yaml( # noqa: C901, PLR0912, PLR0915 +def validate_scenario_yaml( scenario_yaml: str, integrated_graph_path: Path | None = None, *, run_ngraph: bool = True, - hw_component_map: dict[str, object] | None = None, - optics_map: dict[str, object] | None = None, + hw_component_map: dict[str, str] | None = None, + optics_map: dict[str, str] | None = None, ) -> list[str]: """Validate scenario YAML and return a list of issue strings. @@ -32,13 +31,12 @@ def validate_scenario_yaml( # noqa: C901, PLR0912, PLR0915 integrated_graph_path: Optional path to integrated graph JSON for cross-check of metro coordinates. run_ngraph: Check the NetGraph schema, construct a Scenario, and run - topology and hardware audits. False runs dictionary checks only. - hw_component_map: Optional override for role->platform mapping sourced - from configuration. When provided, audits use this instead of any - mapping present in the scenario. - optics_map: Optional override for role-pair->optic mapping sourced from - configuration. When provided, audits use this instead of any mapping - present in the scenario. + topology, per-demand-set DC capacity and hardware audits. False + checks metadata/references only; it does not resolve demand selectors. + hw_component_map: Optional role-to-platform assignments for hardware + coverage checks. Components must be defined in the scenario. + optics_map: Optional directional role assignments for optic coverage + checks. Explicit blueprint hardware is audited directly. Returns: List of human-readable issue strings. Empty list when no issues. @@ -47,32 +45,14 @@ def validate_scenario_yaml( # noqa: C901, PLR0912, PLR0915 try: data = yaml.safe_load(scenario_yaml) or {} - except Exception as e: + except yaml.YAMLError as e: return [f"YAML parse error: {e}"] - if run_ngraph: - try: - import jsonschema # type: ignore[import-not-found] - except Exception as e: # pragma: no cover - environment import error - issues.append(f"ngraph schema: {e}") - else: - try: - with ( - res.files("ngraph.schemas") - .joinpath("scenario.json") - .open("r", encoding="utf-8") as f - ): - schema = json.load(f) - except Exception as e: - issues.append(f"ngraph schema: {e}") - else: - try: - jsonschema.validate(data, schema) # type: ignore[arg-type] - except Exception as e: - issues.append(f"ngraph schema: {e}") + if not isinstance(data, dict): + return ["Scenario must be a YAML mapping"] ig_coords: dict[str, tuple[float, float]] | None = None - if integrated_graph_path is not None and integrated_graph_path.exists(): + if integrated_graph_path is not None: try: text = integrated_graph_path.read_text(encoding="utf-8") ig_json = json.loads(text) @@ -91,10 +71,6 @@ def validate_scenario_yaml( # noqa: C901, PLR0912, PLR0915 ) ) - try: - for _msg in issues: - logger.error(_msg) - except Exception: # pragma: no cover - pass - + for message in issues: + logger.error(message) return issues diff --git a/topogen/visualization.py b/topogen/visualization.py index ebf1ea3..5dbbc16 100644 --- a/topogen/visualization.py +++ b/topogen/visualization.py @@ -1,164 +1,59 @@ -"""Visualization utilities for topology generation.""" +"""Render current corridor, site, and blueprint contracts without format guessing.""" from __future__ import annotations +from contextlib import contextmanager from pathlib import Path -from typing import TYPE_CHECKING, Any -import contextily as cx # type: ignore -import geopandas as gpd # type: ignore +import contextily as cx import matplotlib.pyplot as plt import networkx as nx import numpy as np from matplotlib.patches import Circle -from shapely.geometry import LineString, Point +from ngraph import Network -from topogen.log_config import get_logger +from topogen.metro_clusters import MetroCluster -if TYPE_CHECKING: - from topogen.metro_clusters import MetroCluster - -logger = get_logger(__name__) - - -def export_cluster_map( - centroids: np.ndarray, output_path: Path, conus_boundary_path: Path, target_crs: str -) -> None: - """Export JPEG preview map of cluster centroids. - - Args: - centroids: Cluster centroids (x, y coordinates). - output_path: Path to save JPEG file. - conus_boundary_path: Path to CONUS boundary shapefile for visual context. - target_crs: Target coordinate reference system. - - Raises: - RuntimeError: If rendering or file output fails. - """ - fig = None +@contextmanager +def _figure(output_path: Path, *, dpi: int, figsize=(12, 8), panels=1): + """Close figures on every path; propagate drawing and output failures.""" + fig, axes = plt.subplots(1, panels, figsize=figsize) try: - logger.info(f"Exporting cluster map to {output_path}") - - if len(centroids) == 0: - raise ValueError("Cannot create map: no centroids provided") - - if not target_crs or not target_crs.strip(): - raise ValueError("Cannot create map: invalid or empty target_crs") - - if not output_path: - raise ValueError("Cannot create map: invalid output_path") - - if not conus_boundary_path or not conus_boundary_path.exists(): - raise ValueError( - f"Cannot create map: CONUS boundary file not found at {conus_boundary_path}" - ) - - try: - from shapely.geometry import Point - - geometry = [Point(float(x), float(y)) for x, y in centroids] - gdf = gpd.GeoDataFrame( - { - "x": centroids[:, 0].astype(float), - "y": centroids[:, 1].astype(float), - }, - geometry=geometry, - crs=target_crs, - ) # type: ignore - logger.debug( - f"Created centroids GeoDataFrame: {len(gdf)} points in {target_crs}" - ) - except Exception as e: - raise RuntimeError( - f"Failed to create GeoDataFrame for centroids: {e}" - ) from e - - # Urban areas are already filtered; use the boundary as a map background. - try: - from topogen.geo_utils import create_conus_mask - - conus_poly = create_conus_mask(conus_boundary_path, target_crs) - except Exception as e: - raise RuntimeError( - f"Failed to load CONUS boundary for map context: {e}" - ) from e + yield fig, axes + fig.tight_layout() + output_path.parent.mkdir(parents=True, exist_ok=True) + fig.savefig(output_path, dpi=dpi, format="jpeg", bbox_inches="tight") + finally: + plt.close(fig) - try: - fig, ax = plt.subplots(figsize=(12, 8)) - logger.debug("Created matplotlib figure (12x8) for cluster map") - except Exception as e: - raise RuntimeError(f"Failed to create matplotlib figure: {e}") from e - try: - conus_poly.plot(ax=ax, edgecolor="k", facecolor="none", linewidth=0.4) - except Exception as e: - raise RuntimeError(f"Failed to plot CONUS boundary: {e}") from e +def _boundary(ax, path: Path, target_crs: str) -> None: + from topogen.geo_utils import create_conus_mask - try: - gdf.plot(ax=ax, markersize=24, color="red", alpha=0.7) - logger.debug(f"Plotted {len(gdf)} cluster centroids (red circles)") - except Exception as e: - raise RuntimeError(f"Failed to plot cluster centroids: {e}") from e + create_conus_mask(path, target_crs).plot( + ax=ax, edgecolor="#777777", facecolor="#f6f7f9", linewidth=0.4 + ) - try: - cx.add_basemap(ax, crs=gdf.crs, attribution=False, alpha=0.3) - logger.debug("Added contextily basemap to cluster map") - except Exception as e: - raise RuntimeError(f"Failed to add basemap: {e}") from e - try: - ax.set_axis_off() - ax.set_title(f"Metro Clusters (n={len(centroids)})", fontsize=14, pad=20) - plt.tight_layout() - except Exception as e: - raise RuntimeError(f"Failed to style the plot: {e}") from e - - try: - logger.debug(f"Saving cluster map to {output_path} (DPI=150, format=JPEG)") - plt.savefig(output_path, dpi=150, format="jpeg", bbox_inches="tight") - except Exception as e: - raise RuntimeError(f"Failed to save JPEG file to {output_path}: {e}") from e - finally: - plt.close(fig) - fig = None - - if not output_path.exists(): - raise RuntimeError(f"JPEG file was not created at {output_path}") - - file_size = output_path.stat().st_size - if file_size < 1000: # Less than 1KB suggests a problem - raise RuntimeError(f"JPEG file appears corrupt (only {file_size} bytes)") - - logger.info( - f"Saved centroid preview → {output_path} ({file_size / 1024:.1f} KB)" - ) - - except Exception as e: - if fig is not None: - try: - plt.close(fig) - except Exception: - pass - - if output_path.exists(): - try: - output_path.unlink() - logger.debug(f"Cleaned up partial file: {output_path}") - except Exception: - pass - - if isinstance(e, (ValueError, RuntimeError)): - raise - else: - raise RuntimeError( - f"Unexpected error during cluster map export: {e}" - ) from e +def export_cluster_map( + centroids: np.ndarray, output_path: Path, conus_boundary_path: Path, target_crs: str +) -> None: + """Draw projected metro points over the configured CONUS boundary and basemap.""" + if len(centroids) == 0: + raise ValueError("Cannot create map: no centroids provided") + with _figure(output_path, dpi=150) as (_, ax): + _boundary(ax, conus_boundary_path, target_crs) + ax.scatter(centroids[:, 0], centroids[:, 1], s=24, color="#c42c41") + cx.add_basemap(ax, crs=target_crs, attribution=False, alpha=0.3) + ax.set_title(f"Metro clusters: {len(centroids)}") + ax.set_aspect("equal") + ax.set_axis_off() def export_integrated_graph_map( metros: list[MetroCluster], - graph: nx.Graph, + graph: nx.MultiGraph, output_path: Path, conus_boundary_path: Path, target_crs: str, @@ -166,382 +61,48 @@ def export_integrated_graph_map( use_real_geometry: bool = False, dpi: int = 150, ) -> None: - """Export JPEG preview map of integrated graph with metro clusters and corridors. - - Args: - metros: List of metro clusters. - graph: Integrated NetworkX graph with corridor tags. - output_path: Path to save JPEG file. - conus_boundary_path: Path to CONUS boundary shapefile for visual context. - target_crs: Target coordinate reference system. - dpi: Output image dots-per-inch when saving. - - Raises: - RuntimeError: If rendering or file output fails. - """ - fig = None - - try: - logger.info(f"Exporting integrated graph map to {output_path}") - - if len(metros) == 0: - raise ValueError("Cannot create map: no metros provided") - - if graph is None or len(graph.nodes) == 0: - raise ValueError("Cannot create map: graph is empty or None") - - if not target_crs or not target_crs.strip(): - raise ValueError("Cannot create map: invalid or empty target_crs") - - if not output_path: - raise ValueError("Cannot create map: invalid output_path") - - if not conus_boundary_path or not conus_boundary_path.exists(): - raise ValueError( - f"Cannot create map: CONUS boundary file not found at {conus_boundary_path}" - ) - - centroids = np.array([metro.coordinates for metro in metros]) - metro_radii_m = [ - float(getattr(metro, "radius_km", 0.0)) * 1000.0 for metro in metros - ] - - corridor_lines = [] - metro_id_to_coords = {metro.metro_id: metro.coordinates for metro in metros} - - corridor_pairs = set() - - # Handle both full highway graphs (with corridor tags) and corridor graphs (direct edges) - corridor_edges_found = 0 - - is_corridor_graph = any( - data.get("edge_type") == "corridor" for u, v, data in graph.edges(data=True) - ) - - if is_corridor_graph: - # Direct corridor graph - construct linestrings from edge geometry when available - logger.debug("Processing corridor graph (direct metro-to-metro edges)") - for _u, _v, data in graph.edges(data=True): - if data.get("edge_type") != "corridor": - continue - corridor_edges_found += 1 - - if use_real_geometry: - geom = data.get("geometry") - if not (isinstance(geom, list) and len(geom) >= 2): - raise ValueError( - "Configured to use real geometry, but corridor edge missing geometry" - ) - try: - coords = [(float(x), float(y)) for (x, y) in geom] - except Exception as e: - raise ValueError( - f"Invalid corridor geometry coordinates: {e}" - ) from e - corridor_lines.append(LineString(coords)) - else: - metro_a_id = data.get("metro_a") - metro_b_id = data.get("metro_b") - if metro_a_id and metro_b_id: - if metro_a_id not in metro_id_to_coords: - raise ValueError( - f"Corridor references unknown metro ID: {metro_a_id}" - ) - if metro_b_id not in metro_id_to_coords: - raise ValueError( - f"Corridor references unknown metro ID: {metro_b_id}" - ) - a_coords = metro_id_to_coords[metro_a_id] - b_coords = metro_id_to_coords[metro_b_id] - corridor_lines.append(LineString([a_coords, b_coords])) - else: - raise ValueError( - "Corridor edge missing metro IDs for straight-line rendering" - ) - else: - # Full highway graph - extract from corridor tags - logger.debug( - "Processing full highway graph (extracting from corridor tags)" - ) - for _u, _v, data in graph.edges(data=True): - if "corridor" in data and data["corridor"]: - corridor_edges_found += 1 - for corridor_info in data["corridor"]: - try: - metro_a_id = corridor_info["metro_a"] - metro_b_id = corridor_info["metro_b"] - - if metro_a_id not in metro_id_to_coords: - raise ValueError( - f"Corridor references unknown metro ID: {metro_a_id}" - ) - if metro_b_id not in metro_id_to_coords: - raise ValueError( - f"Corridor references unknown metro ID: {metro_b_id}" - ) - - pair = tuple(sorted([metro_a_id, metro_b_id])) - corridor_pairs.add(pair) - - except KeyError as e: - raise ValueError( - f"Invalid corridor metadata: missing key {e}" - ) from e - except Exception as e: - raise RuntimeError( - f"Error processing corridor metadata: {e}" - ) from e - - # For full graph, create LineString for each unique metro pair (straight line) - if not is_corridor_graph: - for metro_a_id, metro_b_id in corridor_pairs: - coords_a = metro_id_to_coords[metro_a_id] - coords_b = metro_id_to_coords[metro_b_id] - corridor_lines.append(LineString([coords_a, coords_b])) - - if corridor_edges_found > 0 and len(corridor_lines) == 0: - raise RuntimeError( - f"Corridor extraction failed: found {corridor_edges_found} corridor-tagged edges " - "but no valid metro pairs could be extracted" - ) - - logger.info( - f"Found {len(corridor_lines)} corridor connections from {corridor_edges_found} corridor-tagged edges" - ) - logger.debug( - f"Extracted {len(corridor_pairs)} unique metro pairs from corridor metadata" - ) - - if len(corridor_lines) == 0: - logger.warning( - "No corridor connections found for visualization. " - "Map will show metro clusters only without corridor lines." - ) - - try: - geometry = [Point(float(x), float(y)) for x, y in centroids] - metros_gdf = gpd.GeoDataFrame( - { - "x": centroids[:, 0].astype(float), - "y": centroids[:, 1].astype(float), - "name": [metro.name for metro in metros], - "radius_km": [float(metro.radius_km) for metro in metros], - }, - geometry=geometry, - crs=target_crs, - ) # type: ignore - logger.debug( - f"Created metros GeoDataFrame: {len(metros_gdf)} points in {target_crs}" - ) - except Exception as e: - raise RuntimeError(f"Failed to create GeoDataFrame for metros: {e}") from e - - corridors_gdf = None - if corridor_lines: - try: - corridors_gdf = gpd.GeoDataFrame( - {"corridor_id": list(range(len(corridor_lines)))}, - geometry=corridor_lines, - crs=target_crs, - ) # type: ignore - logger.debug( - f"Created corridors GeoDataFrame: {len(corridors_gdf)} lines in {target_crs}" + """Draw each corridor using road geometry or its metro endpoints.""" + if not metros: + raise ValueError("Cannot create map: no metros provided") + if not graph.is_multigraph(): + raise TypeError("Expected a corridor MultiGraph") + by_id = {metro.metro_id: metro for metro in metros} + with _figure(output_path, dpi=dpi) as (_, ax): + _boundary(ax, conus_boundary_path, target_crs) + for _, _, data in graph.edges(data=True): + points = ( + data["geometry"] + if use_real_geometry + else [ + by_id[data["metro_a"]].coordinates, + by_id[data["metro_b"]].coordinates, + ] + ) + xs, ys = zip(*points, strict=False) + ax.plot(xs, ys, color="#397eb5", linewidth=1.1, alpha=0.75) + for metro in metros: + x, y = metro.coordinates + ax.add_patch( + Circle( + (x, y), + metro.radius_km * 1000, + fill=False, + color="#999999", + linewidth=0.6, ) - except Exception as e: - raise RuntimeError( - f"Failed to create GeoDataFrame for corridors: {e}" - ) from e - else: - logger.debug("No corridor lines to create GeoDataFrame") - - try: - from topogen.geo_utils import create_conus_mask - - conus_poly = create_conus_mask(conus_boundary_path, target_crs) - except Exception as e: - raise RuntimeError( - f"Failed to load CONUS boundary for map context: {e}" - ) from e - - try: - fig, ax = plt.subplots(figsize=(14, 10)) - logger.debug("Created matplotlib figure (14x10)") - except Exception as e: - raise RuntimeError(f"Failed to create matplotlib figure: {e}") from e - - try: - conus_poly.plot(ax=ax, edgecolor="k", facecolor="none", linewidth=0.4) - except Exception as e: - raise RuntimeError(f"Failed to plot CONUS boundary: {e}") from e - - # Plot corridors first (so they appear behind metros) - if corridors_gdf is not None: - try: - corridors_gdf.plot(ax=ax, color="blue", linewidth=1.5, alpha=0.6) - logger.debug(f"Plotted {len(corridors_gdf)} corridor connections") - except Exception as e: - raise RuntimeError(f"Failed to plot corridor connections: {e}") from e - - # Draw circles using configured metro radii. - try: - for (x0, y0), r_m in zip(centroids, metro_radii_m, strict=False): - if r_m > 0.0: - circ = Circle( - (float(x0), float(y0)), - float(r_m), - fill=True, - facecolor="royalblue", - edgecolor="royalblue", - linewidth=0.8, - alpha=0.2, - zorder=1, - ) - ax.add_patch(circ) - except Exception as e: - raise RuntimeError(f"Failed to draw metro radius circles: {e}") from e - - # Place labels outside metro circles at hash-derived angles. - label_extent_x: list[float] = [] - label_extent_y: list[float] = [] - try: - import math as _m - - metro_names_list = [str(m.name) for m in metros] - for (x0, y0), r_m, label in zip( - centroids, metro_radii_m, metro_names_list, strict=False - ): - try: - if float(r_m) <= 0.0: - continue - h = abs(hash(label)) - phi = (h % 360) * _m.pi / 180.0 - offset = max(0.08 * float(r_m), 8000.0) - bx = float(x0) + float(r_m) * _m.cos(phi) - by = float(y0) + float(r_m) * _m.sin(phi) - lx = float(x0) + (float(r_m) + offset) * _m.cos(phi) - ly = float(y0) + (float(r_m) + offset) * _m.sin(phi) - ax.plot( - [bx, lx], - [by, ly], - color="gray", - linewidth=0.5, - alpha=0.5, - zorder=2, - ) - ha = "left" if _m.cos(phi) >= 0.0 else "right" - ax.text( - lx, - ly, - label, - fontsize=8, - color="black", - ha=ha, - va="center", - zorder=6, - bbox=dict( - boxstyle="round,pad=0.15", fc="white", ec="none", alpha=0.5 - ), - ) - label_extent_x.append(lx) - label_extent_y.append(ly) - except Exception: - continue - except Exception: - pass - - try: - metros_gdf.plot( - ax=ax, - markersize=60, - color="red", - alpha=0.8, - edgecolor="darkred", - linewidth=0.5, ) - logger.debug(f"Plotted {len(metros_gdf)} metro centroids (red circles)") - except Exception as e: - raise RuntimeError(f"Failed to plot metro centroids: {e}") from e - - try: - cx.add_basemap(ax, crs=metros_gdf.crs, attribution=False, alpha=0.3) - logger.debug("Added contextily basemap to integrated graph map") - except Exception as e: - raise RuntimeError(f"Failed to add basemap: {e}") from e - - try: - ax.set_axis_off() - corridor_count = len(corridor_lines) if corridor_lines else 0 - ax.set_title( - f"Integrated Graph: {len(metros)} Metro Clusters, {corridor_count} Corridors", - fontsize=14, - pad=20, - ) - # Include metro circle extents (and labels) in view - try: - circ_xs: list[float] = [] - circ_ys: list[float] = [] - for (x0, y0), r_m in zip(centroids, metro_radii_m, strict=False): - circ_xs.extend([float(x0) - float(r_m), float(x0) + float(r_m)]) - circ_ys.extend([float(y0) - float(r_m), float(y0) + float(r_m)]) - all_xs = circ_xs + label_extent_x - all_ys = circ_ys + label_extent_y - if len(all_xs) >= 2 and len(all_ys) >= 2: - pad = 0.05 - min_x, max_x = min(all_xs), max(all_xs) - min_y, max_y = min(all_ys), max(all_ys) - dx = max_x - min_x - dy = max_y - min_y - if dx > 0 and dy > 0: - ax.set_xlim(min_x - pad * dx, max_x + pad * dx) - ax.set_ylim(min_y - pad * dy, max_y + pad * dy) - except Exception: - pass - plt.tight_layout() - except Exception as e: - raise RuntimeError(f"Failed to style the plot: {e}") from e - - try: - logger.debug( - f"Saving integrated graph visualization to {output_path} (DPI={int(dpi)}, format=JPEG)" + ax.scatter([x], [y], s=25, color="#c42c41", zorder=3) + ax.annotate( + metro.name, + (x, y), + xytext=(4, 5), + textcoords="offset points", + fontsize=7, ) - plt.savefig(output_path, dpi=int(dpi), format="jpeg", bbox_inches="tight") - except Exception as e: - raise RuntimeError(f"Failed to save JPEG file to {output_path}: {e}") from e - finally: - plt.close(fig) - fig = None - - if not output_path.exists(): - raise RuntimeError(f"JPEG file was not created at {output_path}") - - file_size = output_path.stat().st_size - if file_size < 1000: # Less than 1KB suggests a problem - raise RuntimeError(f"JPEG file appears corrupt (only {file_size} bytes)") - - logger.info( - f"Saved integrated graph preview → {output_path} ({file_size / 1024:.1f} KB)" - ) - - except Exception as e: - if fig is not None: - try: - plt.close(fig) - except Exception: - pass - - if output_path.exists(): - try: - output_path.unlink() - logger.debug(f"Cleaned up partial file: {output_path}") - except Exception: - pass - - if isinstance(e, (ValueError, RuntimeError)): - raise - else: - raise RuntimeError( - f"Unexpected error during integrated graph map export: {e}" - ) from e + cx.add_basemap(ax, crs=target_crs, attribution=False, alpha=0.3) + ax.set_title(f"{len(metros)} metros · {graph.number_of_edges()} corridors") + ax.set_aspect("equal") + ax.set_axis_off() def export_site_graph_map( @@ -553,340 +114,80 @@ def export_site_graph_map( dpi: int = 300, target_crs: str | None = None, ) -> None: - """Export a JPEG visualization of the site-level network graph. - - Draws each metro's radius as a thin circle, places sites at their assigned - ``pos_x``/``pos_y`` coordinates, and renders edges as straight segments - between sites. Local edges are labeled with adjacency capacity - (``target_capacity``, then ``base_capacity``, then ``capacity``). Inter-metro - capacity is aggregated into one label per directed metro pair. - - Required node attributes: - - ``pos_x`` and ``pos_y``: site coordinates in target CRS units. - - ``center_x`` and ``center_y``: metro center coordinates. - - ``radius_m``: metro radius in meters. - - Args: - G: Site-level MultiGraph with required node metadata. - output_path: Path to save the JPEG. - figure_size: Matplotlib figure size in inches (width, height). - metro_scale: Multiplier applied to visualized metro circle radius. - dpi: Output image dots-per-inch when saving. - target_crs: Target coordinate reference system string for basemap. - - Raises: - ValueError: If required attributes are missing. - RuntimeError: If rendering or file output fails. - """ - fig = None - try: - logger.info(f"Exporting site graph visualization to {output_path}") - - if G is None or G.number_of_nodes() == 0: - raise ValueError("Cannot create site graph map: graph is empty or None") - - # Validate node attributes and collect per-metro circle parameters - metros: dict[int, tuple[float, float, float]] = {} - metro_names: dict[int, str] = {} - xs: list[float] = [] - ys: list[float] = [] - for node_id, data in G.nodes(data=True): - try: - x = float(data["pos_x"]) # type: ignore[index] - y = float(data["pos_y"]) # type: ignore[index] - cx = float(data["center_x"]) # type: ignore[index] - cy = float(data["center_y"]) # type: ignore[index] - r = float(data["radius_m"]) # type: ignore[index] - idx = int(data.get("metro_idx", 0)) - except Exception as exc: - raise ValueError( - f"Node '{node_id}' missing required visualization attributes" - ) from exc - xs.append(x) - ys.append(y) - if idx not in metros: - metros[idx] = (cx, cy, r) - # Capture metro canonical name if present - try: - name_val = str(data.get("metro_name", "")).strip() - if name_val: - metro_names[idx] = name_val - except Exception: - pass - - try: - fig, ax = plt.subplots(figsize=figure_size) - except Exception as e: - raise RuntimeError(f"Failed to create matplotlib figure: {e}") from e - - label_extent_x: list[float] = [] - label_extent_y: list[float] = [] - for _idx, (cx, cy, r) in metros.items(): - rr = float(r) * float(metro_scale) - if rr > 0.0: - try: - circ = Circle( - (cx, cy), - rr, - fill=False, - edgecolor="gray", - linewidth=0.8, - alpha=0.6, - ) - ax.add_patch(circ) - except Exception as e: - raise RuntimeError(f"Failed to draw metro circle: {e}") from e - try: - label = metro_names.get(_idx, f"metro{_idx}") - import math as _m - - h = abs(hash(label)) - phi = (h % 360) * _m.pi / 180.0 - offset = max(0.08 * rr, 8000.0) - lx = float(cx) + (rr + offset) * _m.cos(phi) - ly = float(cy) + (rr + offset) * _m.sin(phi) - # Keep extents inclusive of labels for autoscale - label_extent_x.append(lx) - label_extent_y.append(ly) - # Leader line from circle boundary to label - bx = float(cx) + rr * _m.cos(phi) - by = float(cy) + rr * _m.sin(phi) - ax.plot( - [bx, lx], [by, ly], color="gray", linewidth=0.5, alpha=0.5, zorder=3 - ) - ha = "left" if _m.cos(phi) >= 0.0 else "right" - ax.text( - lx, - ly, - label, - fontsize=8, - color="black", - ha=ha, - va="center", - zorder=4, - bbox=dict( - boxstyle="round,pad=0.15", fc="white", ec="none", alpha=0.5 - ), + """Draw positioned sites and label total site-edge capacities in Gbps.""" + if not G: + raise ValueError("Cannot create site graph map: graph is empty") + positions = { + node: (data["pos_x"], data["pos_y"]) for node, data in G.nodes(data=True) + } + metros = {} + for _, data in G.nodes(data=True): + metros[data["metro_idx"]] = ( + data["center_x"], + data["center_y"], + data["radius_m"], + data["metro_name"], + ) + with _figure(output_path, dpi=dpi, figsize=figure_size) as (_, ax): + for _idx, (x, y, radius, name) in sorted(metros.items()): + ax.add_patch( + Circle( + (x, y), + radius * metro_scale, + fill=False, + color="#999999", + linewidth=0.7, ) - except Exception: - pass - - for u, v, k, data in G.edges(keys=True, data=True): - try: - ux = float(G.nodes[u]["pos_x"]) # type: ignore[index] - uy = float(G.nodes[u]["pos_y"]) # type: ignore[index] - vx = float(G.nodes[v]["pos_x"]) # type: ignore[index] - vy = float(G.nodes[v]["pos_y"]) # type: ignore[index] - except Exception as e: - raise ValueError(f"Edge {u}-{v} endpoints missing positions") from e - - try: - ax.plot([ux, vx], [uy, vy], color="steelblue", linewidth=1.2, alpha=0.9) - except Exception as e: - raise RuntimeError(f"Failed to draw edge {u}-{v}") from e - - val = data.get( - "target_capacity", data.get("base_capacity", data.get("capacity", None)) ) - # Skip inter-metro adjacency labels; corridor will be labeled once per metro pair - if str(data.get("link_type", "")) == "inter_metro_corridor": - continue - if val is not None: - try: - cap = float(val) - # Hash-based offsets reduce overlap across edge labels. - edge_key = f"{u}|{v}|{k}" - h = abs(hash(edge_key)) - # Along-edge parameter in [0.35, 0.65] - t = 0.35 + 0.30 * ((h % 7) / 6.0) - mx = (1.0 - t) * ux + t * vx - my = (1.0 - t) * uy + t * vy - # Slight perpendicular offset for readability (vary magnitude/sign) - dx = vx - ux - dy = vy - uy - import math as _m - - length = _m.hypot(dx, dy) - if length > 0.0: - step = (h >> 3) % 4 # 0..3 - off_frac = 0.03 + 0.02 * float(step) # 0.03..0.09 - sign = -1.0 if ((h >> 9) % 2 == 0) else 1.0 - off = sign * off_frac - ox = -dy * off - oy = dx * off - else: - ox = 0.0 - oy = 0.0 - ax.text( - mx + ox, - my + oy, - f"{cap:,.0f}", - fontsize=6, - color="black", - ha="center", - va="center", - bbox=dict( - boxstyle="round,pad=0.2", fc="white", ec="none", alpha=0.6 - ), - zorder=5, - ) - except Exception: - # Labeling is optional; continue on format errors - pass - - # Aggregate corridor capacities per directed metro pair and place labels along centerline - corridor_caps: dict[tuple[str, str], float] = {} - for _u2, _v2, _k2, d2 in G.edges(keys=True, data=True): - if str(d2.get("link_type", "")) != "inter_metro_corridor": - continue - s_name = d2.get("source_metro") - t_name = d2.get("target_metro") - if not isinstance(s_name, str) or not isinstance(t_name, str): - continue - cap_val = d2.get("target_capacity", d2.get("base_capacity", None)) - try: - cap_num = float(cap_val) if cap_val is not None else 0.0 - except Exception: - cap_num = 0.0 - key = (s_name, t_name) - corridor_caps[key] = corridor_caps.get(key, 0.0) + cap_num - - name_to_center: dict[str, tuple[float, float]] = {} - for idx, (cx, cy, _r) in metros.items(): - nm = metro_names.get(idx, f"metro{idx}") - name_to_center[nm] = (float(cx), float(cy)) - - corr_label_x: list[float] = [] - corr_label_y: list[float] = [] - for (s_name, t_name), cap_num in corridor_caps.items(): - a = name_to_center.get(str(s_name)) - b = name_to_center.get(str(t_name)) - if a is None or b is None: - continue - ax0, ay0 = a - bx0, by0 = b - dx = bx0 - ax0 - dy = by0 - ay0 - import math as _m - - length = _m.hypot(dx, dy) - if length <= 0.0: - continue - h = abs(hash(f"{s_name}->{t_name}")) - tpar = 0.45 + 0.10 * ((h % 5) / 4.0) # 0.45..0.55 - mx = ax0 + tpar * dx - my = ay0 + tpar * dy - step = (h >> 3) % 4 - off_frac = 0.04 + 0.02 * float(step) # 0.04..0.10 - sign = -1.0 if ((h >> 9) % 2 == 0) else 1.0 - ox = sign * (-dy) * off_frac - oy = sign * (dx) * off_frac - lx = mx + ox - ly = my + oy - corr_label_x.append(lx) - corr_label_y.append(ly) - try: - ax.text( - lx, - ly, - f"{cap_num:,.0f}", - fontsize=7, - color="black", - ha="center", - va="center", - bbox=dict( - boxstyle="round,pad=0.2", fc="white", ec="none", alpha=0.6 - ), - zorder=5, - ) - except Exception: - pass - - try: + ax.annotate( + name, + (x, y + radius * metro_scale), + xytext=(0, 4), + textcoords="offset points", + ha="center", + fontsize=8, + ) + capacities = {} + for u, v, data in G.edges(data=True): + x0, y0 = positions[u] + x1, y1 = positions[v] + ax.plot([x0, x1], [y0, y1], color="#397eb5", linewidth=0.8, alpha=0.7) + if data["link_type"] == "inter_metro_corridor": + pair = tuple(sorted((G.nodes[u]["metro_idx"], G.nodes[v]["metro_idx"]))) + capacities[pair] = capacities.get(pair, 0.0) + data["target_capacity"] + for (u, v), capacity in capacities.items(): + x0, y0 = metros[u][:2] + x1, y1 = metros[v][:2] + ax.text( + (x0 + x1) / 2, + (y0 + y1) / 2, + f"{capacity:,.0f}", + fontsize=7, + ha="center", + bbox={"facecolor": "white", "edgecolor": "none", "alpha": 0.8}, + ) + for node, data in G.nodes(data=True): + x, y = positions[node] ax.scatter( - xs, - ys, - s=20, - c="crimson", - alpha=0.9, - edgecolors="white", - linewidths=0.6, + [x], + [y], + marker="s" if data["site_kind"] == "dc" else "o", + color="#c42c41", + s=30, zorder=3, ) - except Exception as e: - raise RuntimeError(f"Failed to draw site nodes: {e}") from e - - try: - if isinstance(target_crs, str) and target_crs.strip(): - import contextily as _cx # type: ignore - - _cx.add_basemap(ax, crs=target_crs, attribution=False, alpha=0.3) - except Exception as e: - raise RuntimeError(f"Failed to add basemap to site graph: {e}") from e - - # Style and bounds - try: - # Use 'box' so explicit limits below are respected without warnings - ax.set_aspect("equal", adjustable="box") - pad = 0.05 - circ_xs: list[float] = [] - circ_ys: list[float] = [] - for cx, cy, r in metros.values(): - rr = float(r) * float(metro_scale) - circ_xs.extend([float(cx) - rr, float(cx) + rr]) - circ_ys.extend([float(cy) - rr, float(cy) + rr]) - all_xs = xs + circ_xs + label_extent_x + corr_label_x - all_ys = ys + circ_ys + label_extent_y + corr_label_y - if all_xs and all_ys: - min_x, max_x = min(all_xs), max(all_xs) - min_y, max_y = min(all_ys), max(all_ys) - dx = max_x - min_x - dy = max_y - min_y - ax.set_xlim(min_x - pad * dx, max_x + pad * dx) - ax.set_ylim(min_y - pad * dy, max_y + pad * dy) - ax.set_axis_off() - ax.set_title("Site-level Network Graph", fontsize=14, pad=16) - plt.tight_layout() - except Exception as e: - raise RuntimeError(f"Failed to style site graph plot: {e}") from e - - try: - plt.savefig(output_path, dpi=int(dpi), format="jpeg", bbox_inches="tight") - except Exception as e: - raise RuntimeError(f"Failed to save site graph JPEG: {e}") from e - finally: - plt.close(fig) - fig = None - - if not output_path.exists(): - raise RuntimeError(f"JPEG file was not created at {output_path}") - - size = output_path.stat().st_size - if size < 1000: - raise RuntimeError(f"JPEG file appears corrupt (only {size} bytes)") - logger.info( - f"Saved site graph visualization → {output_path} ({size / 1024:.1f} KB)" + if target_crs is not None: + cx.add_basemap(ax, crs=target_crs, attribution=False, alpha=0.3) + ax.set_title( + f"{len(G)} sites · {G.number_of_edges()} adjacencies · corridor totals in Gbps" ) - except Exception as e: - if fig is not None: - try: - plt.close(fig) - except Exception: - pass - if output_path.exists(): - try: - output_path.unlink() - except Exception: - pass - if isinstance(e, (ValueError, RuntimeError)): - raise - raise RuntimeError(f"Unexpected error during site graph map export: {e}") from e + ax.set_aspect("equal") + ax.set_axis_off() def export_blueprint_diagram( blueprint_name: str, - blueprint_def: dict[str, Any], - net: Any, + net: Network, selected_site_path: str, output_path: Path, *, @@ -894,144 +195,44 @@ def export_blueprint_diagram( figure_size: tuple[int, int] = (14, 6), seed: int = 7, ) -> None: - """Export a two-panel JPEG of a blueprint and one expanded site. - - The left panel shows group-level links from blueprint ``nodes`` and ``links``. - The right panel shows concrete nodes and internal links under - ``selected_site_path`` (for example, ``metro1/pop2``). External links are omitted. - """ - fig = None - try: - if not isinstance(blueprint_def, dict): - raise ValueError("blueprint_def must be a mapping") - if "nodes" not in blueprint_def or not isinstance( - blueprint_def.get("nodes"), dict - ): - raise ValueError("blueprint_def must include a 'nodes' mapping") - if "links" not in blueprint_def or not isinstance( - blueprint_def.get("links"), (list, tuple) - ): - raise ValueError("blueprint_def must include a 'links' list") - - from .blueprint_viz import build_abstract_view, collect_concrete_site - - abs_view = build_abstract_view(blueprint_def, include_self_loops=True) - - internal_nodes, node_pos, internal_links = collect_concrete_site( - net, str(selected_site_path) + """Draw blueprint groups and the already expanded devices of one site.""" + from .blueprint_viz import build_abstract_view, collect_concrete_site + + view = build_abstract_view(net, selected_site_path, include_self_loops=True) + nodes, positions, links = collect_concrete_site(net, selected_site_path) + if not nodes: + raise ValueError(f"No expanded nodes found under site '{selected_site_path}'") + with _figure(output_path, dpi=dpi, figsize=figure_size, panels=2) as (fig, axes): + abstract, concrete = axes + layout = nx.spring_layout(view.graph, seed=seed) + nx.draw_networkx_nodes( + view.graph, layout, node_size=900, node_color="#f0f0ff", ax=abstract ) - if not internal_nodes: - raise ValueError( - f"No expanded nodes found under site '{selected_site_path}'" - ) - - fig, axes = plt.subplots(1, 2, figsize=figure_size) - ax_abs, ax_conc = axes - - # Abstract panel drawing - try: - import networkx as _nx - - pos = _nx.spring_layout(abs_view.graph, seed=seed) - _nx.draw_networkx_nodes( - abs_view.graph, pos, node_size=900, node_color="#f0f0ff", ax=ax_abs - ) - _nx.draw_networkx_labels( - abs_view.graph, pos, labels=abs_view.node_labels, font_size=8, ax=ax_abs - ) - _nx.draw_networkx_edges( - abs_view.graph, pos, width=1.2, edge_color="#666", ax=ax_abs - ) - _nx.draw_networkx_edge_labels( - abs_view.graph, - pos, - edge_labels=abs_view.edge_labels, - font_size=7, - ax=ax_abs, - ) - # Draw self-loops around nodes if present - try: - import math as _m - - for node, label in abs_view.self_loops: - if node not in pos: - continue - x, y = pos[node] - r = 0.15 # small loop radius in layout units - theta = np.linspace(0.0, 2.0 * _m.pi, 64) - xs = x + r * np.cos(theta) - ys = y + r * np.sin(theta) - ax_abs.plot(xs, ys, color="#888", linewidth=1.0) - ax_abs.text( - x + r * 1.1, - y + r * 1.1, - label, - fontsize=7, - ha="left", - va="bottom", - bbox=dict( - boxstyle="round,pad=0.2", fc="white", ec="none", alpha=0.6 - ), - ) - except Exception: - pass - ax_abs.set_title(f"Abstract: {blueprint_name}") - ax_abs.set_axis_off() - except Exception as e: # pragma: no cover - visualization fallback - raise RuntimeError(f"Failed to render abstract panel: {e}") from e - - # Concrete panel drawing - try: - for s, t, _cap in internal_links: - if s in node_pos and t in node_pos: - x0, y0 = node_pos[s] - x1, y1 = node_pos[t] - ax_conc.plot([x0, x1], [y0, y1], color="#4a90e2", linewidth=1.0) - - xs = [node_pos[n][0] for n in internal_nodes if n in node_pos] - ys = [node_pos[n][1] for n in internal_nodes if n in node_pos] - ax_conc.scatter( - xs, ys, s=80, c="#cc2a36", edgecolors="white", linewidths=0.6 - ) - for n in internal_nodes: - if n not in node_pos: - continue - x, y = node_pos[n] - short = n.split("/", 2)[-1] - ax_conc.text(x, y, short, fontsize=7, ha="center", va="bottom") - - ax_conc.set_title(f"Concrete: {selected_site_path}") - ax_conc.set_axis_off() - except Exception as e: # pragma: no cover - visualization fallback - raise RuntimeError(f"Failed to render concrete panel: {e}") from e - - fig.suptitle(f"Blueprint '{blueprint_name}' example", fontsize=12) - plt.tight_layout() - - try: - plt.savefig(output_path, dpi=int(dpi), format="jpeg", bbox_inches="tight") - finally: - plt.close(fig) - fig = None - - if not output_path.exists(): - raise RuntimeError(f"JPEG file was not created at {output_path}") - if output_path.stat().st_size < 1000: - raise RuntimeError("Blueprint diagram JPEG appears corrupt (<1KB)") - logger.info("Saved blueprint diagram → %s", str(output_path)) - except Exception as e: - if fig is not None: - try: - plt.close(fig) - except Exception: - pass - if output_path.exists(): - try: - output_path.unlink() - except Exception: - pass - if isinstance(e, (ValueError, RuntimeError)): - raise - raise RuntimeError( - f"Unexpected error during blueprint diagram export: {e}" - ) from e + nx.draw_networkx_labels( + view.graph, layout, labels=view.node_labels, font_size=8, ax=abstract + ) + nx.draw_networkx_edges( + view.graph, layout, width=1.2, edge_color="#666666", ax=abstract + ) + nx.draw_networkx_edge_labels( + view.graph, layout, edge_labels=view.edge_labels, font_size=7, ax=abstract + ) + for node, label in view.self_loops: + x, y = layout[node] + abstract.add_patch(Circle((x, y), 0.15, fill=False, color="#888888")) + abstract.text(x + 0.16, y + 0.16, label, fontsize=7) + for source, target, _ in links: + x0, y0 = positions[source] + x1, y1 = positions[target] + concrete.plot([x0, x1], [y0, y1], color="#397eb5", linewidth=0.8) + for node in nodes: + x, y = positions[node] + concrete.scatter([x], [y], s=60, color="#c42c41") + concrete.text( + x, y, node.split("/", 2)[-1], fontsize=6, ha="center", va="bottom" + ) + abstract.set_title(f"Abstract: {blueprint_name}") + concrete.set_title(f"Concrete: {selected_site_path}") + for ax in axes: + ax.set_axis_off() + fig.suptitle(f"Blueprint {blueprint_name}") diff --git a/topogen/workflows_lib.py b/topogen/workflows_lib.py index 9febda4..02def9f 100644 --- a/topogen/workflows_lib.py +++ b/topogen/workflows_lib.py @@ -7,12 +7,12 @@ from __future__ import annotations from copy import deepcopy -from pathlib import Path from typing import Any -import yaml from ngraph.workflow.parse import build_workflow_steps +from topogen.library_io import load_user_library + _BUILTIN_WORKFLOWS: dict[str, list[dict[str, Any]]] = { "design_analysis_brief": [ {"type": "NetworkStats", "name": "network_statistics"}, @@ -52,24 +52,6 @@ } -def _load_user_library(file_name: str) -> dict[str, Any]: - """Load user workflow library from ``lib/`` if present.""" - lib_path = Path.cwd() / "lib" / file_name - if not lib_path.exists(): - return {} - - try: - with lib_path.open("r", encoding="utf-8") as f: - data = yaml.safe_load(f) or {} - except Exception as exc: # noqa: BLE001 - raise ValueError(f"Failed to parse YAML: {lib_path}") from exc - - if not isinstance(data, dict): - raise ValueError(f"User library YAML must be a mapping: {lib_path}") - - return data - - def get_builtin_workflows( matrix_name: str = "baseline_traffic_matrix", ) -> dict[str, list[dict[str, Any]]]: @@ -83,7 +65,7 @@ def get_builtin_workflows( for step in steps: if "demand_set" in step: step["demand_set"] = matrix_name - user_workflows = _load_user_library("workflows.yml") + user_workflows = load_user_library("workflows.yml") workflows.update(user_workflows) for name, steps in workflows.items(): if not isinstance(steps, list) or any(not isinstance(s, dict) for s in steps): diff --git a/typings/contextily/__init__.pyi b/typings/contextily/__init__.pyi new file mode 100644 index 0000000..82242ce --- /dev/null +++ b/typings/contextily/__init__.pyi @@ -0,0 +1,17 @@ +"""Project-used subset of Contextily's API, checked against version 1.7.1. + +Only add_basemap and the arguments used by TopoGen are declared. Check upstream +contextily/plotting.py before extending this subset or updating its signature. +""" + +from typing import Literal + +from matplotlib.axes import Axes + +def add_basemap( + ax: Axes, + *, + crs: str | None = ..., + attribution: str | Literal[False] | None = ..., + alpha: float | None = ..., +) -> None: ...