diff --git a/tests/test_registry_builder.py b/tests/test_registry_builder.py index f34cc90..198a3fe 100644 --- a/tests/test_registry_builder.py +++ b/tests/test_registry_builder.py @@ -487,6 +487,11 @@ def test_path_contract_year_layout() -> None: assert loc["multi_file"] is True assert "/*/" in loc["path"] + # mart per-anno su più anni: stesso trattamento del clean + mart_loc = contract.mart_location("multi_year", "mart_trend", year=2024, years=[2023, 2024]) + assert mart_loc["multi_file"] is True + assert mart_loc["path"] == "gs://dataciviclab-mart/multi_year/*/mart_trend.parquet" + @pytest.mark.contract def test_path_contract_flat_layout() -> None: @@ -876,11 +881,14 @@ def test_no_sections_returns_empty(self, tmp_path: Path) -> None: class TestMartMultiYearLocation: - """Mart con years esplicite → location flat; per-anno → year. + """Mart con years esplicite → flat; mart per-anno → glob multi-anno. Regressione issue #463: il runner scrive i mart multi-anno flat (data/mart/{dataset}/{table}.parquet), ma il builder li pubblicava con path anno inesistente (year=max). Ora tabella con years → flat. + + Regressione mart per-anno: un path puntuale all'anno massimo esponeva un + solo anno ai consumer del registry (dcl-mcp). Ora glob su tutti gli anni. """ @pytest.mark.contract @@ -916,11 +924,45 @@ def test_multi_year_mart_flat(self, tmp_path: Path) -> None: assert not errors["derive"], errors["derive"] by_table = {m["table"]: m for m in catalog["marts"]} - # multi-anno → flat + # multi-anno (years esplicite) → flat, un file assert by_table["mart_trend"]["location"]["path"] == ( "gs://dataciviclab-mart/test/my_dataset/mart_trend.parquet" ) - # per-anno → year - assert by_table["mart_sintesi"]["location"]["path"] == ( - "gs://dataciviclab-mart/test/my_dataset/2025/mart_sintesi.parquet" + assert by_table["mart_trend"]["location"]["multi_file"] is False + # per-anno su dataset multi-anno → glob su tutti gli anni, multi_file + loc = by_table["mart_sintesi"]["location"] + assert loc["path"] == "gs://dataciviclab-mart/test/my_dataset/*/mart_sintesi.parquet" + assert loc["multi_file"] is True + + @pytest.mark.contract + def test_per_year_mart_single_year_stays_exact(self, tmp_path: Path) -> None: + """Mart per-anno su dataset mono-anno → path puntuale, multi_file False.""" + from toolkit.registry.builders import build_mart_catalog + from toolkit.registry.layout import RepoLayout + from toolkit.registry.paths import PathContract + + ds_dir = tmp_path / "datasets" / "one-year" + ds_dir.mkdir(parents=True) + (ds_dir / "dataset.yml").write_text( + "root: '../../out'\n" + "dataset:\n" + " name: 'one_year'\n" + " source_id: 'src'\n" + " years: [2024]\n" + "mart:\n" + " tables:\n" + " - name: 'mart_sintesi'\n" + " sql: 'sql/mart_sintesi.sql'\n", + encoding="utf-8", + ) + + layout = RepoLayout( + repo_root=tmp_path, dataset_dirs=("datasets",), source_repo="dataciviclab/test" ) + contract = PathContract(prefix="test", clean_layout="year", mart_layout="year") + catalog, errors = build_mart_catalog(layout, path_contract=contract) + + assert not errors["derive"], errors["derive"] + loc = catalog["marts"][0]["location"] + assert loc["path"] == "gs://dataciviclab-mart/test/one_year/2024/mart_sintesi.parquet" + assert loc["multi_file"] is False diff --git a/toolkit/registry/builders.py b/toolkit/registry/builders.py index 262b938..2fa407e 100644 --- a/toolkit/registry/builders.py +++ b/toolkit/registry/builders.py @@ -294,7 +294,9 @@ def build_mart_catalog( "(layout 'year' senza years dichiarati)" ) continue - entry["location"] = contract.mart_location(manifest.slug, name, year=year) + entry["location"] = contract.mart_location( + manifest.slug, name, year=year, years=sorted(manifest.years) + ) marts.append(entry) catalog: dict[str, Any] = { diff --git a/toolkit/registry/paths.py b/toolkit/registry/paths.py index 99cb1ec..4a54c17 100644 --- a/toolkit/registry/paths.py +++ b/toolkit/registry/paths.py @@ -61,8 +61,6 @@ def clean_location(self, slug: str, years: list[int]) -> dict[str, Any]: multi = len(years) > 1 if multi: - from lab_connectors.gcs.paths import get_bucket - bucket = get_bucket("clean") path = f"gs://{bucket}/{prefix}{slug}/*/{slug}_*_clean.parquet" else: @@ -88,8 +86,33 @@ def mart_parquet_url(self, dataset: str, table: str, year: int | None = None) -> kwargs = {"slug": dataset, "year": str(year), "table": table} return gs_url("mart", pattern, prefix=prefix, **kwargs) - def mart_location(self, dataset: str, table: str, year: int | None = None) -> dict[str, Any]: - """Location per il mart_catalog.""" + def mart_location( + self, + dataset: str, + table: str, + year: int | None = None, + *, + years: list[int] | None = None, + ) -> dict[str, Any]: + """Location per il mart_catalog. + + Speculare a ``clean_location``: + + - mart multi-anno (``year=None``): un file flat, ``multi_file=False`` + - mart per-anno su un solo anno: path puntuale, ``multi_file=False`` + - mart per-anno su più anni: glob ``{dataset}/*/{table}.parquet`` con + ``multi_file=True``. I file per-anno hanno lo stesso nome in + directory diverse, quindi un path puntuale all'anno massimo + esporrebbe un solo anno ai consumer del registry. + """ + prefix = f"{self.prefix}/" if self.prefix else "" + if self.mart_layout != "flat" and year is not None and years and len(years) > 1: + bucket = get_bucket("mart") + return { + "type": "gcs", + "path": f"gs://{bucket}/{prefix}{dataset}/*/{table}.parquet", + "multi_file": True, + } return { "type": "gcs", "path": self.mart_parquet_url(dataset, table, year=year), diff --git a/toolkit/registry/schemas/registry.schema.json b/toolkit/registry/schemas/registry.schema.json index 8a10fb0..6a4a758 100644 --- a/toolkit/registry/schemas/registry.schema.json +++ b/toolkit/registry/schemas/registry.schema.json @@ -200,7 +200,8 @@ "type": "string" }, "multi_file": { - "type": "boolean" + "type": "boolean", + "description": "true = path con glob su più file (clean multi-anno, mart per-anno su dataset multi-anno); false = file singolo (clean mono-anno, mart flat o mono-anno)." } }, "additionalProperties": false