Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
52 changes: 47 additions & 5 deletions tests/test_registry_builder.py
Original file line number Diff line number Diff line change
Expand Up @@ -487,6 +487,11 @@ def test_path_contract_year_layout() -> None:
assert loc["multi_file"] is True
assert "/*/" in loc["path"]

# mart per-anno su più anni: stesso trattamento del clean
mart_loc = contract.mart_location("multi_year", "mart_trend", year=2024, years=[2023, 2024])
assert mart_loc["multi_file"] is True
assert mart_loc["path"] == "gs://dataciviclab-mart/multi_year/*/mart_trend.parquet"


@pytest.mark.contract
def test_path_contract_flat_layout() -> None:
Expand Down Expand Up @@ -876,11 +881,14 @@ def test_no_sections_returns_empty(self, tmp_path: Path) -> None:


class TestMartMultiYearLocation:
"""Mart con years esplicite → location flat; per-anno → year.
"""Mart con years esplicite → flat; mart per-anno → glob multi-anno.

Regressione issue #463: il runner scrive i mart multi-anno flat
(data/mart/{dataset}/{table}.parquet), ma il builder li pubblicava con
path anno inesistente (year=max). Ora tabella con years → flat.

Regressione mart per-anno: un path puntuale all'anno massimo esponeva un
solo anno ai consumer del registry (dcl-mcp). Ora glob su tutti gli anni.
"""

@pytest.mark.contract
Expand Down Expand Up @@ -916,11 +924,45 @@ def test_multi_year_mart_flat(self, tmp_path: Path) -> None:

assert not errors["derive"], errors["derive"]
by_table = {m["table"]: m for m in catalog["marts"]}
# multi-anno → flat
# multi-anno (years esplicite) → flat, un file
assert by_table["mart_trend"]["location"]["path"] == (
"gs://dataciviclab-mart/test/my_dataset/mart_trend.parquet"
)
# per-anno → year
assert by_table["mart_sintesi"]["location"]["path"] == (
"gs://dataciviclab-mart/test/my_dataset/2025/mart_sintesi.parquet"
assert by_table["mart_trend"]["location"]["multi_file"] is False
# per-anno su dataset multi-anno → glob su tutti gli anni, multi_file
loc = by_table["mart_sintesi"]["location"]
assert loc["path"] == "gs://dataciviclab-mart/test/my_dataset/*/mart_sintesi.parquet"
assert loc["multi_file"] is True

@pytest.mark.contract
def test_per_year_mart_single_year_stays_exact(self, tmp_path: Path) -> None:
"""Mart per-anno su dataset mono-anno → path puntuale, multi_file False."""
from toolkit.registry.builders import build_mart_catalog
from toolkit.registry.layout import RepoLayout
from toolkit.registry.paths import PathContract

ds_dir = tmp_path / "datasets" / "one-year"
ds_dir.mkdir(parents=True)
(ds_dir / "dataset.yml").write_text(
"root: '../../out'\n"
"dataset:\n"
" name: 'one_year'\n"
" source_id: 'src'\n"
" years: [2024]\n"
"mart:\n"
" tables:\n"
" - name: 'mart_sintesi'\n"
" sql: 'sql/mart_sintesi.sql'\n",
encoding="utf-8",
)

layout = RepoLayout(
repo_root=tmp_path, dataset_dirs=("datasets",), source_repo="dataciviclab/test"
)
contract = PathContract(prefix="test", clean_layout="year", mart_layout="year")
catalog, errors = build_mart_catalog(layout, path_contract=contract)

assert not errors["derive"], errors["derive"]
loc = catalog["marts"][0]["location"]
assert loc["path"] == "gs://dataciviclab-mart/test/one_year/2024/mart_sintesi.parquet"
assert loc["multi_file"] is False
4 changes: 3 additions & 1 deletion toolkit/registry/builders.py
Original file line number Diff line number Diff line change
Expand Up @@ -294,7 +294,9 @@ def build_mart_catalog(
"(layout 'year' senza years dichiarati)"
)
continue
entry["location"] = contract.mart_location(manifest.slug, name, year=year)
entry["location"] = contract.mart_location(
manifest.slug, name, year=year, years=sorted(manifest.years)
)
marts.append(entry)

catalog: dict[str, Any] = {
Expand Down
31 changes: 27 additions & 4 deletions toolkit/registry/paths.py
Original file line number Diff line number Diff line change
Expand Up @@ -61,8 +61,6 @@ def clean_location(self, slug: str, years: list[int]) -> dict[str, Any]:

multi = len(years) > 1
if multi:
from lab_connectors.gcs.paths import get_bucket

bucket = get_bucket("clean")
path = f"gs://{bucket}/{prefix}{slug}/*/{slug}_*_clean.parquet"
else:
Expand All @@ -88,8 +86,33 @@ def mart_parquet_url(self, dataset: str, table: str, year: int | None = None) ->
kwargs = {"slug": dataset, "year": str(year), "table": table}
return gs_url("mart", pattern, prefix=prefix, **kwargs)

def mart_location(self, dataset: str, table: str, year: int | None = None) -> dict[str, Any]:
"""Location per il mart_catalog."""
def mart_location(
self,
dataset: str,
table: str,
year: int | None = None,
*,
years: list[int] | None = None,
) -> dict[str, Any]:
"""Location per il mart_catalog.

Speculare a ``clean_location``:

- mart multi-anno (``year=None``): un file flat, ``multi_file=False``
- mart per-anno su un solo anno: path puntuale, ``multi_file=False``
- mart per-anno su più anni: glob ``{dataset}/*/{table}.parquet`` con
``multi_file=True``. I file per-anno hanno lo stesso nome in
directory diverse, quindi un path puntuale all'anno massimo
esporrebbe un solo anno ai consumer del registry.
"""
prefix = f"{self.prefix}/" if self.prefix else ""
if self.mart_layout != "flat" and year is not None and years and len(years) > 1:
bucket = get_bucket("mart")
return {
"type": "gcs",
"path": f"gs://{bucket}/{prefix}{dataset}/*/{table}.parquet",
"multi_file": True,
}
return {
"type": "gcs",
"path": self.mart_parquet_url(dataset, table, year=year),
Expand Down
3 changes: 2 additions & 1 deletion toolkit/registry/schemas/registry.schema.json
Original file line number Diff line number Diff line change
Expand Up @@ -200,7 +200,8 @@
"type": "string"
},
"multi_file": {
"type": "boolean"
"type": "boolean",
"description": "true = path con glob su più file (clean multi-anno, mart per-anno su dataset multi-anno); false = file singolo (clean mono-anno, mart flat o mono-anno)."
}
},
"additionalProperties": false
Expand Down
Loading