From a1888db2dea081d391133f9015965ff3136813f8 Mon Sep 17 00:00:00 2001 From: Manuel Schlund Date: Wed, 26 Aug 2026 12:14:47 +0200 Subject: [PATCH 1/8] Remove get_config_user --- esmvalcore/_main.py | 50 ---------------------- tests/integration/test_main.py | 78 ---------------------------------- 2 files changed, 128 deletions(-) diff --git a/esmvalcore/_main.py b/esmvalcore/_main.py index 10fcec24c4..7575e0ce79 100644 --- a/esmvalcore/_main.py +++ b/esmvalcore/_main.py @@ -356,56 +356,6 @@ def _copy_config_file( shutil.copy2(in_file, out_file) logger.info("Copy finished.") - @classmethod - def get_config_user( - cls, - overwrite: bool = False, - path: str | Path | None = None, - ) -> None: - """Copy default configuration to a given path. - - Copy default configuration to a given path or, if a `path` is not - provided, install it in the default `~/.config/esmvaltool/` directory. - - Parameters - ---------- - overwrite: - Overwrite an existing file. - path: - If not provided, the file will be copied to - `~/.config/esmvaltool/`. - - .. deprecated:: 2.13.0:: - - This function is deprecated and will be removed in ESMValCore - version 2.16.0. Use the ``copy`` method instead. - - """ - from esmvalcore.exceptions import ESMValCoreDeprecationWarning - - deprecation_msg = ( - "The 'esmvaltool config get_config_user' command is deprecated and " - "will be removed in ESMValCore version 2.16.0. Use the command " - "`esmvaltool config copy defaults/config-user.yml` instead." - ) - warnings.warn( - deprecation_msg, - category=ESMValCoreDeprecationWarning, - stacklevel=1, - ) - from .config._config_object import DEFAULT_CONFIG_DIR - - in_file = DEFAULT_CONFIG_DIR / "config-user.yml" - if path is None: - out_file = ( - Path.home() / ".config" / "esmvaltool" / "config-user.yml" - ) - else: - out_file = Path(path) - if not out_file.suffix: # out_file looks like a directory - out_file = out_file / "config-user.yml" - cls._copy_config_file(in_file, out_file, overwrite) - @classmethod def get_config_developer( cls, diff --git a/tests/integration/test_main.py b/tests/integration/test_main.py index 333db60194..600f7bfd8d 100644 --- a/tests/integration/test_main.py +++ b/tests/integration/test_main.py @@ -344,81 +344,3 @@ def test_get_config_developer_bad_option_fails(): ): with pytest.raises(FireExit): run() - - -@patch( - "esmvalcore._main.Config.get_config_user", - new=wrapper(Config.get_config_user), -) -def test_get_config_user(): - """Test esmvaltool config get_config_user command.""" - with arguments("esmvaltool", "config", "get_config_user"): - run() - - -def test_get_config_user_no_path(mocker, tmp_path): - """Test esmvaltool config get_config_user command.""" - mocker.patch.object(esmvalcore._main.Path, "home", return_value=tmp_path) - with arguments("esmvaltool", "config", "get_config_user"): - run() - config_file = tmp_path / ".config" / "esmvaltool" / "config-user.yml" - assert config_file.is_file() - - -def test_get_config_user_path(tmp_path): - """Test esmvaltool config get_config_user command.""" - new_path = tmp_path / "subdir" - with arguments( - "esmvaltool", - "config", - "get_config_user", - f"--path={new_path}", - ): - run() - assert (new_path / "config-user.yml").is_file() - - -def test_get_config_user_overwrite(tmp_path): - """Test esmvaltool config get_config_user command.""" - config_user = tmp_path / "config-user.yml" - config_user.write_text("old text") - with arguments( - "esmvaltool", - "config", - "get_config_user", - f"--path={tmp_path}", - "--overwrite", - ): - run() - assert config_user.read_text() != "old text" - - -def test_get_config_user_no_overwrite(tmp_path): - """Test esmvaltool config get_config_user command.""" - config_user = tmp_path / "configuration_file.yml" - config_user.write_text("old text") - with arguments( - "esmvaltool", - "config", - "get_config_user", - f"--path={config_user}", - ): - with pytest.raises(SystemExit): - run() - assert config_user.read_text() == "old text" - - -@patch( - "esmvalcore._main.Config.get_config_user", - new=wrapper(Config.get_config_user), -) -def test_get_config_user_bad_option_fails(): - """Test esmvaltool config get_config_user command.""" - with arguments( - "esmvaltool", - "config", - "get_config_user", - "--bad_option=path", - ): - with pytest.raises(FireExit): - run() From 29ed79ef34881c5d291d1d4d081fa20d15632990 Mon Sep 17 00:00:00 2001 From: Manuel Schlund Date: Wed, 26 Aug 2026 12:15:42 +0200 Subject: [PATCH 2/8] Remove get_config_developer --- esmvalcore/_main.py | 42 ------------------ tests/integration/test_main.py | 78 ---------------------------------- 2 files changed, 120 deletions(-) diff --git a/esmvalcore/_main.py b/esmvalcore/_main.py index 7575e0ce79..c2fa76a53c 100644 --- a/esmvalcore/_main.py +++ b/esmvalcore/_main.py @@ -356,48 +356,6 @@ def _copy_config_file( shutil.copy2(in_file, out_file) logger.info("Copy finished.") - @classmethod - def get_config_developer( - cls, - overwrite: bool = False, - path: str | Path | None = None, - ) -> None: - """Copy default config-developer.yml file to a given path. - - Copy default config-developer.yml file to a given path or, if a path is - not provided, install it in the default `~/.esmvaltool` folder. - - Parameters - ---------- - overwrite: boolean - Overwrite an existing file. - path: str - If not provided, the file will be copied to `~/.esmvaltool`. - - """ - from esmvalcore.exceptions import ESMValCoreDeprecationWarning - - deprecation_msg = ( - "The config-developer.yml file and the associated " - "'esmvaltool config get_config_developer' command are deprecated " - "and support for them will be removed in ESMValCore version 2.16.0. " - "Please configure data sources, cmor tables, and preprocessor " - "filename templates under `projects` instead." - ) - warnings.warn( - deprecation_msg, - category=ESMValCoreDeprecationWarning, - stacklevel=1, - ) - in_file = Path(__file__).parent / "config-developer.yml" - if path is None: - out_file = Path.home() / ".esmvaltool" / "config-developer.yml" - else: - out_file = Path(path) - if not out_file.suffix: # out_file looks like a directory - out_file = out_file / "config-developer.yml" - cls._copy_config_file(in_file, out_file, overwrite) - class Recipes: """List, show and retrieve installed recipes. diff --git a/tests/integration/test_main.py b/tests/integration/test_main.py index 600f7bfd8d..634488e105 100644 --- a/tests/integration/test_main.py +++ b/tests/integration/test_main.py @@ -266,81 +266,3 @@ def test_config_show_brief_by_default(capsys: pytest.CaptureFixture) -> None: assert "projects" in cfg for project in cfg["projects"]: assert "extra_facets" not in cfg["projects"][project] - - -@patch( - "esmvalcore._main.Config.get_config_developer", - new=wrapper(Config.get_config_developer), -) -def test_get_config_developer(): - """Test esmvaltool config get_config_developer command.""" - with arguments("esmvaltool", "config", "get_config_developer"): - run() - - -def test_get_config_developer_no_path(mocker, tmp_path): - """Test esmvaltool config get_config_developer command.""" - mocker.patch.object(esmvalcore._main.Path, "home", return_value=tmp_path) - with arguments("esmvaltool", "config", "get_config_developer"): - run() - config_file = tmp_path / ".esmvaltool" / "config-developer.yml" - assert config_file.is_file() - - -def test_get_config_developer_path(tmp_path): - """Test esmvaltool config get_config_developer command.""" - new_path = tmp_path / "subdir" - with arguments( - "esmvaltool", - "config", - "get_config_developer", - f"--path={new_path}", - ): - run() - assert (new_path / "config-developer.yml").is_file() - - -def test_get_config_developer_overwrite(tmp_path): - """Test esmvaltool config get_config_developer command.""" - config_developer = tmp_path / "config-developer.yml" - config_developer.write_text("old text") - with arguments( - "esmvaltool", - "config", - "get_config_developer", - f"--path={tmp_path}", - "--overwrite", - ): - run() - assert config_developer.read_text() != "old text" - - -def test_get_config_developer_no_overwrite(tmp_path): - """Test esmvaltool config get_config_developer command.""" - config_developer = tmp_path / "configuration_file.yml" - config_developer.write_text("old text") - with arguments( - "esmvaltool", - "config", - "get_config_developer", - f"--path={config_developer}", - ): - with pytest.raises(SystemExit): - run() - assert config_developer.read_text() == "old text" - - -@patch( - "esmvalcore._main.Config.get_config_developer", - new=wrapper(Config.get_config_developer), -) -def test_get_config_developer_bad_option_fails(): - """Test esmvaltool config get_config_developer command.""" - with arguments( - "esmvaltool", - "config", - "get_config_developer", - "--bad_option=path", - ): - with pytest.raises(FireExit): - run() From 3932c2dc3b1aae9e3c0b2e779cc6311ad4b3d86d Mon Sep 17 00:00:00 2001 From: Manuel Schlund Date: Wed, 26 Aug 2026 12:26:29 +0200 Subject: [PATCH 3/8] Remove DataSource class --- esmvalcore/local.py | 38 -------------------- tests/unit/io/local/test_get_data_sources.py | 24 +------------ tests/unit/test_local.py | 29 --------------- 3 files changed, 1 insertion(+), 90 deletions(-) delete mode 100644 tests/unit/test_local.py diff --git a/esmvalcore/local.py b/esmvalcore/local.py index f4b8ede78b..e125602d7c 100644 --- a/esmvalcore/local.py +++ b/esmvalcore/local.py @@ -35,7 +35,6 @@ from esmvalcore.typing import FacetValue __all__ = [ - "DataSource", "LocalDataSource", "LocalFile", "find_files", @@ -141,43 +140,6 @@ def _get_data_sources(project: str) -> list[LocalDataSource]: raise KeyError(msg) -class DataSource(LocalDataSource): - """Data source for finding files on a local filesystem. - - .. deprecated:: 2.14.0 - This class is deprecated and will be removed in version 2.16.0. - Please use :class:`esmvalcore.local.LocalDataSource` instead. - """ - - def __init__(self, *args, **kwargs): - msg = ( - "The 'esmvalcore.local.LocalDataSource' class is deprecated and will be " - "removed in version 2.16.0. Please use 'esmvalcore.local.LocalDataSource'" - ) - warnings.warn(msg, DeprecationWarning, stacklevel=2) - super().__init__(*args, **kwargs) - - @property - def regex_pattern(self) -> str: - """Get regex pattern that can be used to extract facets from paths.""" - return self._regex_pattern - - def get_glob_patterns(self, **facets: FacetValue) -> list[Path]: - """Compose the globs that will be used to look for files.""" - try: - return self._get_glob_patterns(**facets) - except _MissingFacetError as exc: - raise RecipeError(exc.args[0]) from exc - - def path2facets(self, path: Path, add_timerange: bool) -> dict[str, str]: - """Extract facets from path.""" - return self._path2facets(path, add_timerange) - - def find_files(self, **facets: FacetValue) -> list[LocalFile]: - """Find files.""" - return self.find_data(**facets) - - def find_files( *, debug: bool = False, diff --git a/tests/unit/io/local/test_get_data_sources.py b/tests/unit/io/local/test_get_data_sources.py index d3ccf59e71..5421279264 100644 --- a/tests/unit/io/local/test_get_data_sources.py +++ b/tests/unit/io/local/test_get_data_sources.py @@ -9,7 +9,7 @@ import esmvalcore.cmor.table from esmvalcore.config import CFG from esmvalcore.io.local import LocalDataSource -from esmvalcore.local import DataSource, _get_data_sources +from esmvalcore.local import _get_data_sources if TYPE_CHECKING: import pytest_mock @@ -70,25 +70,3 @@ def test_get_data_sources_nodefault(monkeypatch): ) with pytest.raises(KeyError): _get_data_sources("CMIP6") - - -def test_data_source_deprecated(mocker: pytest_mock.MockerFixture) -> None: - """Test that DataSource is deprecated.""" - mocker.patch.object(DataSource, "_path2facets") - mocker.patch.object(DataSource, "find_data") - with pytest.deprecated_call(): - data_source = DataSource( - name="test", - project="CMIP6", - priority=1, - rootpath=Path("/climate_data"), - dirname_template="/", - filename_template="*.nc", - ) - - assert data_source.regex_pattern - assert data_source.get_glob_patterns() == [Path("/climate_data/*.nc")] - data_source.path2facets(Path("/climate_data/some_file.nc"), False) - data_source._path2facets.assert_called() # type: ignore[attr-defined] - data_source.find_files(dataset="a") - data_source.find_data.assert_called() # type: ignore[attr-defined] diff --git a/tests/unit/test_local.py b/tests/unit/test_local.py deleted file mode 100644 index 656624bb27..0000000000 --- a/tests/unit/test_local.py +++ /dev/null @@ -1,29 +0,0 @@ -"""Tests for the (deprecated) esmvalcore.local module.""" - -from pathlib import Path - -import pytest - -from esmvalcore.exceptions import RecipeError -from esmvalcore.local import DataSource - - -def test_get_glob_patterns_missing_facets() -> None: - """Test that get_glob_patterns raises when required facets are missing.""" - local_data_source = DataSource( - name="test", - project="test", - priority=1, - rootpath=Path("/climate_data"), - dirname_template="{dataset}", - filename_template="{short_name}*nc", - ) - facets = { - "short_name": "tas", - } - expected_message = ( - "Unable to complete path '{dataset}' because the facet 'dataset' has " - "not been specified." - ) - with pytest.raises(RecipeError, match=expected_message): - local_data_source.get_glob_patterns(**facets) From 870519f61cb9d103a4097ce63af364c40a75f3a3 Mon Sep 17 00:00:00 2001 From: Manuel Schlund Date: Wed, 26 Aug 2026 12:28:15 +0200 Subject: [PATCH 4/8] Run pre-commit --- esmvalcore/_main.py | 1 - tests/unit/io/local/test_get_data_sources.py | 4 ---- 2 files changed, 5 deletions(-) diff --git a/esmvalcore/_main.py b/esmvalcore/_main.py index c2fa76a53c..197f19ddbe 100644 --- a/esmvalcore/_main.py +++ b/esmvalcore/_main.py @@ -33,7 +33,6 @@ import os import re import sys -import warnings from importlib.metadata import entry_points from pathlib import Path from typing import TYPE_CHECKING diff --git a/tests/unit/io/local/test_get_data_sources.py b/tests/unit/io/local/test_get_data_sources.py index 5421279264..00cc5d84a1 100644 --- a/tests/unit/io/local/test_get_data_sources.py +++ b/tests/unit/io/local/test_get_data_sources.py @@ -1,7 +1,6 @@ from __future__ import annotations from pathlib import Path -from typing import TYPE_CHECKING import pytest @@ -11,9 +10,6 @@ from esmvalcore.io.local import LocalDataSource from esmvalcore.local import _get_data_sources -if TYPE_CHECKING: - import pytest_mock - @pytest.mark.parametrize( "rootpath_drs", From eec44b55c2ce3800fa43b39f5a7c4fb2f63a8717 Mon Sep 17 00:00:00 2001 From: Manuel Schlund Date: Wed, 26 Aug 2026 13:17:22 +0200 Subject: [PATCH 5/8] Added option ignore_datetimes_in_filename to LocalDataSource --- esmvalcore/io/local.py | 167 ++++++++++++++++++------------- tests/unit/io/local/test_time.py | 65 +++++------- 2 files changed, 123 insertions(+), 109 deletions(-) diff --git a/esmvalcore/io/local.py b/esmvalcore/io/local.py index b0ae5b81d5..da7afdf62b 100644 --- a/esmvalcore/io/local.py +++ b/esmvalcore/io/local.py @@ -188,73 +188,6 @@ def _get_start_end_date_from_filename( return start_date, end_date -def _get_start_end_date(file: str | Path) -> tuple[str, str]: - """Get the start and end dates as a string from a file. - - This function first tries to read the dates from the filename and only - if that fails, it will try to read them from the content of the file. - - Parameters - ---------- - file: - The file to read the start and end data from. - - Returns - ------- - tuple[str, str] - The start and end date. - - Raises - ------ - ValueError - Start or end date cannot be determined. - """ - start_date, end_date = _get_start_end_date_from_filename(file) - - # As final resort, try to get the dates from the file contents - if ( - (start_date is None or end_date is None) - and isinstance(file, (str, Path)) - and Path(file).exists() - ): - logger.debug("Must load file %s for daterange ", file) - with Dataset(file) as dataset: - for variable in dataset.variables.values(): - var_name = _get_var_name(variable) - attrs = variable.ncattrs() - if ( - var_name == "time" - and "units" in attrs - and "calendar" in attrs - ): - time_units = Unit( - variable.getncattr("units"), - calendar=variable.getncattr("calendar"), - ) - start_date = isodate.date_isoformat( - time_units.num2date(variable[0]), - format=isodate.isostrf.DATE_BAS_COMPLETE, - ) - end_date = isodate.date_isoformat( - time_units.num2date(variable[-1]), - format=isodate.isostrf.DATE_BAS_COMPLETE, - ) - break - - if start_date is None or end_date is None: - msg = ( - f"File {file} datetimes do not match a recognized pattern and " - f"time coordinate can not be read from the file" - ) - raise ValueError(msg) - - # Remove potential '-' characters from datetimes - start_date = start_date.replace("-", "") - end_date = end_date.replace("-", "") - - return start_date, end_date - - def _dates_to_timerange(start_date: int | str, end_date: int | str) -> str: """Convert ``start_date`` and ``end_date`` to ``timerange``. @@ -562,6 +495,25 @@ class LocalDataSource(esmvalcore.io.protocol.DataSource): calling :meth:`LocalFile.to_iris`. """ + ignore_datetimes_in_filename: bool = False + """Ignore date times specified in file names. + + By default, if possible, the time range spanned by a file is determined by + datetimes given in its file name. For example, the file + ``my-model_20000101-20051231.nc`` will be assigned a start date of + 2000-01-01 and an end date of 2005-12-31. Only if reading datetimes from + the file name fails, the actual file will be opened to determine the time + range from the file contents. + + If this option is set to ``True``, date times in file names are ignored, + and the time range is always determined from the contents of the file. + + Note that in the vast majority of cases, reading datetimes from file names + works very well. In addition, opening files to determine time range is much + slower than just reading file names. Thus, this option should only be set + to ``True`` if absolutely necessary. + """ + def __post_init__(self) -> None: """Set further attributes.""" self.rootpath = Path(os.path.expandvars(self.rootpath)).expanduser() @@ -638,7 +590,84 @@ def find_data(self, **facets: FacetValue) -> list[LocalFile]: return files - def _path2facets(self, path: Path, add_timerange: bool) -> dict[str, str]: + def _get_start_end_date(self, file: Path) -> tuple[str, str]: + """Get the start and end datetimes as a string. + + This function first tries to read the datetimes from the filename and + only if that fails, it will try to read them from the contents of the + file. + + If + :attr:`~esmvalcore.io.local.LocalDataSource.ignore_datetimes_in_filename` + is set to ``True``, datetimes are always read from the contents of the + file. + + Parameters + ---------- + file: + The file to read the start and end data from. + + Returns + ------- + : + The start and end date. + + Raises + ------ + ValueError + Start or end date cannot be determined. + + """ + start_date = end_date = None + + if not self.ignore_datetimes_in_filename: + start_date, end_date = _get_start_end_date_from_filename(file) + + # Read datetimes from file contents if necessary + if (start_date is None or end_date is None) and file.exists(): + logger.debug("Must load file %s for daterange ", file) + with Dataset(file) as dataset: + for variable in dataset.variables.values(): + var_name = _get_var_name(variable) + attrs = variable.ncattrs() + if ( + var_name == "time" + and "units" in attrs + and "calendar" in attrs + ): + time_units = Unit( + variable.getncattr("units"), + calendar=variable.getncattr("calendar"), + ) + start_date = isodate.date_isoformat( + time_units.num2date(variable[0]), + format=isodate.isostrf.DATE_BAS_COMPLETE, + ) + end_date = isodate.date_isoformat( + time_units.num2date(variable[-1]), + format=isodate.isostrf.DATE_BAS_COMPLETE, + ) + break + + if start_date is None or end_date is None: + msg = ( + f"File {file} datetimes do not match a recognized pattern and " + f"time coordinate can not be read from the file" + ) + raise ValueError(msg) + + # Remove potential '-' characters from datetimes + start_date = start_date.replace("-", "") + end_date = end_date.replace("-", "") + + return start_date, end_date + + def _path2facets( + self, + path: Path, + *, + add_timerange: bool, + ) -> dict[str, str]: """Extract facets from path.""" facets: dict[str, str] = {} @@ -649,7 +678,7 @@ def _path2facets(self, path: Path, add_timerange: bool) -> dict[str, str]: if add_timerange: try: - start_date, end_date = _get_start_end_date(path) + start_date, end_date = self._get_start_end_date(path) except ValueError: pass else: diff --git a/tests/unit/io/local/test_time.py b/tests/unit/io/local/test_time.py index 19f0fc4d82..6b41fe02b5 100644 --- a/tests/unit/io/local/test_time.py +++ b/tests/unit/io/local/test_time.py @@ -3,33 +3,28 @@ from pathlib import Path import iris -import pyesgf import pytest from cf_units import Unit -from esmvalcore.io.esgf import ESGFFile from esmvalcore.io.local import ( + LocalDataSource, LocalFile, _dates_to_timerange, - _get_start_end_date, _replace_years_with_timerange, _truncate_dates, ) -def _get_esgf_file(path): - """Get ESGFFile object.""" - result = pyesgf.search.results.FileResult( - json={ - "dataset_id": "CMIP6.ABC.v1|something.org", - "dataset_id_template_": ["%(mip_era)s.%(source_id)s"], - "project": ["CMIP6"], - "size": 10, - "title": path, - }, - context=None, +@pytest.fixture +def local_data_source(): + return LocalDataSource( + name="test-source", + project="test-project", + priority=1, + rootpath="", + dirname_template="", + filename_template="", ) - return ESGFFile([result]) @pytest.mark.parametrize( @@ -72,7 +67,7 @@ def _get_esgf_file(path): ["E5sf00_1H_2000-01-01_2001-12-31_167.grb", "20000101", "20011231"], ], ) -def test_get_start_end_date(case): +def test_get_start_end_date(case, local_data_source): """Tests for _get_start_end_date function.""" filename, case_start, case_end = case @@ -80,30 +75,20 @@ def test_get_start_end_date(case): # file, which fails here because the file is not there. if case_start is None and case_end is None: with pytest.raises(ValueError): - _get_start_end_date(filename) + local_data_source._get_start_end_date(Path(filename)) with pytest.raises(ValueError): - _get_start_end_date(Path(filename)) - with pytest.raises(ValueError): - _get_start_end_date(LocalFile(filename)) - with pytest.raises(ValueError): - _get_start_end_date(_get_esgf_file(filename).name) + local_data_source._get_start_end_date(LocalFile(filename)) else: - start, end = _get_start_end_date(filename) - assert case_start == start - assert case_end == end - start, end = _get_start_end_date(Path(filename)) - assert case_start == start - assert case_end == end - start, end = _get_start_end_date(LocalFile(filename)) + start, end = local_data_source._get_start_end_date(Path(filename)) assert case_start == start assert case_end == end - start, end = _get_start_end_date(_get_esgf_file(filename).name) + start, end = local_data_source._get_start_end_date(LocalFile(filename)) assert case_start == start assert case_end == end -def test_read_years_from_cube(tmp_path): +def test_read_years_from_cube(local_data_source, tmp_path): """Try to get years from cube if no date in filename.""" temp_file = LocalFile(tmp_path / "test.nc") cube = iris.cube.Cube([0, 0, 0, 0], var_name="var") @@ -114,12 +99,12 @@ def test_read_years_from_cube(tmp_path): ) cube.add_dim_coord(time, 0) iris.save(cube, temp_file) - start, end = _get_start_end_date(temp_file) + start, end = local_data_source._get_start_end_date(temp_file) assert int(start[:4]) == 1990 assert int(end[:4]) == 1991 -def test_read_datetime_from_cube(tmp_path): +def test_read_datetime_from_cube(local_data_source, tmp_path): """Try to get datetime from cube if no date in filename.""" temp_file = tmp_path / "test.nc" cube = iris.cube.Cube([0, 0, 0, 0], var_name="var") @@ -132,12 +117,12 @@ def test_read_datetime_from_cube(tmp_path): ) cube.add_dim_coord(time, 0) iris.save(cube, temp_file) - start, end = _get_start_end_date(temp_file) + start, end = local_data_source._get_start_end_date(temp_file) assert start == "19900101" assert end == "19910102" -def test_raises_if_unable_to_deduce_no_time(tmp_path): +def test_raises_if_unable_to_deduce_no_time(local_data_source, tmp_path): """Try to get time from cube if no date in filename.""" temp_file = tmp_path / "test.nc" cube = iris.cube.Cube([0, 0], var_name="var") @@ -150,10 +135,10 @@ def test_raises_if_unable_to_deduce_no_time(tmp_path): cube.add_dim_coord(not_time, 0) iris.save(cube, temp_file) with pytest.raises(ValueError): - _get_start_end_date(temp_file) + local_data_source._get_start_end_date(temp_file) -def test_raises_if_unable_to_deduce_no_time_units(tmp_path): +def test_raises_if_unable_to_deduce_no_time_units(local_data_source, tmp_path): """Try to get time from cube if no date in filename.""" temp_file = tmp_path / "test.nc" cube = iris.cube.Cube([0, 0], var_name="var") @@ -161,13 +146,13 @@ def test_raises_if_unable_to_deduce_no_time_units(tmp_path): cube.add_dim_coord(time, 0) iris.save(cube, temp_file) with pytest.raises(ValueError): - _get_start_end_date(temp_file) + local_data_source._get_start_end_date(temp_file) -def test_fails_if_no_date_present(): +def test_fails_if_no_date_present(local_data_source): """Test raises if no date is present.""" with pytest.raises(ValueError): - _get_start_end_date("var_whatever") + local_data_source._get_start_end_date(Path("var_whatever")) def test_get_timerange_from_years(): From 1b21cb8acd4937dc61444270e67e853914ab84af Mon Sep 17 00:00:00 2001 From: Manuel Schlund Date: Wed, 26 Aug 2026 14:26:23 +0200 Subject: [PATCH 6/8] Nicer debug message --- esmvalcore/io/local.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/esmvalcore/io/local.py b/esmvalcore/io/local.py index da7afdf62b..0a24612a37 100644 --- a/esmvalcore/io/local.py +++ b/esmvalcore/io/local.py @@ -625,7 +625,16 @@ def _get_start_end_date(self, file: Path) -> tuple[str, str]: # Read datetimes from file contents if necessary if (start_date is None or end_date is None) and file.exists(): - logger.debug("Must load file %s for daterange ", file) + reason = ( + f"data source '{self.name}' is set up with ignore_datetimes_in_filename=True" + if self.ignore_datetimes_in_filename + else "it cannot be read from file name" + ) + logger.debug( + "Opening file %s to determine time range because %s", + file, + reason, + ) with Dataset(file) as dataset: for variable in dataset.variables.values(): var_name = _get_var_name(variable) From 2b1c5ca4b45c2580c96ae503b55fc0b342e7abb6 Mon Sep 17 00:00:00 2001 From: Manuel Schlund Date: Wed, 26 Aug 2026 14:26:29 +0200 Subject: [PATCH 7/8] Add test --- tests/unit/io/local/test_time.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/tests/unit/io/local/test_time.py b/tests/unit/io/local/test_time.py index 6b41fe02b5..bdc3062f31 100644 --- a/tests/unit/io/local/test_time.py +++ b/tests/unit/io/local/test_time.py @@ -122,6 +122,30 @@ def test_read_datetime_from_cube(local_data_source, tmp_path): assert end == "19910102" +def test_ignore_datetimes_in_filename(tmp_path): + data_source = LocalDataSource( + name="test-source", + project="test-project", + priority=1, + rootpath="", + dirname_template="", + filename_template="", + ignore_datetimes_in_filename=True, + ) + temp_file = LocalFile(tmp_path / "test_1850-1900.nc") + cube = iris.cube.Cube([0, 0, 0, 0], var_name="var") + time = iris.coords.DimCoord( + [0, 100, 200, 366], + standard_name="time", + units="days since 1990-01-01", + ) + cube.add_dim_coord(time, 0) + iris.save(cube, temp_file) + start, end = data_source._get_start_end_date(temp_file) + assert int(start[:4]) == 1990 + assert int(end[:4]) == 1991 + + def test_raises_if_unable_to_deduce_no_time(local_data_source, tmp_path): """Try to get time from cube if no date in filename.""" temp_file = tmp_path / "test.nc" From f684a30a238c45a5c0ca34ad243546dff16a53aa Mon Sep 17 00:00:00 2001 From: Manuel Schlund Date: Wed, 26 Aug 2026 14:44:44 +0200 Subject: [PATCH 8/8] Expand ICON docs --- doc/quickstart/find_data.rst | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/doc/quickstart/find_data.rst b/doc/quickstart/find_data.rst index 4324bcb828..5bca849990 100644 --- a/doc/quickstart/find_data.rst +++ b/doc/quickstart/find_data.rst @@ -472,6 +472,29 @@ as configured in: To use this configuration, run ``esmvaltool config copy data-native-icon.yml`` and adapt the ``rootpath`` to your system. +.. hint:: + + If your ICON output consists of files that span multiple years (e.g., the + file ``exp_19000101T000000Z.nc`` actually contains 5 years, not 1), you need + to configure your data source with ``ignore_datetimes_in_filename=True``. + + Example: + + .. code-block:: yaml + + projects: + ICON: + data: + icon: &icon + type: esmvalcore.io.local.LocalDataSource + rootpath: /path/to/my/icon/exps + dirname_template: "{exp}" + filename_template: "{exp}_{var_type}*.nc" + ignore_warnings: + - message: "Failed to create 'height' dimension coordinate: The 'height' DimCoord bounds array must be strictly monotonic." + module: iris + ignore_datetimes_in_filename: true + Currently, two different versions of ICON are supported: 1. ICON-A, which is based on ECHAM physics (deprecated): select via ``dataset: