diff --git a/doc/quickstart/find_data.rst b/doc/quickstart/find_data.rst index 4324bcb828..5bca849990 100644 --- a/doc/quickstart/find_data.rst +++ b/doc/quickstart/find_data.rst @@ -472,6 +472,29 @@ as configured in: To use this configuration, run ``esmvaltool config copy data-native-icon.yml`` and adapt the ``rootpath`` to your system. +.. hint:: + + If your ICON output consists of files that span multiple years (e.g., the + file ``exp_19000101T000000Z.nc`` actually contains 5 years, not 1), you need + to configure your data source with ``ignore_datetimes_in_filename=True``. + + Example: + + .. code-block:: yaml + + projects: + ICON: + data: + icon: &icon + type: esmvalcore.io.local.LocalDataSource + rootpath: /path/to/my/icon/exps + dirname_template: "{exp}" + filename_template: "{exp}_{var_type}*.nc" + ignore_warnings: + - message: "Failed to create 'height' dimension coordinate: The 'height' DimCoord bounds array must be strictly monotonic." + module: iris + ignore_datetimes_in_filename: true + Currently, two different versions of ICON are supported: 1. ICON-A, which is based on ECHAM physics (deprecated): select via ``dataset: diff --git a/esmvalcore/io/local.py b/esmvalcore/io/local.py index b0ae5b81d5..0a24612a37 100644 --- a/esmvalcore/io/local.py +++ b/esmvalcore/io/local.py @@ -188,73 +188,6 @@ def _get_start_end_date_from_filename( return start_date, end_date -def _get_start_end_date(file: str | Path) -> tuple[str, str]: - """Get the start and end dates as a string from a file. - - This function first tries to read the dates from the filename and only - if that fails, it will try to read them from the content of the file. - - Parameters - ---------- - file: - The file to read the start and end data from. - - Returns - ------- - tuple[str, str] - The start and end date. - - Raises - ------ - ValueError - Start or end date cannot be determined. - """ - start_date, end_date = _get_start_end_date_from_filename(file) - - # As final resort, try to get the dates from the file contents - if ( - (start_date is None or end_date is None) - and isinstance(file, (str, Path)) - and Path(file).exists() - ): - logger.debug("Must load file %s for daterange ", file) - with Dataset(file) as dataset: - for variable in dataset.variables.values(): - var_name = _get_var_name(variable) - attrs = variable.ncattrs() - if ( - var_name == "time" - and "units" in attrs - and "calendar" in attrs - ): - time_units = Unit( - variable.getncattr("units"), - calendar=variable.getncattr("calendar"), - ) - start_date = isodate.date_isoformat( - time_units.num2date(variable[0]), - format=isodate.isostrf.DATE_BAS_COMPLETE, - ) - end_date = isodate.date_isoformat( - time_units.num2date(variable[-1]), - format=isodate.isostrf.DATE_BAS_COMPLETE, - ) - break - - if start_date is None or end_date is None: - msg = ( - f"File {file} datetimes do not match a recognized pattern and " - f"time coordinate can not be read from the file" - ) - raise ValueError(msg) - - # Remove potential '-' characters from datetimes - start_date = start_date.replace("-", "") - end_date = end_date.replace("-", "") - - return start_date, end_date - - def _dates_to_timerange(start_date: int | str, end_date: int | str) -> str: """Convert ``start_date`` and ``end_date`` to ``timerange``. @@ -562,6 +495,25 @@ class LocalDataSource(esmvalcore.io.protocol.DataSource): calling :meth:`LocalFile.to_iris`. """ + ignore_datetimes_in_filename: bool = False + """Ignore date times specified in file names. + + By default, if possible, the time range spanned by a file is determined by + datetimes given in its file name. For example, the file + ``my-model_20000101-20051231.nc`` will be assigned a start date of + 2000-01-01 and an end date of 2005-12-31. Only if reading datetimes from + the file name fails, the actual file will be opened to determine the time + range from the file contents. + + If this option is set to ``True``, date times in file names are ignored, + and the time range is always determined from the contents of the file. + + Note that in the vast majority of cases, reading datetimes from file names + works very well. In addition, opening files to determine time range is much + slower than just reading file names. Thus, this option should only be set + to ``True`` if absolutely necessary. + """ + def __post_init__(self) -> None: """Set further attributes.""" self.rootpath = Path(os.path.expandvars(self.rootpath)).expanduser() @@ -638,7 +590,93 @@ def find_data(self, **facets: FacetValue) -> list[LocalFile]: return files - def _path2facets(self, path: Path, add_timerange: bool) -> dict[str, str]: + def _get_start_end_date(self, file: Path) -> tuple[str, str]: + """Get the start and end datetimes as a string. + + This function first tries to read the datetimes from the filename and + only if that fails, it will try to read them from the contents of the + file. + + If + :attr:`~esmvalcore.io.local.LocalDataSource.ignore_datetimes_in_filename` + is set to ``True``, datetimes are always read from the contents of the + file. + + Parameters + ---------- + file: + The file to read the start and end data from. + + Returns + ------- + : + The start and end date. + + Raises + ------ + ValueError + Start or end date cannot be determined. + + """ + start_date = end_date = None + + if not self.ignore_datetimes_in_filename: + start_date, end_date = _get_start_end_date_from_filename(file) + + # Read datetimes from file contents if necessary + if (start_date is None or end_date is None) and file.exists(): + reason = ( + f"data source '{self.name}' is set up with ignore_datetimes_in_filename=True" + if self.ignore_datetimes_in_filename + else "it cannot be read from file name" + ) + logger.debug( + "Opening file %s to determine time range because %s", + file, + reason, + ) + with Dataset(file) as dataset: + for variable in dataset.variables.values(): + var_name = _get_var_name(variable) + attrs = variable.ncattrs() + if ( + var_name == "time" + and "units" in attrs + and "calendar" in attrs + ): + time_units = Unit( + variable.getncattr("units"), + calendar=variable.getncattr("calendar"), + ) + start_date = isodate.date_isoformat( + time_units.num2date(variable[0]), + format=isodate.isostrf.DATE_BAS_COMPLETE, + ) + end_date = isodate.date_isoformat( + time_units.num2date(variable[-1]), + format=isodate.isostrf.DATE_BAS_COMPLETE, + ) + break + + if start_date is None or end_date is None: + msg = ( + f"File {file} datetimes do not match a recognized pattern and " + f"time coordinate can not be read from the file" + ) + raise ValueError(msg) + + # Remove potential '-' characters from datetimes + start_date = start_date.replace("-", "") + end_date = end_date.replace("-", "") + + return start_date, end_date + + def _path2facets( + self, + path: Path, + *, + add_timerange: bool, + ) -> dict[str, str]: """Extract facets from path.""" facets: dict[str, str] = {} @@ -649,7 +687,7 @@ def _path2facets(self, path: Path, add_timerange: bool) -> dict[str, str]: if add_timerange: try: - start_date, end_date = _get_start_end_date(path) + start_date, end_date = self._get_start_end_date(path) except ValueError: pass else: diff --git a/tests/unit/io/local/test_time.py b/tests/unit/io/local/test_time.py index 19f0fc4d82..bdc3062f31 100644 --- a/tests/unit/io/local/test_time.py +++ b/tests/unit/io/local/test_time.py @@ -3,33 +3,28 @@ from pathlib import Path import iris -import pyesgf import pytest from cf_units import Unit -from esmvalcore.io.esgf import ESGFFile from esmvalcore.io.local import ( + LocalDataSource, LocalFile, _dates_to_timerange, - _get_start_end_date, _replace_years_with_timerange, _truncate_dates, ) -def _get_esgf_file(path): - """Get ESGFFile object.""" - result = pyesgf.search.results.FileResult( - json={ - "dataset_id": "CMIP6.ABC.v1|something.org", - "dataset_id_template_": ["%(mip_era)s.%(source_id)s"], - "project": ["CMIP6"], - "size": 10, - "title": path, - }, - context=None, +@pytest.fixture +def local_data_source(): + return LocalDataSource( + name="test-source", + project="test-project", + priority=1, + rootpath="", + dirname_template="", + filename_template="", ) - return ESGFFile([result]) @pytest.mark.parametrize( @@ -72,7 +67,7 @@ def _get_esgf_file(path): ["E5sf00_1H_2000-01-01_2001-12-31_167.grb", "20000101", "20011231"], ], ) -def test_get_start_end_date(case): +def test_get_start_end_date(case, local_data_source): """Tests for _get_start_end_date function.""" filename, case_start, case_end = case @@ -80,30 +75,20 @@ def test_get_start_end_date(case): # file, which fails here because the file is not there. if case_start is None and case_end is None: with pytest.raises(ValueError): - _get_start_end_date(filename) + local_data_source._get_start_end_date(Path(filename)) with pytest.raises(ValueError): - _get_start_end_date(Path(filename)) - with pytest.raises(ValueError): - _get_start_end_date(LocalFile(filename)) - with pytest.raises(ValueError): - _get_start_end_date(_get_esgf_file(filename).name) + local_data_source._get_start_end_date(LocalFile(filename)) else: - start, end = _get_start_end_date(filename) - assert case_start == start - assert case_end == end - start, end = _get_start_end_date(Path(filename)) - assert case_start == start - assert case_end == end - start, end = _get_start_end_date(LocalFile(filename)) + start, end = local_data_source._get_start_end_date(Path(filename)) assert case_start == start assert case_end == end - start, end = _get_start_end_date(_get_esgf_file(filename).name) + start, end = local_data_source._get_start_end_date(LocalFile(filename)) assert case_start == start assert case_end == end -def test_read_years_from_cube(tmp_path): +def test_read_years_from_cube(local_data_source, tmp_path): """Try to get years from cube if no date in filename.""" temp_file = LocalFile(tmp_path / "test.nc") cube = iris.cube.Cube([0, 0, 0, 0], var_name="var") @@ -114,12 +99,12 @@ def test_read_years_from_cube(tmp_path): ) cube.add_dim_coord(time, 0) iris.save(cube, temp_file) - start, end = _get_start_end_date(temp_file) + start, end = local_data_source._get_start_end_date(temp_file) assert int(start[:4]) == 1990 assert int(end[:4]) == 1991 -def test_read_datetime_from_cube(tmp_path): +def test_read_datetime_from_cube(local_data_source, tmp_path): """Try to get datetime from cube if no date in filename.""" temp_file = tmp_path / "test.nc" cube = iris.cube.Cube([0, 0, 0, 0], var_name="var") @@ -132,12 +117,36 @@ def test_read_datetime_from_cube(tmp_path): ) cube.add_dim_coord(time, 0) iris.save(cube, temp_file) - start, end = _get_start_end_date(temp_file) + start, end = local_data_source._get_start_end_date(temp_file) assert start == "19900101" assert end == "19910102" -def test_raises_if_unable_to_deduce_no_time(tmp_path): +def test_ignore_datetimes_in_filename(tmp_path): + data_source = LocalDataSource( + name="test-source", + project="test-project", + priority=1, + rootpath="", + dirname_template="", + filename_template="", + ignore_datetimes_in_filename=True, + ) + temp_file = LocalFile(tmp_path / "test_1850-1900.nc") + cube = iris.cube.Cube([0, 0, 0, 0], var_name="var") + time = iris.coords.DimCoord( + [0, 100, 200, 366], + standard_name="time", + units="days since 1990-01-01", + ) + cube.add_dim_coord(time, 0) + iris.save(cube, temp_file) + start, end = data_source._get_start_end_date(temp_file) + assert int(start[:4]) == 1990 + assert int(end[:4]) == 1991 + + +def test_raises_if_unable_to_deduce_no_time(local_data_source, tmp_path): """Try to get time from cube if no date in filename.""" temp_file = tmp_path / "test.nc" cube = iris.cube.Cube([0, 0], var_name="var") @@ -150,10 +159,10 @@ def test_raises_if_unable_to_deduce_no_time(tmp_path): cube.add_dim_coord(not_time, 0) iris.save(cube, temp_file) with pytest.raises(ValueError): - _get_start_end_date(temp_file) + local_data_source._get_start_end_date(temp_file) -def test_raises_if_unable_to_deduce_no_time_units(tmp_path): +def test_raises_if_unable_to_deduce_no_time_units(local_data_source, tmp_path): """Try to get time from cube if no date in filename.""" temp_file = tmp_path / "test.nc" cube = iris.cube.Cube([0, 0], var_name="var") @@ -161,13 +170,13 @@ def test_raises_if_unable_to_deduce_no_time_units(tmp_path): cube.add_dim_coord(time, 0) iris.save(cube, temp_file) with pytest.raises(ValueError): - _get_start_end_date(temp_file) + local_data_source._get_start_end_date(temp_file) -def test_fails_if_no_date_present(): +def test_fails_if_no_date_present(local_data_source): """Test raises if no date is present.""" with pytest.raises(ValueError): - _get_start_end_date("var_whatever") + local_data_source._get_start_end_date(Path("var_whatever")) def test_get_timerange_from_years():