diff --git a/changelog/893.feature.md b/changelog/893.feature.md new file mode 100644 index 000000000..a42578118 --- /dev/null +++ b/changelog/893.feature.md @@ -0,0 +1 @@ +Accepts esm-hist alongside historical in diagnostic data requirements. diff --git a/changelog/901.fix.md b/changelog/901.fix.md new file mode 100644 index 000000000..9a27ca09d --- /dev/null +++ b/changelog/901.fix.md @@ -0,0 +1,2 @@ +The PMP provider now installs `pcmdi_metrics` from a fork that chunks the SVD in the variability modes analysis. +This cuts the runtime of the extratropical modes of variability diagnostics by roughly 40x. diff --git a/packages/climate-ref-esmvaltool/src/climate_ref_esmvaltool/diagnostics/climate_drivers_for_fire.py b/packages/climate-ref-esmvaltool/src/climate_ref_esmvaltool/diagnostics/climate_drivers_for_fire.py index 44e8ada90..e45c00ed3 100644 --- a/packages/climate-ref-esmvaltool/src/climate_ref_esmvaltool/diagnostics/climate_drivers_for_fire.py +++ b/packages/climate-ref-esmvaltool/src/climate_ref_esmvaltool/diagnostics/climate_drivers_for_fire.py @@ -34,21 +34,21 @@ class ClimateDriversForFire(ESMValToolDiagnostic): FacetFilter( { "variable_id": ("hurs", "pr", "tas", "tasmax"), - "experiment_id": "historical", + "experiment_id": ("historical", "esm-hist"), "table_id": "Amon", } ), FacetFilter( { "variable_id": ("cVeg", "treeFrac"), - "experiment_id": "historical", + "experiment_id": ("historical", "esm-hist"), "table_id": "Lmon", } ), FacetFilter( { "variable_id": "vegFrac", - "experiment_id": "historical", + "experiment_id": ("historical", "esm-hist"), "table_id": "Emon", } ), @@ -89,7 +89,7 @@ class ClimateDriversForFire(ESMValToolDiagnostic): "tas_tavg-h2m-hxy-u", "tas_tmaxavg-h2m-hxy-u", ), - "experiment_id": "historical", + "experiment_id": ("historical", "esm-hist"), "frequency": "mon", "region": "glb", } @@ -101,7 +101,7 @@ class ClimateDriversForFire(ESMValToolDiagnostic): "treeFrac_tavg-u-hxy-u", "vegFrac_tavg-u-hxy-u", ), - "experiment_id": "historical", + "experiment_id": ("historical", "esm-hist"), "frequency": "mon", "region": "glb", } diff --git a/packages/climate-ref-esmvaltool/src/climate_ref_esmvaltool/diagnostics/cloud_radiative_effects.py b/packages/climate-ref-esmvaltool/src/climate_ref_esmvaltool/diagnostics/cloud_radiative_effects.py index 7ea54c6a0..856014b06 100644 --- a/packages/climate-ref-esmvaltool/src/climate_ref_esmvaltool/diagnostics/cloud_radiative_effects.py +++ b/packages/climate-ref-esmvaltool/src/climate_ref_esmvaltool/diagnostics/cloud_radiative_effects.py @@ -77,7 +77,7 @@ class CloudRadiativeEffects(ESMValToolDiagnostic): FacetFilter( facets={ "variable_id": variables, - "experiment_id": "historical", + "experiment_id": ("historical", "esm-hist"), "table_id": "Amon", } ), @@ -108,7 +108,7 @@ class CloudRadiativeEffects(ESMValToolDiagnostic): "rsut_tavg-u-hxy-u", "rsutcs_tavg-u-hxy-u", ), - "experiment_id": "historical", + "experiment_id": ("historical", "esm-hist"), "frequency": "mon", "region": "glb", } diff --git a/packages/climate-ref-esmvaltool/src/climate_ref_esmvaltool/diagnostics/cloud_scatterplots.py b/packages/climate-ref-esmvaltool/src/climate_ref_esmvaltool/diagnostics/cloud_scatterplots.py index 9ab494013..8bf767a8c 100644 --- a/packages/climate-ref-esmvaltool/src/climate_ref_esmvaltool/diagnostics/cloud_scatterplots.py +++ b/packages/climate-ref-esmvaltool/src/climate_ref_esmvaltool/diagnostics/cloud_scatterplots.py @@ -25,7 +25,7 @@ def get_cmip_data_requirements( ) -> tuple[tuple[DataRequirement, ...], ...]: """Create data requirements for CMIP6 and CMIP7 data.""" cmip7_facets: dict[str, str | Collection[str]] = { - "experiment_id": "historical", + "experiment_id": ("historical", "esm-hist"), "frequency": "mon", "region": "glb", } @@ -39,7 +39,7 @@ def get_cmip_data_requirements( FacetFilter( facets={ "variable_id": variables, - "experiment_id": "historical", + "experiment_id": ("historical", "esm-hist"), "table_id": "Amon", }, ), diff --git a/packages/climate-ref-esmvaltool/src/climate_ref_esmvaltool/diagnostics/enso.py b/packages/climate-ref-esmvaltool/src/climate_ref_esmvaltool/diagnostics/enso.py index 4e775e2e3..8c28dc463 100644 --- a/packages/climate-ref-esmvaltool/src/climate_ref_esmvaltool/diagnostics/enso.py +++ b/packages/climate-ref-esmvaltool/src/climate_ref_esmvaltool/diagnostics/enso.py @@ -38,14 +38,14 @@ class ENSOBasicClimatology(ESMValToolDiagnostic): FacetFilter( facets={ "variable_id": ("pr", "tauu"), - "experiment_id": "historical", + "experiment_id": ("historical", "esm-hist"), "table_id": "Amon", }, ), FacetFilter( facets={ "variable_id": "tos", - "experiment_id": "historical", + "experiment_id": ("historical", "esm-hist"), "table_id": "Omon", }, ), @@ -75,7 +75,7 @@ class ENSOBasicClimatology(ESMValToolDiagnostic): "pr_tavg-u-hxy-u", "tauu_tavg-u-hxy-u", ), - "experiment_id": "historical", + "experiment_id": ("historical", "esm-hist"), "frequency": "mon", "region": "glb", }, @@ -83,7 +83,7 @@ class ENSOBasicClimatology(ESMValToolDiagnostic): FacetFilter( facets={ "branded_variable": "tos_tavg-u-hxy-sea", - "experiment_id": "historical", + "experiment_id": ("historical", "esm-hist"), "frequency": "mon", "region": "glb", }, @@ -310,7 +310,7 @@ class ENSOCharacteristics(ESMValToolDiagnostic): FacetFilter( facets={ "variable_id": "tos", - "experiment_id": "historical", + "experiment_id": ("historical", "esm-hist"), "table_id": "Omon", }, ), @@ -331,7 +331,7 @@ class ENSOCharacteristics(ESMValToolDiagnostic): FacetFilter( facets={ "branded_variable": "tos_tavg-u-hxy-sea", - "experiment_id": "historical", + "experiment_id": ("historical", "esm-hist"), "frequency": "mon", "region": "glb", }, diff --git a/packages/climate-ref-esmvaltool/src/climate_ref_esmvaltool/diagnostics/ozone.py b/packages/climate-ref-esmvaltool/src/climate_ref_esmvaltool/diagnostics/ozone.py index 298afba3c..5979babfd 100644 --- a/packages/climate-ref-esmvaltool/src/climate_ref_esmvaltool/diagnostics/ozone.py +++ b/packages/climate-ref-esmvaltool/src/climate_ref_esmvaltool/diagnostics/ozone.py @@ -33,7 +33,7 @@ FacetFilter( facets={ "variable_id": "toz", - "experiment_id": "historical", + "experiment_id": ("historical", "esm-hist"), "table_id": "AERmon", }, ), @@ -69,7 +69,7 @@ FacetFilter( facets={ "variable_id": "toz", - "experiment_id": "historical", + "experiment_id": ("historical", "esm-hist"), "branded_variable": "toz_tavg-u-hxy-u", "frequency": "mon", "region": "glb", @@ -307,7 +307,7 @@ class O3ZonalMeanProfiles(ESMValToolDiagnostic): FacetFilter( facets={ "variable_id": "o3", - "experiment_id": "historical", + "experiment_id": ("historical", "esm-hist"), "table_id": "Amon", }, ), diff --git a/packages/climate-ref-esmvaltool/src/climate_ref_esmvaltool/diagnostics/regional_historical_changes.py b/packages/climate-ref-esmvaltool/src/climate_ref_esmvaltool/diagnostics/regional_historical_changes.py index 68d6941ad..a097b62eb 100644 --- a/packages/climate-ref-esmvaltool/src/climate_ref_esmvaltool/diagnostics/regional_historical_changes.py +++ b/packages/climate-ref-esmvaltool/src/climate_ref_esmvaltool/diagnostics/regional_historical_changes.py @@ -130,7 +130,7 @@ class RegionalHistoricalAnnualCycle(ESMValToolDiagnostic): FacetFilter( facets={ "variable_id": variables, - "experiment_id": "historical", + "experiment_id": ("historical", "esm-hist"), "table_id": "Amon", }, ), @@ -159,7 +159,7 @@ class RegionalHistoricalAnnualCycle(ESMValToolDiagnostic): "tas_tavg-h2m-hxy-u", "ua_tavg-p19-hxy-air", ), - "experiment_id": "historical", + "experiment_id": ("historical", "esm-hist"), "frequency": "mon", }, ), @@ -415,7 +415,7 @@ class RegionalHistoricalTimeSeries(RegionalHistoricalAnnualCycle): FacetFilter( facets={ "variable_id": variables, - "experiment_id": "historical", + "experiment_id": ("historical", "esm-hist"), "table_id": "Amon", }, ), @@ -444,7 +444,7 @@ class RegionalHistoricalTimeSeries(RegionalHistoricalAnnualCycle): "tas_tavg-h2m-hxy-u", "ua_tavg-p19-hxy-air", ), - "experiment_id": "historical", + "experiment_id": ("historical", "esm-hist"), "frequency": "mon", "region": "glb", }, @@ -671,7 +671,7 @@ class RegionalHistoricalTrend(ESMValToolDiagnostic): FacetFilter( facets={ "variable_id": variables, - "experiment_id": "historical", + "experiment_id": ("historical", "esm-hist"), "table_id": "Amon", }, ), @@ -700,7 +700,7 @@ class RegionalHistoricalTrend(ESMValToolDiagnostic): "tas_tavg-h2m-hxy-u", "ua_tavg-p19-hxy-air", ), - "experiment_id": "historical", + "experiment_id": ("historical", "esm-hist"), "frequency": "mon", }, ), diff --git a/packages/climate-ref-esmvaltool/src/climate_ref_esmvaltool/diagnostics/sea_ice_area_basic.py b/packages/climate-ref-esmvaltool/src/climate_ref_esmvaltool/diagnostics/sea_ice_area_basic.py index 8ced6d77b..8df617518 100644 --- a/packages/climate-ref-esmvaltool/src/climate_ref_esmvaltool/diagnostics/sea_ice_area_basic.py +++ b/packages/climate-ref-esmvaltool/src/climate_ref_esmvaltool/diagnostics/sea_ice_area_basic.py @@ -98,7 +98,7 @@ class SeaIceAreaBasic(ESMValToolDiagnostic): FacetFilter( facets={ "variable_id": "siconc", - "experiment_id": "historical", + "experiment_id": ("historical", "esm-hist"), "table_id": "SImon", }, ), @@ -123,7 +123,7 @@ class SeaIceAreaBasic(ESMValToolDiagnostic): FacetFilter( facets={ "branded_variable": "siconc_tavg-u-hxy-u", - "experiment_id": "historical", + "experiment_id": ("historical", "esm-hist"), "frequency": "mon", "region": "glb", }, diff --git a/packages/climate-ref-esmvaltool/src/climate_ref_esmvaltool/diagnostics/sea_ice_sensitivity.py b/packages/climate-ref-esmvaltool/src/climate_ref_esmvaltool/diagnostics/sea_ice_sensitivity.py index bf3120076..c75e17047 100644 --- a/packages/climate-ref-esmvaltool/src/climate_ref_esmvaltool/diagnostics/sea_ice_sensitivity.py +++ b/packages/climate-ref-esmvaltool/src/climate_ref_esmvaltool/diagnostics/sea_ice_sensitivity.py @@ -39,14 +39,14 @@ class SeaIceSensitivity(ESMValToolDiagnostic): FacetFilter( facets={ "variable_id": "siconc", - "experiment_id": "historical", + "experiment_id": ("historical", "esm-hist"), "table_id": "SImon", }, ), FacetFilter( facets={ "variable_id": "tas", - "experiment_id": "historical", + "experiment_id": ("historical", "esm-hist"), "table_id": "Amon", }, ), @@ -83,7 +83,7 @@ class SeaIceSensitivity(ESMValToolDiagnostic): FacetFilter( facets={ "branded_variable": "siconc_tavg-u-hxy-u", - "experiment_id": "historical", + "experiment_id": ("historical", "esm-hist"), "frequency": "mon", "region": "glb", }, @@ -91,7 +91,7 @@ class SeaIceSensitivity(ESMValToolDiagnostic): FacetFilter( facets={ "branded_variable": "tas_tavg-h2m-hxy-u", - "experiment_id": "historical", + "experiment_id": ("historical", "esm-hist"), "frequency": "mon", "region": "glb", }, diff --git a/packages/climate-ref-example/src/climate_ref_example/surface_temperature.py b/packages/climate-ref-example/src/climate_ref_example/surface_temperature.py index d73ad4607..7b362df0b 100644 --- a/packages/climate-ref-example/src/climate_ref_example/surface_temperature.py +++ b/packages/climate-ref-example/src/climate_ref_example/surface_temperature.py @@ -338,7 +338,7 @@ class GlobalMeanSurfaceTemperatureBias(Diagnostic): FacetFilter( facets={ "variable_id": (_MODEL_VARIABLE,), - "experiment_id": ("historical",), + "experiment_id": ("historical", "esm-hist"), } ), ), @@ -358,7 +358,7 @@ class GlobalMeanSurfaceTemperatureBias(Diagnostic): FacetFilter( facets={ "variable_id": (_MODEL_VARIABLE,), - "experiment_id": ("historical",), + "experiment_id": ("historical", "esm-hist"), } ), ), diff --git a/packages/climate-ref-ilamb/src/climate_ref_ilamb/standard.py b/packages/climate-ref-ilamb/src/climate_ref_ilamb/standard.py index 921dd9051..686a93873 100644 --- a/packages/climate-ref-ilamb/src/climate_ref_ilamb/standard.py +++ b/packages/climate-ref-ilamb/src/climate_ref_ilamb/standard.py @@ -914,7 +914,7 @@ def __init__( # noqa: PLR0915, PLR0912 filters={ "variable_id": all_variable_ids, "frequency": "mon", - "experiment_id": ("historical", "land-hist"), + "experiment_id": ("historical", "esm-hist", "land-hist"), "table_id": ( "AERmonZ", "Amon", @@ -938,7 +938,7 @@ def __init__( # noqa: PLR0915, PLR0912 filters={ "branded_variable": branded_variables, "frequency": "mon", - "experiment_id": ("historical", "land-hist"), + "experiment_id": ("historical", "esm-hist", "land-hist"), "region": "glb", }, group_by=("experiment_id", "source_id", "variant_label", "grid_label"), diff --git a/packages/climate-ref-pmp/src/climate_ref_pmp/__init__.py b/packages/climate-ref-pmp/src/climate_ref_pmp/__init__.py index 18543b265..1fd030962 100644 --- a/packages/climate-ref-pmp/src/climate_ref_pmp/__init__.py +++ b/packages/climate-ref-pmp/src/climate_ref_pmp/__init__.py @@ -30,6 +30,11 @@ _REGISTRY_NAME = "pmp-climatology" +# A fork of pcmdi_metrics 3.9.2 that chunks the variability-modes SVD, cutting its runtime by roughly 40x. +# Drop the pin once that lands upstream, and re-cut it if the conda pin in environment.yml moves. +_PMP_COMMIT = "d0a79e54d02f89310852f9210dd3287e3e065e31" +_PMP_URL = f"git+https://github.com/lewisjared/pcmdi_metrics.git@{_PMP_COMMIT}" + # Create the PMP diagnostics provider # PMP uses a conda environment to run the diagnostics @@ -120,6 +125,7 @@ def ingest_data(self, config: Config, db: Any) -> None: provider = PMPDiagnosticProvider("PMP", __version__) +provider.pip_packages = [_PMP_URL] # Annual cycle diagnostics and metrics diff --git a/packages/climate-ref-pmp/src/climate_ref_pmp/diagnostics/annual_cycle.py b/packages/climate-ref-pmp/src/climate_ref_pmp/diagnostics/annual_cycle.py index 2ca9c0815..6d8a65cff 100644 --- a/packages/climate-ref-pmp/src/climate_ref_pmp/diagnostics/annual_cycle.py +++ b/packages/climate-ref-pmp/src/climate_ref_pmp/diagnostics/annual_cycle.py @@ -82,7 +82,7 @@ def make_data_requirement( FacetFilter( facets={ "frequency": "mon", - "experiment_id": ("amip", "historical", "hist-GHG"), + "experiment_id": ("amip", "historical", "esm-hist", "hist-GHG"), "variable_id": (variable_id,), } ), @@ -92,7 +92,7 @@ def make_data_requirement( FacetFilter( facets={ "branded_variable": (_BRANDED_VARIABLE_NAMES[variable_id],), - "experiment_id": ("amip", "historical", "hist-GHG"), + "experiment_id": ("amip", "historical", "esm-hist", "hist-GHG"), "frequency": "mon", "region": "glb", } @@ -291,7 +291,7 @@ class AnnualCycle(CommandLineDiagnostic): "statistic", "season", ) - version = 5 + version = 6 _variable_obs_pairs = ( # ERA-5 as reference dataset, spatial 2-D variables diff --git a/packages/climate-ref-pmp/src/climate_ref_pmp/diagnostics/enso.py b/packages/climate-ref-pmp/src/climate_ref_pmp/diagnostics/enso.py index 3c2a11ca7..7bc072025 100644 --- a/packages/climate-ref-pmp/src/climate_ref_pmp/diagnostics/enso.py +++ b/packages/climate-ref-pmp/src/climate_ref_pmp/diagnostics/enso.py @@ -43,7 +43,7 @@ class ENSO(CommandLineDiagnostic): reconstruction_inputs = PMP_RECONSTRUCTION_INPUTS - version = 2 + version = 3 facets = ( "kind", @@ -56,7 +56,9 @@ class ENSO(CommandLineDiagnostic): "reference_source_id", ) - def __init__(self, metrics_collection: str, experiments: Collection[str] = ("historical",)) -> None: + def __init__( + self, metrics_collection: str, experiments: Collection[str] = ("historical", "esm-hist") + ) -> None: self.name = metrics_collection self.slug = metrics_collection.lower() self.metrics_collection = metrics_collection @@ -193,7 +195,7 @@ def __init__(self, metrics_collection: str, experiments: Collection[str] = ("his def _get_data_requirements( self, - experiments: Collection[str] = ("historical",), + experiments: Collection[str] = ("historical", "esm-hist"), ) -> tuple[tuple[DataRequirement, DataRequirement], ...]: cmip6_filters = [ FacetFilter( diff --git a/packages/climate-ref-pmp/src/climate_ref_pmp/diagnostics/variability_modes.py b/packages/climate-ref-pmp/src/climate_ref_pmp/diagnostics/variability_modes.py index 2ffdc817c..307c47ee9 100644 --- a/packages/climate-ref-pmp/src/climate_ref_pmp/diagnostics/variability_modes.py +++ b/packages/climate-ref-pmp/src/climate_ref_pmp/diagnostics/variability_modes.py @@ -40,7 +40,7 @@ class ExtratropicalModesOfVariability(CommandLineDiagnostic): ts_modes = ("PDO", "NPGO", "AMO") psl_modes = ("NAO", "NAM", "PNA", "NPO", "SAM") - version = 3 + version = 4 facets = ( "kind", @@ -71,7 +71,7 @@ def _get_data_requirements( FacetFilter( facets={ "frequency": "mon", - "experiment_id": ("historical", "hist-GHG", *extra_experiments), + "experiment_id": ("historical", "esm-hist", "hist-GHG", *extra_experiments), "variable_id": model_variable, } ) @@ -81,7 +81,7 @@ def _get_data_requirements( FacetFilter( facets={ "branded_variable": (_BRANDED_VARIABLE_NAMES[model_variable],), - "experiment_id": ("historical", "hist-GHG", *extra_experiments), + "experiment_id": ("historical", "esm-hist", "hist-GHG", *extra_experiments), "frequency": "mon", "region": "glb", }